How can I use the AWS SDK v2 for Go with DigitalOcean Spaces?

Question

I'm trying to use the AWS v2 SDK for Go to list all objects in a given bucket on DigitalOcean Spaces. Their documentation gives examples of how to use the v1 SDK to do this, but my app uses v2. I know I could technically use both, but I'd rather not if possible.

Here's what I've got so far:

package main

import (
    "context"
    "fmt"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

func main() {

    customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
        return aws.Endpoint{
            URL: "https://sfo2.digitaloceanspaces.com",
        }, nil
    })
    cfg, err := config.LoadDefaultConfig(
        context.TODO(),
        config.WithRegion("us-east-1"),
        config.WithEndpointResolverWithOptions(customResolver),
        config.WithCredentialsProvider(aws.AnonymousCredentials{}),
    )
    if err != nil {
        fmt.Println(err)
    }

    s3Client := s3.NewFromConfig(cfg)

    var continuationToken *string
    continuationToken = nil

    for {
        output, err := s3Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
            Bucket:            aws.String("stats"),
            ContinuationToken: continuationToken},
        )
        if err != nil {
            fmt.Println(err)
        }

        for _, obj := range output.Contents {
            fmt.Println(obj)
        }

        if output.IsTruncated == false {
            break
        }

        continuationToken = output.ContinuationToken
    }
}

This is the error I'm getting:

operation error S3: ListObjectsV2, https response error StatusCode: 400, RequestID: tx0000000000000051339d4-00620701db-2174fe1c-sfo2a, HostID: 2174fe1c-sfo2a-sfo, api error InvalidArgument: UnknownError

The error seems to indicate there's something wrong with my request but I don't know what.

score 0 · Answer 1 · answered Nov 02 '22 at 03:59

For pagination i think you need to do it via a pagination function like this

    // Create the Paginator for the ListObjectsV2 operation.
    p := s3.NewListObjectsV2Paginator(client, params, func(o *s3.ListObjectsV2PaginatorOptions) {
        if v := int32(maxKeys); v != 0 {
            o.Limit = v
        }
    })

Here's a fully working example I'm using to read from a digital ocean spaces bucket

package s3

import (
    "context"
    "os"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/credentials"
    "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

func read(ctx context.Context) error {
    // Define the parameters for the session you want to create.

    spacesKey := os.Getenv("SPACES_KEY")
    spacesSecret := os.Getenv("SPACES_SECRET")

    creds := credentials.NewStaticCredentialsProvider(spacesKey, spacesSecret, "")

    customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
        return aws.Endpoint{
            URL: "https://sfo3.digitaloceanspaces.com",
        }, nil
    })
    cfg, err := config.LoadDefaultConfig(ctx,
        config.WithRegion("us-east-1"),
        config.WithCredentialsProvider(creds),
        config.WithEndpointResolverWithOptions(customResolver))
    if err != nil {
        return err
    }
    // Create an Amazon S3 service client
    awsS3Client := s3.NewFromConfig(cfg)
    input := &s3.GetObjectInput{
        Bucket: aws.String("zeus-fyi"),
        Key:    aws.String("test.txt"),
    }
    downloader := manager.NewDownloader(awsS3Client)
    newFile, err := os.Create("./local-test.txt")
    if err != nil {
        return err
    }
    defer newFile.Close()
    _, err = downloader.Download(ctx, newFile, input)
    if err != nil {
        return err
    }
    return err
}

How can I use the AWS SDK v2 for Go with DigitalOcean Spaces?

1 Answers1