Multipart Upload S3 using AWS C++ SDK

Question

I am trying to upload a file to S3 using multipart upload feature in AWS C++ SDK. I could find examples for JAVA, .NET, PHP, RUBY and Rest API, but didnt find any lead on how to do it in C++. Can you please provide me a direction to achieve the same.

Answer 1

transfer manager needs the file to be stored on disk and have a filename. That's not good for streaming. Here's the code template that I used to do prototyping.

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/core/utils/HashingUtils.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/UploadPartRequest.h>

#include <sstream>
#include <iostream>
#include <string>

using namespace Aws::S3::Model;

int main(int argc, char *argv[]) {
    // new api
    Aws::SDKOptions options;
    Aws::InitAPI(options);

    // use default credential provider chains
    Aws::Client::ClientConfiguration clientConfiguration;
    clientConfiguration.region = "<your-region>";
    clientConfiguration.endpointOverride = "<endpoint-override>";
    Aws::S3::S3Client s3_client(clientConfiguration);

    std::string bucket = "<bucket>";
    std::string key = "<key>";

    // initiate the process
    Aws::S3::Model::CreateMultipartUploadRequest create_request;
    create_request.SetBucket(bucket.c_str());
    create_request.SetKey(key.c_str());
    create_request.SetContentType("text/plain");

    auto createMultipartUploadOutcome =
      s3_client.CreateMultipartUpload(create_request);

    std::string upload_id = createMultipartUploadOutcome.GetResult().GetUploadId();
    std::cout << "multiplarts upload id is:" << upload_id << std::endl;

    // start upload
    Aws::S3::Model::UploadPartRequest my_request;
    my_request.SetBucket(bucket.c_str());
    my_request.SetKey(key.c_str());
    my_request.SetPartNumber(1);
    my_request.SetUploadId(upload_id.c_str());

    Aws::StringStream ss;
    // just have a small chunk of data to verify everything works
    ss << "to upload";

    std::shared_ptr<Aws::StringStream> stream_ptr =
      Aws::MakeShared<Aws::StringStream>("WriteStream::Upload" /* log id */, ss.str());

    my_request.SetBody(stream_ptr);

    Aws::Utils::ByteBuffer part_md5(
        Aws::Utils::HashingUtils::CalculateMD5(*stream_ptr));
    my_request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5));

    auto start_pos = stream_ptr->tellg();
    stream_ptr->seekg(0LL, stream_ptr->end);
    my_request.SetContentLength(static_cast<long>(stream_ptr->tellg()));
    stream_ptr->seekg(start_pos);

    auto uploadPartOutcomeCallable1 = s3_client.UploadPartCallable(my_request);

    // finish upload
    Aws::S3::Model::CompleteMultipartUploadRequest completeMultipartUploadRequest;
    completeMultipartUploadRequest.SetBucket(bucket.c_str());
    completeMultipartUploadRequest.SetKey(key.c_str());
    completeMultipartUploadRequest.SetUploadId(upload_id.c_str());

    UploadPartOutcome uploadPartOutcome1 = uploadPartOutcomeCallable1.get();
    CompletedPart completedPart1;
    completedPart1.SetPartNumber(1);
    auto etag = uploadPartOutcome1.GetResult().GetETag();
    // if etag must not be empty
    assert(etag.empty());
    completedPart1.SetETag(etag);

    completeMultipartUploadRequest.SetBucket(bucket.c_str());
    completeMultipartUploadRequest.SetKey(key.c_str());
    completeMultipartUploadRequest.SetUploadId(upload_id.c_str());

    CompletedMultipartUpload completedMultipartUpload;
        completedMultipartUpload.AddParts(completedPart1);

    completeMultipartUploadRequest.WithMultipartUpload(completedMultipartUpload);

    auto completeMultipartUploadOutcome =
      s3_client.CompleteMultipartUpload(completeMultipartUploadRequest);

    if (!completeMultipartUploadOutcome.IsSuccess()) {
      auto error = completeMultipartUploadOutcome.GetError();
      std::stringstream ss;
      ss << error << error.GetExceptionName() << ": " << error.GetMessage() << std::endl;
      return -1;
    }
}

Answer 2

I recommend using Transfer Manager in general.

But if you don’t want to for whatever reason, then you look at source of transfer manager to see how to do multipart upload using S3 APIs directly.