Is it theoretically possible to use `putRecord` in Kinesis via singular http POST request?

Question

I've been experiencing some issues with AWS Kinesis inasmuch as I have a stream set up and I want to use a standard http POST request to invoke a Kinesis PutRecord call on my stream. I'm doing this because bundle-size of my resultant javascript application matters and I'd rather not import the aws-sdk to accomplish something that should (on paper) be possible.

Just so you know, I've looked at this other stack overflow question about the same thing and It was... sort of informational.

Now, I already have a method to sigv4 sign a request using an access key, secret token, and session token. but when I finally get the result of signing the request and send it using the in-browser fetch api, the service tanks with (or with a json object citing the same thing, depending on my Content-Type header, I guess) as the result.

Here's the code I'm working with

// There is a global function "sign" that does sigv4 signing
// ...

var payload = {
    Data: { task: "Get something working in kinesis" },
    PartitionKey: "1",
    StreamName: "MyKinesisStream"
}

var credentials =  {
    "accessKeyId": "<access.key>",
    "secretAccessKey": "<secret.key>",
    "sessionToken": "<session.token>",
    "expiration": 1528922673000
}

function signer({ url, method, data }) {
    // Wrapping with URL for piecemeal picking of parsed pieces
    const parsed = new URL(url);

    const [ service, region ] = parsed.host.split(".");

    const signed = sign({
        method,
        service,
        region,
        url,

        // Hardcoded
        headers : {
            Host           : parsed.host,
            "Content-Type" : "application/json; charset=UTF-8",

            "X-Amz-Target" : "Kinesis_20131202.PutRecord"
        },

        body : JSON.stringify(data),
    }, credentials);

    return signed;
}

// Specify method, url, data body
var signed = signer({
    method: "POST",
    url: "https://kinesis.us-west-2.amazonaws.com",
    data : JSON.stringify(payload)
});

var request = fetch(signed.url, signed);

When I look at the result of request, I get this:

{
    Output: { 
       __type: "com.amazon.coral.service#InternalFailure"},
       Version: "1.0" 
}

Now I'm unsure as to whether Kinesis is actually failing here, or if my input is malformed?

here's what the signed request looks like

{
    "method": "POST",
    "service": "kinesis",
    "region": "us-west-2",
    "url": "https://kinesis.us-west-2.amazonaws.com",
    "headers": {
        "Host": "kinesis.us-west-2.amazonaws.com",
        "Content-Type": "application/json; charset=UTF-8",
        "X-Amz-Target": "Kinesis_20131202.PutRecord",
        "X-Amz-Date": "20180613T203123Z",
        "X-Amz-Security-Token": "<session.token>",
        "Authorization": "AWS4-HMAC-SHA256 Credential=<access.key>/20180613/us-west-2/kinesis/aws4_request, SignedHeaders=content-type;host;x-amz-target, Signature=ba20abb21763e5c8e913527c95a0c7efba590cf5ff1df3b770d4d9b945a10481"
    },
    "body": "\"{\\\"Data\\\":{\\\"task\\\":\\\"Get something working in kinesis\\\"},\\\"PartitionKey\\\":\\\"1\\\",\\\"StreamName\\\":\\\"MyKinesisStream\\\"}\"",
    "test": {
        "canonical": "POST\n/\n\ncontent-type:application/json; charset=UTF-8\nhost:kinesis.us-west-2.amazonaws.com\nx-amz-target:Kinesis_20131202.PutRecord\n\ncontent-type;host;x-amz-target\n508d2454044bffc25250f554c7b4c8f2e0c87c2d194676c8787867662633652a",
        "sts": "AWS4-HMAC-SHA256\n20180613T203123Z\n20180613/us-west-2/kinesis/aws4_request\n46a252f4eef52991c4a0903ab63bca86ec1aba09d4275dd8f5eb6fcc8d761211",
        "auth": "AWS4-HMAC-SHA256 Credential=<access.key>/20180613/us-west-2/kinesis/aws4_request, SignedHeaders=content-type;host;x-amz-target, Signature=ba20abb21763e5c8e913527c95a0c7efba590cf5ff1df3b770d4d9b945a10481"
    }

(the test key is used by the library that generates the signature, so ignore that) (Also there are probably extra slashes in the body because I pretty printed the response object using JSON.stringify).

My question: Is there something I'm missing? Does Kinesis require headers a, b, and c and I'm only generating two of them? Or is this internal error an actual failure. I'm lost because the response suggests nothing I can do on my end.

I appreciate any help!

Edit: As a secondary question, am I using the X-Amz-Target header correctly? This is how you reference calling a service function so long as you're hitting that service endpoint, no?

Update: Followinh Michael's comments, I've gotten somewhere, but I still haven't solved the problem. Here's what I did:

I made sure that in my payload I'm only running JSON.stringify on the Data property.

I also modified the Content-Type header to be "Content-Type" : "application/x-amz-json-1.1" and as such, I'm getting slightly more useful error messages back.

Now, my payload is still mostly the same:

var payload = {
    Data: JSON.stringify({ task: "Get something working in kinesis" }),
    PartitionKey: "1",
    StreamName: "MyKinesisStream"
}

and my signer function body looks like this:

 function signer({ url, method, data }) {
    // Wrapping with URL for piecemeal picking of parsed pieces
    const parsed = new URL(url);

    const [ service, region ] = parsed.host.split(".");

    const signed = sign({
        method,
        service,
        region,
        url,

        // Hardcoded
        headers : {
            Host           : parsed.host,
            "Content-Type" : "application/json; charset=UTF-8",

            "X-Amz-Target" : "Kinesis_20131202.PutRecord"
        },

        body : data,
    }, credentials);

    return signed;
}

So I'm passing in an object that is partially serialized (at least Data is) and when I send this to the service, I get a response of:

{"__type":"SerializationException"}

which is at least marginally helpful because it tells me that my input is technically incorrect. However, I've done a few things in an attempt to correct this:

• I've run JSON.stringify on the entire payload
• I've changed my Data key to just be a string value to see if it would go through
• I've tried running JSON.stringify on Data and then running btoa because I read on another post that that worked for someone.

But I'm still getting the same error. I feel like I'm so close. Can you spot anything I might be missing or something I haven't tried? I've gotten sporadic unknownoperationexceptions but I think right now this Serialization has me stumped.

Edit 2:

As it turns out, Kinesis will only accept a base64 encoded string. This is probably a nicety that the aws-sdk provides, but essentially all it took was Data: btoa(JSON.stringify({ task: "data"})) in the payload to get it working

Answer 1

While I'm not certain this is the only issue, it seems like you are sending a request body that contains an incorrectly serialized (double-encoded) payload.

var obj = { foo: 'bar'};

JSON.stringify(obj) returns a string...

'{"foo": "bar"}' // the ' are not part of the string, I'm using them to illustrate that this is a thing of type string.

...and when parsed with a JSON parser, this returns an object.

{ foo: 'bar' }

However, JSON.stringify(JSON.stringify(obj)) returns a different string...

'"{\"foo\": \"bar\"}"'

...but when parsed, this returns a string.

 '{"foo": "bar"}'

The service endpoint expects to parse the body and get an object, not a string... so, parsing the request body (from the service's perspective) doesn't return the correct type. The error seems to be a failure of the service to parse your request at a very low level.

In your code, body: JSON.stringify(data) should just be body: data because earlier, you already created a JSON object with data: JSON.stringify(payload).

As written, you are effectively setting body to JSON.stringify(JSON.stringify(payload)).

Answer 2

Not sure if you ever figured this out, but this question pops up on Google when searching for how to do this. The one piece I think you are missing is that the Record Data field must be base64 encoded. Here's a chunk of NodeJS code that will do this (using PutRecords).

And for anyone asking, why not just use the SDK? I currently must stream data from a cluster that cannot be updated to a NodeJS version that the SDK requires due to other dependencies. Yay.

const https = require('https')
const aws4  = require('aws4')
const request = function(o) { https.request(o, function(res) { res.pipe(process.stdout) }).end(o.body || '') }

const _publish_kinesis = function(logs) {
    const kin_logs = logs.map(function (l) {
        let blob = JSON.stringify(l) + '\n'
        let buff = Buffer.from(blob, 'binary');
        let base64data = buff.toString('base64');

        return {
            Data: base64data,
            PartitionKey: '0000'
        }
    })

    while(kin_logs.length > 0) {
        let data = JSON.stringify({
            Records: kin_logs.splice(0,250),
            StreamName: 'your-streamname'
        })

        let _request = aws4.sign({
            hostname: 'kinesis.us-west-2.amazonaws.com',
            method: 'POST',
            body: data,
            path: '/?Action=PutRecords',
            headers: {
                'Content-Type': 'application/x-amz-json-1.1',
                'X-Amz-Target': 'Kinesis_20131202.PutRecords'
            },
         }, {
            secretAccessKey: "****",
            accessKeyId: "****"
           // sessionToken: "<your-session-token>"
         })

        request(_request)
    }
}

var logs = [{
  'timeStamp': new Date().toISOString(),
  'value': 'test02',
},{
  'timeStamp': new Date().toISOString(),
  'value': 'test01',
}]
_publish_kinesis(logs)