0

I have a function which:

  1. request data from a mongodb
  2. based on the results makes API calls to Ebay
  3. inserts the Ebay API response into another mongodb

I had a version (even more hacky) working (see gist) but not all of the calls were getting executed.

I've tried converting the function to async / await but I'm getting Error: socket hung up errors. In my experience calling the Ebay API this is not due to the request I'm making to API endpoint (i.e. the URL that I am GETing) but some error in the rest of the function.

I'm wondering if it's to do with the await I have in the for / of loop.

Can anyone advise on the correct way to structure this function with async / await?

exports = async function(payload) {
const axios = require("axios");
const throttledQueue = require('throttled-queue'); // not used in this example, but I need to throttle the requests once I figure out the rest of the function
let throttle = throttledQueue(5, 1000, true);

let ebayCollection = context.services.get("mongodb-atlas").db("cards").collection("ebay-prices");
let cardCollection = context.services.get("mongodb-atlas").db("cards").collection("stuart collection");


//Get wanted cards from card collection
let ownedCards = await cardCollection.find({
    forTrade: "Yes"
}).toArray(); //change back to status:Want after testing.


async function insertCards(ownedCards) {
    try {
        for (let card of ownedCards) {
            let url = "http://svcs.ebay.com/services/search/FindingService/v1?GLOBAL-ID=EBAY-US&REST-PAYLOAD&keywords=" + card.brand + " " + card.series + " " + card.player +
                "&itemFilter(0).name=SoldItemsOnly&itemFilter(0).value=true&OPERATION-NAME=findCompletedItems&paginationInput.entriesPerPage=6&paginationInput.pageNumber=1&RESPONSE-DATA-FORMAT=json&SECURITY-APPNAME=<myapp>&SERVICE-NAME=FindingService&SERVICE-VERSION=1.12.0";
            let cardId = card.card_id;
            console.log(url);
            const resp = await axios.get(url);
            console.log(resp);
            for (const item in resp.data.findCompletedItemsResponse[0].searchResult[0].item) {
                const itemId = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].itemId[0];
                const title = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].title[0];
                const galleryURL = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].galleryURL[0];
                const endDate = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].listingInfo[0].endTime[0];
                const price = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].sellingStatus[0].currentPrice[0].__value__;
                const currencyId = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].sellingStatus[0].currentPrice[0]['@currencyId'];

                const result = ebayCollection.updateOne({
                    itemId: itemId
                }, {
                    $addToSet: {
                        cardId: cardId
                    },
                    $set: {
                        itemId: itemId,
                        title: title,
                        endDate: endDate,
                        price: price,
                        currencyId: currencyId,
                        galleryURL: galleryURL
                    }
                }, {
                    upsert: true
                })

            }
        }

    } catch (error) {
        console.log("THIS IS THE ERROR" + error); // catches both errors
    }
}

await insertCards(ownedCards);
};
Stuart Brown
  • 977
  • 2
  • 22
  • 47
  • Please post full of error (which line) and the call-stack (if posible) for easier to debug your code. – namgold Sep 30 '20 at 09:25
  • You did not encode your query string. Try encoding it using `querystring` module – Salil Sep 30 '20 at 10:54
  • @namgold the error just gets output in my catch block. The console shows THIS IS THE ERRORError: socket hang up > result: { "$undefined": true } > result (JavaScript): EJSON.parse('{"$undefined":true}') – Stuart Brown Sep 30 '20 at 11:20
  • @Salil Thanks for the comment. I don't think that's the issue - the gist version that I link to in the question uses the query string the same way and the request works – Stuart Brown Sep 30 '20 at 11:22

1 Answers1

1

OK, after going through all the comments on this question NodeJS - What does "socket hang up" actually mean? I changed my request to https and the problems have stopped.

I've tweaked my code a little and incorporated throttled-queue to enable rate limiting - it's below in case it is of use to anyone.

exports = async function(payload) {
const axios = require("axios");
const throttledQueue = require('throttled-queue');
let throttle = throttledQueue(15, 10000, true);

let ebayCollection = context.services.get("mongodb-atlas").db("cards").collection("ebay-prices");
let cardCollection = context.services.get("mongodb-atlas").db("cards").collection("stuart collection");


//Get wanted cards from card collection
let ownedCards = await cardCollection.find({status: "Want"}).toArray(); //change back to status:Want after testing.


async function insertCards(ownedCards) {
    try {
      for (let card of ownedCards) {
        let url = "https://svcs.ebay.com/services/search/FindingService/v1?GLOBAL-ID=EBAY-US&REST-PAYLOAD&keywords=" + card.brand + " " + card.series + " " + card.player +
            "&itemFilter(0).name=SoldItemsOnly&itemFilter(0).value=true&OPERATION-NAME=findCompletedItems&paginationInput.entriesPerPage=6&paginationInput.pageNumber=1&RESPONSE-DATA-FORMAT=json&SECURITY-APPNAME=<myapp>&SERVICE-NAME=FindingService&SERVICE-VERSION=1.12.0";
            let cardId = card.card_id;
            console.log(url);
           const resp = await axios.get(url);
           console.log(resp);
           for (const item in resp.data.findCompletedItemsResponse[0].searchResult[0].item) {
            const itemId = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].itemId[0];
            const title = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].title[0];
            const galleryURL = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].galleryURL[0];
            const endDate = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].listingInfo[0].endTime[0];
            const price = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].sellingStatus[0].currentPrice[0].__value__;
            const currencyId = resp.data.findCompletedItemsResponse[0].searchResult[0].item[item].sellingStatus[0].currentPrice[0]['@currencyId'];

            const result = ebayCollection.updateOne({
                itemId: itemId
            }, {
                $addToSet: {
                    cardId: cardId
                },
                $set: {
                    itemId: itemId,
                    title: title,
                    endDate: endDate,
                    price: price,
                    currencyId: currencyId,
                    galleryURL: galleryURL
                }
            }, {
                upsert: true
            })

           }
      }

    } catch (error) {
      console.log("THIS IS THE ERROR"+ error); // catches both errors
    }
  }

  throttle(
    await function() {
        // make the request.
        insertCards(ownedCards);
    });
};
Stuart Brown
  • 977
  • 2
  • 22
  • 47