Node.js - Get google results in JSON

Question

I'm iterating over 2 arrays in order to construct google searches that return true if any search results contain content from both arrays. Using Node.js, what's the best way to get consumable json results?

module.exports = function(context, cb) {
    var google = require('googleapis');
    var request = require('request');
    var url = 'https://www.googleapis.com/customsearch/v1?key='+context.data.key+'&cx='+context.data.cx+'&q=lectures';
    console.log(url);
    request(url,function(error, response, result){
        if(!error){
            console.log(result);
            cb(null, result);
        }
    });
};

which gives the following error:

{
 "error": {
  "errors": [
   {
    "domain": "usageLimits",
    "reason": "accessNotConfigured",
    "message": "Access Not Configured. The API (CustomSearch API) is not enabled for your project. Please use the Google Developers Console to update your configuration.",
    "extendedHelp": "https://console.developers.google.com"
   }
  ],
  "code": 403,
  "message": "Access Not Configured. The API (CustomSearch API) is not enabled for your project. Please use the Google Developers Console to update your configuration."
 }

This is close but not a duplicate of Getting Error 403: Access Not Configured. Please use Google Developers Console to activate the API for your project

Answer 1

You can use a third-party service like SerpApi to scrape Google and get back structured JSON.

Example using the Node.js library:

var gsr = require('GoogleSearchResults')
let serp = new gsr.GoogleSearchResults("demo")
serp.json({
 q: "Coffee", 
 location: "Portland"
}, (result) => {
  console.log(result)
})

Answer 2

NODE.JS Convert GTFS file into json.
Add File Content in mongoDB in json format

var request = require('request');
var express = require('express');
var app = express();
var GtfsRealtimeBindings = require('gtfs-realtime-bindings');

var mysql = require('mysql');
var mongo = require('mongodb');

var jsonData

var requestSettings = {
    method: 'GET',
    url: 'http://datamine.mta.info/mta_esi.php?key=948444f7ec34a8c264a22d2b044d88d6&feed_id=1',
    encoding: null
};

var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";


app.get('/', function (req, res) {
    request(requestSettings, function (error, response, body) {
        if (!error && response.statusCode == 200) {
            jsonData = GtfsRealtimeBindings.FeedMessage.decode(body);
            // jsonData.entity.forEach(function (entity) {
            //     if (entity.trip_update) {
            res.send(jsonData);
            //     }
            // });
            MongoClient.connect(url, function (err, db) {
                if (err) throw err;
                var dbo = db.db("mydb");
                dbo.collection("customers").insertMany(jsonData.entity, function (err, res) {
                    if (err) throw err;
                    console.log("Number of documents inserted: " + res.insertedCount);
                    db.close();
                });
            });
        }
    });
});





app.listen(3010, () => console.log('Example app listening on port 3010!'))

Answer 3

I forgot to enable the api here https://console.developers.google.com/apis/api/customsearch/overview?project=ai-query

Thanks @djechlin for pointing to near-duplicate answer.