Split JSON API's object into chunks in express app

Question

I want to separate my API's output into different pages. I want to call them like this: http://127.0.0.1:3000/api/articles/0/<API-TOKEN>

Which is returning the first page with 2-3, etc. articles.

Full code can be found here: https://github.com/DJviolin/horgalleryNode/blob/master/routes/api.js

I have dummy data JSON file:

[
  {
    "articles": [
      {
        "id": "0",
        "url": "audrey-hepburn",
        "title": "Audrey Hepburn",
        "body": "Nothing is impossible, the word itself says 'I'm possible'!",
        "category": "foo",
        "tags": [ "foo" ]
      },
      {
        "id": "1",
        "url": "walt-disney",
        "title": "Walt Disney",
        "body": "You may not realize it when it happens, but a kick in the teeth may be the best thing in the world for you.",
        "category": "foo",
        "tags": [ "foo", "bar" ]
      },
      {
        "id": "2",
        "url": "unknown",
        "title": "Unknown",
        "body": "Even the greatest was once a beginner. Don't be afraid to take that first step.",
        "category": "bar",
        "tags": [ "foo", "bar", "baz" ]
      },
      {
        "id": "3",
        "url": "neale-donald-walsch",
        "title": "Neale Donald Walsch",
        "body": "You are afraid to die, and you're afraid to live. What a way to exist.",
        "category": "bar",
        "tags": [ "foo", "bar", "baz" ]
      }
    ]
  },
  {
    "users": [
      { "name": "Admin" },
      { "name": "User" }
    ]
  }
]

Which is called into my API router this way:

function fsAsync(callback) {
  fs.readFile(__dirname + '/../public/articles/data.json', 'utf8', function(err, data) {
    if (err) {
      return callback(err);
    }
    callback(null, JSON.parse(data));
  });
};

I calling every articles like this at this route: http://127.0.0.1:3000/api/articles/<API-TOKEN>

router.get('/articles/:token', function(req, res) {
  fsAsync(function(err, data) {
    if (err) {
      return res.send(err);
    }
    var articles = data[0].articles;

    var q = articles.filter(function (article) {
      // return article.id === req.params.id;
      return article && apiToken === req.params.token;
    });
    res.json(q);
  });
});

However I want to separate this API's output into different pages when I rendering this API route: http://127.0.0.1:3000/api/articles/0/<API-TOKEN>

I tried to implement the array.slice method described here: https://stackoverflow.com/a/8495740/1442219

How can it be achieved?

Thank You!

Update:

One thing could cause problem if I spliting a JSON object into chunks, than it will need to parse everything from that object first, than decide where to split? What if user visiting the last page at 567? This means the code first have to query through millions of millions of line to return the wanted page? What if it's a database, not a JSON with dummy data? In sort, what are the best practices to return a specific page for a blog from JSON / Mongodb / etc. source?

Update 2:

This returns the first two article from the object:

// http://127.0.0.1:3000/api/articles/c/<API-TOKEN>
router.get('/articles/c/:token', function(req, res) {
  fsAsync(function(err, data) {
    if (err) {
      return res.send(err);
    }
    var articles = data[0].articles;

    var count = 0;
    var countMultiply = count * 2;
    var a = countMultiply + 0;
    var b = countMultiply + 2;
    var c = articles.slice(a, b);
    console.log(c);

    var q = c.filter(function (article) {
      // return article.id === req.params.id;
      return article && apiToken === req.params.token;
    });
    res.json(q); // (0*2+0=0, 0*2+2=2), (1*2+0=2, 1*2+2=4), (2*2+0=4, 2*2+2=6), (3*2+0=6, 3*2+2=8)
  });
});

How can I automate this able to use 0, 1, 2, 3 for page separators? So the first page at http://127.0.0.1:3000/api/articles/0/<API-TOKEN> URL to return articles (0,2) and the second page at http://127.0.0.1:3000/api/articles/1/<API-TOKEN> URL to return articles (2,4)?

You can not do it without traversing all articles. To find specific article you have to traverse all data. Once you collect all data than you can decide what to send, coz _**limit**_ and _**skip**_ will work on collected data. — xdeepakv, Feb 14 '16 at 15:13
I see. This is the way only with JSON data or databases also? So at every request the server have to run through thousands of line to return the specific 5 or 10 that the visitor wants to see? — Lanti, Feb 14 '16 at 15:23
You can optimize it like, count the document while traversing and matching. Once the document size reach to _**skip+limit**_ stop. This is what you can do. — xdeepakv, Feb 15 '16 at 18:29

Lanti · Accepted Answer · 2016-02-14T17:05:49.240

Update 3:

Looks like it's working:

// http://127.0.0.1:3000/api/articles/page/0/<API-TOKEN>
router.get('/articles/page/:id/:token', function(req, res) {
  fsAsync(function(err, data) {
    if (err) {
      return res.send(err);
    }
    var articles = data[0].articles.reverse();

    var count = req.params.id; // Page number
    var multiplier = 2; // Posts per page

    var countResult = count * multiplier;
    var a = countResult + 0;
    var b = countResult + multiplier;
    var c = articles.slice(a, b);

    var pagesLength = articles.length / multiplier;
    var pagesLengthCeil = Math.ceil(pagesLength); // Sum of all pages

    console.log('pagesLengthCeil: ' + pagesLengthCeil);
    console.log(c);

    var q = c.filter(function (article) {
      // return article.id === req.params.id;
      return article && apiToken === req.params.token;
    });
    res.json(q); // (0*2+0=0, 0*2+2=2), (1*2+0=2, 1*2+2=4), (2*2+0=4, 2*2+2=6), (3*2+0=6, 3*2+2=8)
  });
});

But I still don't know is it an efficient way to do this with huge JSON files or a database?

Split JSON API's object into chunks in express app

1 Answers1