How can I access the response headers of a request that is piped to a feedparser

Question

I am trying to parse an RSS feed using request js and feedparser-promised libraries. I am able to parse the feed using the below code.

import Bottleneck from 'bottleneck';

const feedparser = require('feedparser-promised');
const limiter = new Bottleneck({
  maxConcurrent: 1,
  minTime: 333,
});

const httpOptions = {
      uri: val.sourcefeedurl,
      resolveWithFullResponse: true,
      method: 'GET',
      pool: false,
      headers: {
        'If-None-Match': val.etag,
        'If-Modified-Since': val.LastModified,
        Connection: 'keep-alive',
        ciphers: 'DES-CBC3-SHA',
      },
    };
const response = await limiter.schedule(() => feedparser.parse(httpOptions));

But since I use the feedparser-promised library I am not able to cache the etag and Last Modified from the response headers.

I tried modifying feedparser-promised like this

'use strict';

const request = require('request');

const feedParser = require('./feedParser');

const parse = (requestOptions, feedparserOptions) => {
  const metaData = {};
  return new Promise((resolve, reject) => {
    request.get(requestOptions).on('error', reject).on('response', async resp => {
      if (resp.statusCode === 304) {
        reject('Source not modified');
      } else if (resp.statusCode === 200) {
        metaData.etagin = await resp.headers.etag;
        metaData.LastModifiedin = await resp.headers['last-modified'];
        metaData.LastModifiedLocal = await resp.headers['last-modified'];
        // console.log(metaData);
      }
    }).pipe(feedParser(feedparserOptions).on('error', reject).on('response', resolve));
  });
};

module.exports = {
  parse
};

Below is the feedParser file

'use strict';

const FeedParserStream = require('feedparser');

module.exports = (feedparserOptions, metaData) => {
  // console.log(metaData, 'herre');
  const parsedItems = [];
  const feedparser = new FeedParserStream(feedparserOptions);
  // console.log(feedparser);
  feedparser.on('readable', () => {
    // console.log(resp);
    let item;
    while (item = feedparser.read()) {
      parsedItems.push(item);
    }

    return parsedItems;
  }).on('end', function next() {
    this.emit('response', parsedItems);
  });
  return feedparser;
};

So my question is how do I return the response headers along with the parsedItems (as in the code) while resolving the promise.

Help is very much appreciated.

Have you tried to parse the etag/last modified with another parser? — sundar, Aug 12 '19 at 16:11

score 1 · Accepted Answer · answered Aug 14 '19 at 10:48

Pass the metaData on end like

'use strict';

const FeedParserStream = require('feedparser');

module.exports = (feedparserOptions, metaData) => {
  // console.log(metaData, 'herre');
  const parsedItems = [];
  const feedparser = new FeedParserStream(feedparserOptions);
  // console.log(feedparser);
  feedparser.on('readable', () => {
    // console.log(resp);
    let item;
    while (item = feedparser.read()) {
      parsedItems.push(item);
    }

    return parsedItems;
  }).on('end', function next() {
    this.emit('response', { parsedItems, metaData });
  });
  return feedparser;
};

and your feed-parser promised as

'use strict';

const request = require('request');

const feedParser = require('./feedParser');

const parse = (requestOptions, feedparserOptions) => {
  const metaData = {};
  return new Promise((resolve, reject) => {
    request.get(requestOptions).on('error', reject).on('response', async resp => {
      if (resp.statusCode === 304) {
        reject('Source not modified');
      } else if (resp.statusCode === 200) {
        metaData.etagin = await resp.headers.etag;
        metaData.LastModifiedin = await resp.headers['last-modified'];
        metaData.LastModifiedLocal = await resp.headers['last-modified'];
        // console.log(metaData);
      }
    }).pipe(feedParser(feedparserOptions, metaData).on('error', reject).on('response', resolve));
  });
};

module.exports = {
  parse
};

How can I access the response headers of a request that is piped to a feedparser

1 Answers1