1

I'm trying to write a markdown pre-processor that can include other files in the output.

Unfortunately, I have discovered that "the pause method does not stop queued up line events".

The solution that I had written would finish reading and writing the original file and then tack the other files on at the end:

import { createInterface } from "readline";
import * as fs from "fs";
import * as path from "path";
import minimist from "minimist";

const argv = minimist(process.argv.slice(2));

const inputFile = path.resolve(argv._[0]);
const outputFile = path.resolve(argv._[1]);

const inputFileReadStream = createInterface({
    "input": fs.createReadStream(inputFile)
});

const writeStream = fs.createWriteStream(outputFile);

readStream.on("line", function readLine(line) {
    if (line.startsWith("@import") === true) {
        inputFileReadStream.pause();

        console.log(line.substring(line.indexOf("\"") + 1, line.lastIndexOf("\"")));

        const readStream = createInterface({
            "input": fs.createReadStream(path.resolve(path.dirname(inputFile), line.substring(line.indexOf("\"") + 1, line.lastIndexOf("\""))))
        });

        readStream.on("line", function(line) {
            readLine(line);
        });

        readStream.on("end", function() {
            inputFileReadStream.resume();
        });
    } else {
        writeStream.write(line + "\n");
    }
});

So I tried modifying it to use the line-by-line library mentioned in the post that I linked to see if that would work:

let readStream = new (require("line-by-line"))(fs.createReadStream(inputFile));

const writeStream = fs.createWriteStream(outputFile);

readStream.on("line", function readLine(line) {
    if (line.startsWith("@import") === true) {
        const tempStream = readStream;

        tempStream.pause();

        console.log(line.substring(line.indexOf("\"") + 1, line.lastIndexOf("\"")));

        readStream = new (require("line-by-line"))(fs.createReadStream(path.resolve(path.dirname(inputFile), line.substring(line.indexOf("\"") + 1, line.lastIndexOf("\"")))));

        readStream.on("line", function(line) {
            readLine(line);
        });

        readStream.on("end", function() {
            tempStream.resume();
        });
    } else {
        writeStream.write(line + "\n");
    }
});

But this produced a file that progressively interspersed the lines from the imported files into the main file.

My question is: Is there a way to have readline respond to pause events properly?

Could this be fixable with a Transform?

Here's a link to the repository if it helps: https://github.com/brianjenkins94/Rosettas-Node

Brian Jenkins
  • 358
  • 2
  • 20

2 Answers2

0

There are probably negative performance implications to this, but I got what I wanted by setting the highWaterMark value to 1.

import { createInterface } from "readline";
import * as fs from "fs";
import * as path from "path";
import minimist from "minimist";

const argv = minimist(process.argv.slice(2));

const inputFile = path.resolve(argv._[0]);
const outputFile = path.resolve(argv._[1]);

const inputFileReadStream = createInterface({
    "input": fs.createReadStream(inputFile, { "highWaterMark": 1 })
});

const writeStream = fs.createWriteStream(outputFile);

inputFileReadStream.on("line", function readLine(line, readStream = inputFileReadStream) {
    if (line.startsWith("@import") === true) {
        readStream.pause();

        const childReadStream = createInterface({
            "input": fs.createReadStream(path.resolve(path.dirname(inputFile), line.substring(line.indexOf("\"") + 1, line.lastIndexOf("\""))), { "highWaterMark": 1 })
        });

        childReadStream.on("line", function(line) {
            readLine(line, childReadStream);
        });

        childReadStream.on("close", function() {
            readStream.resume();
        });
    } else {
        writeStream.write(line + "\n");
    }
});
Brian Jenkins
  • 358
  • 2
  • 20
0

You can adjust the amount of internal buffering readline performs by through highwaterMark. See https://nodejs.org/api/stream.html#buffering

This is the explanation for Brian's reply.

Ken Lin
  • 1,819
  • 21
  • 21