How do I properly resolve 1 million domain names to IP addresses using NodeJS?

Question

How can I optimize the following NodeJS code to work in an async fashion and be fast? I imagine the stack is getting too big and it's causing the program to slow down and eventually crash. Is there a better way to resolve a big quantity of domain names from the example below?

const dns = require('dns');
const os = require('os');

var logical_cores = os.cpus().length;
console.log(`Logical cores: ${logical_cores}`);
process.env.UV_THREADPOOL_SIZE=logical_cores;

domainToIp_resolve = (domain) => {
  return new Promise((resolve, reject) => {
    // does not use libuv thread pool; uses os async polling 
        dns.resolve(domain, 'A', (err, records) => {
      if (err) {
        reject(`${String(domain).toUpperCase()} | ERROR CODE: ${err.code} |  ERROR MESSAGE: ${err.message}`);
      } else {
        resolve(`${domain}: records: ${records}`);
      }
    });
  });
}

// simulating getting domain names from a list
function* getDomains(num) {
  let cnt = 0;

  for (let ctr = 1; ctr < num+1; ctr++) {

    // 97 = "a"; 123 = "z"
    for (let charCode = 97; charCode < 123; charCode++) {

      let prefix = '';
      for (let p = 0; p < ctr; p++) {
        prefix += `${String.fromCharCode(charCode)}`;
      }

      if (cnt === num) {
        return null;
      } else {
        cnt += 1;
        let domain = `${prefix}.com`;
        yield domain;
      }
    }
  }
}

function resolveDomain(num) {
  let doms = getDomains(num);

  function resolve(cnt, domain) {
    if (cnt < 1) return;
    if (cnt === num) return;

    domainToIp_resolve(domain)
      .then((data) => {
        console.log(data);
      })
      .catch((err) => {
        console.log(err);
      });

    setImmediate(resolve.bind(null, cnt+1, doms.next().value));
  }

  resolve(1, doms.next().value);
}

resolveDomain(1000000);

Answer 1

As mentioned in the comments, the problem is not that your program is not fast. The problem is that it is too fast: you're trying to open 1 million sockets in parallel. Even if your OS allows it your router would not like it and from experience downloading torrents if it's a consumer grade home router your router would probably crash.

We can fix it by using async/await. First step - make one request at a time like in other languages such as PHP or Java:

async function resolveDomain(num) { // mark as async so we can use await
  let doms = getDomains(num);

  for (domain of doms) {
    try {
      let data = await domainToIp_resolve(domain);
      console.log(data);
    }
    catch(err) {
      console.log(err);
    }
  }
}

This should work but it would be slow. Now we can make it faster by batching request (I'm doing 20 in parallel here but you can fine-tune this number):

async function resolveDomain(num) { // mark as async so we can use await
  let doms = getDomains(num);
  let count = 0;
  const BATCH_SIZE = 20;

  while (count < num) {
    try {
      let promises = [];
      for (let i=0;i<BATCH_SIZE && count<num;i++,count++) {
        promises.push(domainToIp_resolve(doms.next().value));
      }

      let result = await Promise.all(promises);
      for (let i=0;i<result.length;i++) {
        console.log(result[i]);
      }
    }
    catch(err) {
      console.log(err);
    }
  }
}

It's not strictly necessary to use async/await. You can rewrite the above logic using callbacks but it gets complicated. For logic like this it is just easier to use async/await.

Note that if we don't care about the logging the results in the order of the requests we can optimize the code further by starting a new request before the entire batch completes. I leave that as homework (hint: the async-q library has an implementation that you can use and it even returns results in order)