I have a node.js script that copies data from a folder into different shared drives. The source folder is huge, so the script runs for hours. The logic is list folders, check that they correspond to certain criteria (created date is between 2018 and 2021) and then move its contents to a different drive accordingly.
The however, sometimes I get the error mentioned in the subject line.
This page discusses the issue, and I did set process.env.UV_THREADPOOL_SIZE = 128, but this does not seem to have fixed the issue.
I could not find any other discussions specific to the Google Drive API. Other such errors on different APIs seem irrelevant.
The relevant source code is as follows:
async function listFiles() {
let files = [];
const drive = _getDrive();
const doListFiles = async(options, pageToken) => {
return await new Promise(async (resolve, reject) => {
// ...
const params = {
pageSize: 1000,
fields: `nextPageToken, files(${listFileData?.join(',')})`,
supportsAllDrives: true,
q,
pageToken,
trashed: false,
};
drive.files.list(params, async (err, res) => {
if (err) {
reject(err);
return console.error('Could not list files:', err);
}
const _files = res.data.files;
if (!_files.length) {
resolve(_files);
}
files = [...files, ..._files];
if (res.data.nextPageToken) {
return resolve(await retry(doListFiles, options, res.data.nextPageToken));
}
return resolve(files);
});
});
};
try {
return await retry(doListFiles, options);
} catch (err) {
console.error(err);
console.log({ isError: true, errMessage: err, ...file });
return [];
}
}
Detailed error message:
Could not list files: FetchError: request to https://www.googleapis.com/drive/v3/files?pageSize=1000&fields=nextPageToken%2C%20files%28id%2Cname%2CmimeType%2CcreatedTime%2Cparents%29&supportsAllDrives=true&q=%271EnHp6QRQK9A7XSzvDdU2H4GOpp1qgF8G%27%20in%20parents%20and%20mimeType%20%3D%20%27application%2Fvnd.google-apps.folder%27&trashed=false failed, reason: getaddrinfo EAI_AGAIN www.googleapis.com
at ClientRequest.<anonymous> (/home/wildhog/Documents/clients/xxxxxxxxx/node_modules/node-fetch/lib/index.js:1491:11)
at ClientRequest.emit (node:events:537:28)
at TLSSocket.socketErrorListener (node:_http_client:465:9)
at TLSSocket.emit (node:events:537:28)
at emitErrorNT (node:internal/streams/destroy:151:8)
at emitErrorCloseNT (node:internal/streams/destroy:116:3)
at process.processTicksAndRejections (node:internal/process/task_queues:82:21) {
type: 'system',
errno: 'EAI_AGAIN',
code: 'EAI_AGAIN',
config: {
url: 'https://www.googleapis.com/drive/v3/files?pageSize=1000&fields=nextPageToken%2C%20files%28id%2Cname%2CmimeType%2CcreatedTime%2Cparents%29&supportsAllDrives=true&q=%271EnHp6QRQK9A7XSzvDdU2H4GOpp1qgF8G%27%20in%20parents%20and%20mimeType%20%3D%20%27application%2Fvnd.google-apps.folder%27&trashed=false',
method: 'GET',
userAgentDirectives: [ [Object] ],
paramsSerializer: [Function (anonymous)],
headers: {
'x-goog-api-client': 'gdcl/6.0.0 gl-node/18.4.0 auth/8.1.0',
'Accept-Encoding': 'gzip',
'User-Agent': 'google-api-nodejs-client/6.0.0 (gzip)',
Authorization: 'Bearer ya29.c.b0AXv0zTPyF3jQwz-LOYiyLGoeTSWDazYmqOO0OKq4IR9PZ7Yu89qddhwWZ5nXndyTrjoKFNkuBZHYa9f_txe-ZHByNq39wm60s2AgwwxMTH-fsfXlIXnIR4F0DYPSff8PEs7mcquyedybSi1EVnhTYATFfa_ZrXv-rkTQ-j9ayHnqCJCO46AhvFDO20i0zIXP5pqxSYKOU-Vovl-mQG0nhrnGGhzLc1Ca.......................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................',
Accept: 'application/json'
},
params: {
pageSize: 1000,
fields: 'nextPageToken, files(id,name,mimeType,createdTime,parents)',
supportsAllDrives: true,
q: "'1EnHp6QRQK9A7XSzvDxxxxxxxxxxxx' in parents and mimeType = 'application/vnd.google-apps.folder'",
trashed: false
},
validateStatus: [Function (anonymous)],
retry: true,
responseType: 'json',
retryConfig: {
currentRetryAttempt: 2,
retry: 3,
httpMethodsToRetry: [Array],
noResponseRetries: 2,
statusCodesToRetry: [Array]
}
}
}
All help is greatly appreciated.
Update
@Tanaike asked for the source code of the retry() function. It implements exponential back off, works fine and is not at the root of the problem, but here goes the code if it makes things simpler.
import logger from "./logger.js";
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
const retry = async (fun, ...args) => {
const maxTries = 5;
let numTries = 0;
const maxDuration = 64000;
while (numTries < maxTries) {
try {
logger.info('Trying', numTries + 1, Date.now(), fun);
const funResult = await fun(...args);
logger.info(
' ~ file: retry.js ~ line 14 ~ retry ~ fun, funResult',
fun,
funResult
);
return funResult;
} catch (err) {
if (!/rate limit exceeded/i.test(err.toString()))
return { isError: true, errMessage: err, ...args[0] };
let sleepDuration = 2 ** numTries * 1000 + Math.random() * 1000;
if (sleepDuration > maxDuration) sleepDuration = maxDuration;
// let sleepDuration = 0;
logger.info(
`Error: ${err}`,
'Will retry after',
sleepDuration,
...args
);
await sleep(sleepDuration);
numTries++;
}
}
return { isError: true, errMessage: 'Too many tries', ...args[0] };
};
