Context: I'm working on code which uses a read stream to download a file from an SFTP server and upload it to GCS via a writeStream, using Nodejs v10.15.3.
Due to an error in the SFTP library I'm working, stream.pipe (that is, piping from the read stream the library produces) is actually broken in Node 10, Because of this, I am attempting to instead upload this file via the following code (where stream is the read stream and unnecessary information has been left out):
let acl = fileMode;
if (fileMode === 'public') {
// options for acl are publicRead and private
// need to add the Read if public
acl += 'Read';
}
var options = {
predefinedAcl: acl,
destination: destPath,
metadata: {
contentType: contentType,
cacheControl: 'no-cache'
}
};
// Add in a check here for if the bucket exists
let file = new File(bucket, destPath);
let writeStream = file.createWriteStream(options);
writeStream.on('finish', () => {
file.getMetadata()
.then((metadata) => {
console.log('metadata', metadata);
return resolve(metadata);
})
.catch(error => {
console.error('Error getting file metadata', error);
return reject(error);
});
});
stream.on('end', () => {
try {
writeStream.end();
} catch (err) {
console.error('Error closing writeStream', err);
return reject(err);
}
});
writeStream.on('error', error => {
console.error('Error in writeStream', error);
return reject(error);
});
stream.on('error', error => {
console.error('Error in stream', error);
return reject(error);
});
let data = stream.read();
while (data) {
writeStream.write(data);
data = stream.read();
}
When I use the while (data) method to stream from our SFTP server to a local file on the filesystem, this works without incident. However, when I try to run this code to upload to our GCS file, I get the following error:
MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 close listeners added. Use emitter.setMaxListeners() to increase limit
Error in writeStream Error: Retry limit exceeded
// stacktrace omitted
Error Uploading to GCS from a stream: Retry limit exceeded
Error: Retry limit exceeded
It seems like I must be doing something wrong here, but I have no idea why this isn't a valid method, nor am I sure if I'm missing some subtlety of streams (which I freely confess are pretty much a black box to me) or an issue with GCS.
EDIT: Okay, this actually appears to be completely unrelated to the SFTP issue. I've tried just uploading a file from the local fs using the recommended method, and am seeing the same error. The more 'streamlined' code I'm trying is:
// Add in a check here for if the bucket exists
let file = new File(bucket, destPath);
fs.createReadStream('sample_file.csv')
.pipe(file.createWriteStream(options))
.on('error', function(err) {
console.error('err', err);
return reject(err);
})
.on('finish', function() {
console.log('resolving');
return resolve({gcsUrl: url});
});
