I am creating an API with Node.js and Express. When a user uploads a file, the server alters it with the sharp image processing library and then uploads it to a Google Cloud Storage bucket. The RAM usage of the node process increases by the size of the uploaded file and never decreases, even when running manual garbage collection. When there is no RAM left to allocate, the server crashes.
Here is a reproduction of the problem:
index.js
const express = require('express')
const GCS = require('@google-cloud/storage')
const Multer = require('multer')
const sharp = require('sharp')
const upload = Multer({ storage: Multer.MemoryStorage, })
const storage = new GCS({
projectId: 'myProject',
keyFilename: './key.json'
}).bucket('myBucket')
const app = express();
app.get("/", (req, res) => {
res.sendFile(__dirname + '/index.html')
})
app.post("/upload", upload.single('media'), (req, res) => {
console.log(req.file)
sharp(req.file.buffer).rotate().resize(1920, 1920).toBuffer().then(b => {
const fileUpload = storage.file('test/' + Date.now())
const fileStream = fileUpload.createWriteStream({
metadata: {
contentType: req.file.mimetype
},
resumable: false
});
fileStream.on('error', (err) => {
console.log(err)
res.send("Server Error")
});
fileStream.on('finish', () => {
fileUpload.makePublic().then(() => {
res.send("DONE")
})
})
fileStream.end(b)
}).catch(e => {
console.log(e)
res.send("Server Error")
})
})
app.listen(3000, () => {
console.log("Server is running...")
})
index.html
<form action="/upload" enctype="multipart/form-data" method="POST">
<input type="file" name="media">
<input type="submit">
</form>
I also performed a test using Multer.DiskStorage, and the RAM usage stayed consistent and did not increase nearly as much. This method, however, is not preferable because of disk storage, directory permissions, etc.
I appreciate any help you may have to offer!
