My client side Tensorflow JS request to save to cloud storage via HTTP request. There is probably a better way to do this but this at least has worked for me.
var api = 'https://us-central1-your-uniquelocation.cloudfunctions.net/uploadModel';
var res = await model.save(tf.io.browserHTTPRequest(api,
{method: 'POST', headers:{'Authorization':'test','Content-Type': 'multipart/form-data; boundary=----WebKitFormBoundaryVoB3gVGBQHG0btQR'}}));
console.log(res);
Then for the index.js for my cloud functions
const functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require("./service-account-credentials.json");
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
//databaseURL: "https://yoururl.firebaseio.com",
storageBucket: "yoururl.appspot.com"
});
var bucket = admin.storage().bucket("yoururl.appspot.com");
const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');
exports.uploadModel = functions.https.onRequest((req, res) => {
res.set('Access-Control-Allow-Origin', '*');
res.set('Access-Control-Allow-Credentials', 'true');
if (req.method === 'OPTIONS') {
// Send response to OPTIONS requests
console.log("send response 204");
res.set('Access-Control-Allow-Methods', 'GET');
res.set('Access-Control-Allow-Headers', 'Authorization');
res.set('Access-Control-Max-Age', '3600');
res.status(204).send();
}
console.log("metho--->"+req.method);
console.log(req.headers);
const busboy = new Busboy({headers: req.headers});
const tmpdir = os.tmpdir();
// This object will accumulate all the fields, keyed by their name
const fields = {};
// This object will accumulate all the uploaded files, keyed by their name.
const uploads = {};
// This code will process each non-file field in the form.
busboy.on('field', (fieldname, val) => {
// TODO(developer): Process submitted field values here
console.log(`Processed field ${fieldname}: ${val}.`);
fields[fieldname] = val;
});
const fileWrites = [];
//res.status(200).send();
// This code will process each file uploaded.
busboy.on('file', (fieldname, file, filename) => {
// Note: os.tmpdir() points to an in-memory file system on GCF
// Thus, any files in it must fit in the instance's memory.
console.log(`Processed file ${filename}`);
const filepath = path.join(tmpdir, filename);
uploads[fieldname] = filepath;
const writeStream = fs.createWriteStream(filepath);
file.pipe(writeStream);
// File was processed by Busboy; wait for it to be written to disk.
const promise = new Promise((resolve, reject) => {
file.on('end', () => {
writeStream.end();
});
writeStream.on('finish', resolve);
writeStream.on('error', reject);
});
fileWrites.push(promise);
});
// Triggered once all uploaded files are processed by Busboy.
// We still need to wait for the disk writes (saves) to complete.
busboy.on('finish', () => {
Promise.all(fileWrites).then(() => {
// TODO(developer): Process saved files here
for (const name in uploads) {
const file = uploads[name];
bucket.upload(file).then(()=>{
fs.unlinkSync(file);
console.log("saved");
});
}
res.send();
});
});
busboy.end(req.rawBody);
});
Update:
Added functionality to send folder names through the http request as well, once again probably not perfect but works, originally pretty hard to figure out.
New index.js for my cloud functions for those who want custom folder names sent through http request.
const functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require("./service-account-credentials.json");
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
storageBucket: "YOURURL.appspot.com"
});
var bucket = admin.storage().bucket();
const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');
exports.uploadModel2s = functions.https.onRequest((req, res) => {
res.set('Access-Control-Allow-Origin', '*');
res.set('Access-Control-Allow-Credentials', 'true');
if (req.method === 'OPTIONS'){
console.log("send response 204");
res.set('Access-Control-Allow-Methods', 'GET');
res.set('Access-Control-Allow-Headers', 'Authorization,ModelId');
res.set('Access-Control-Max-Age', '3600');
return res.status(204).send();
}
console.log("method ---> "+req.method);
const modelid = req.headers['modelid'];
console.log("modelid ---> "+modelid);
console.log(req.headers);
const busboy = new Busboy({headers: req.headers});
const tmpdir = os.tmpdir();
const fields = {};
const uploads = {};
busboy.on('field', (fieldname, val) => {
console.log(`Processed field ${fieldname}: ${val}.`);
fields[fieldname] = val;
});
const fileWrites = [];
busboy.on('file', (fieldname, file, filename) => {
console.log(`Processed 2 file ${modelid} ${filename}`);
const filepath = path.join(tmpdir,filename);
uploads[fieldname] = filepath;
console.log('PATH: '+filepath);
const writeStream = fs.createWriteStream(filepath);
file.pipe(writeStream);
const promise = new Promise((resolve, reject) => {
file.on('end', () => {
writeStream.end();
});
writeStream.on('finish', resolve);
writeStream.on('error', reject);
});
fileWrites.push(promise);
});
busboy.on('finish', () => {
Promise.all(fileWrites).then(() => {
for (const name in uploads){
const file = uploads[name];
const destpath = "/m_"+modelid+"/"+name;
console.log("SAVE ME HERE: "+destpath);
bucket.upload(file,{ destination: destpath,public: true }).then(() => {
if(name == "model.weights.bin"){
fs.unlinkSync(file);
console.log("save weights");
res.send();
}else{
console.log("save model");
}
});
}
});
});
busboy.end(req.rawBody);
});
New client code
var api = 'https://yourproject.cloudfunctions.net/uploadModel2s';
var res = await model.save(tf.io.browserHTTPRequest(api,
{method: 'POST', headers:{'Authorization':'test','ModelId':m}}));
Security risk
In order to speed development I am using public: true on upload so I can access file easily for download, however this is definitely not secure.
