google-cloud-functions - 将 TensorflowJS 模型保存到 firebase 存储的最佳实践？

Question

似乎可以进行基于“post”http 请求的保存。

https://www.tensorflow.org/js/guide/save_load

https://firebase.google.com/docs/functions/http-events

担心我可能会过于复杂，是否有一种简单的方法可以在客户端js中创建json文件(也许也避免保存到本地存储中？)，然后简单地使用put函数上传到存储? https://firebase.google.com/docs/storage/web/upload-files

在这些领域经验不足，找不到任何特定于此场景的内容。

调查:

似乎这是使用云函数 Write image file to Firebase Storage from HTTP function

更新:

目前正在研究使用内置 Tensorflowjs HTTP 请求基于保存(包含权重的 bin 文件和模型结构 json 文件)，该保存将由云函数处理，而不是确定这是否是最好的方法或者是否会起作用，但到目前为止我拥有最好的领先优势。现在还在研究如何通过http请求发送文件夹名称？

Answer 1

我的客户端 Tensorflow JS 请求通过 HTTP 请求保存到云存储。可能有更好的方法来做到这一点，但这至少对我有用。

var api = 'https://us-central1-your-uniquelocation.cloudfunctions.net/uploadModel';
var res = await model.save(tf.io.browserHTTPRequest(api,
        {method: 'POST', headers:{'Authorization':'test','Content-Type': 'multipart/form-data; boundary=----WebKitFormBoundaryVoB3gVGBQHG0btQR'}}));
console.log(res);

然后是我的云函数的index.js

const functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require("./service-account-credentials.json");

admin.initializeApp({
    credential: admin.credential.cert(serviceAccount),
    //databaseURL: "https://yoururl.firebaseio.com",
    storageBucket: "yoururl.appspot.com"
});

var bucket = admin.storage().bucket("yoururl.appspot.com");

const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');


exports.uploadModel = functions.https.onRequest((req, res) => {
    res.set('Access-Control-Allow-Origin', '*');
    res.set('Access-Control-Allow-Credentials', 'true');
    if (req.method === 'OPTIONS') {
        // Send response to OPTIONS requests
        console.log("send response 204"); 

        res.set('Access-Control-Allow-Methods', 'GET');
        res.set('Access-Control-Allow-Headers', 'Authorization');
        res.set('Access-Control-Max-Age', '3600');

        res.status(204).send();
     }

    console.log("metho--->"+req.method);
    console.log(req.headers);

    const busboy = new Busboy({headers: req.headers});
    const tmpdir = os.tmpdir();

    // This object will accumulate all the fields, keyed by their name
    const fields = {};

    // This object will accumulate all the uploaded files, keyed by their name.
    const uploads = {};

    // This code will process each non-file field in the form.
    busboy.on('field', (fieldname, val) => {
        // TODO(developer): Process submitted field values here
        console.log(`Processed field ${fieldname}: ${val}.`);
        fields[fieldname] = val;
    });

    const fileWrites = [];
    //res.status(200).send();

    // This code will process each file uploaded.
    busboy.on('file', (fieldname, file, filename) => {
        // Note: os.tmpdir() points to an in-memory file system on GCF
        // Thus, any files in it must fit in the instance's memory.
        console.log(`Processed file ${filename}`);
        const filepath = path.join(tmpdir, filename);
        uploads[fieldname] = filepath;

        const writeStream = fs.createWriteStream(filepath);
        file.pipe(writeStream);

        // File was processed by Busboy; wait for it to be written to disk.
        const promise = new Promise((resolve, reject) => {
            file.on('end', () => {
                writeStream.end();
            });
            writeStream.on('finish', resolve);
            writeStream.on('error', reject);
        });
        fileWrites.push(promise);
    });

    // Triggered once all uploaded files are processed by Busboy.
    // We still need to wait for the disk writes (saves) to complete.
    busboy.on('finish', () => {
        Promise.all(fileWrites).then(() => {
        // TODO(developer): Process saved files here
            for (const name in uploads) {
                const file = uploads[name];
                bucket.upload(file).then(()=>{
                    fs.unlinkSync(file);
                    console.log("saved");
                });
            }
            res.send();
        });
    });
    busboy.end(req.rawBody);
});

更新:

添加了通过 http 请求发送文件夹名称的功能，再次可能并不完美，但可以工作，最初很难弄清楚。

针对我的云功能的新index.js，适合那些希望通过 http 请求发送自定义文件夹名称的人。

const functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require("./service-account-credentials.json");

admin.initializeApp({
    credential: admin.credential.cert(serviceAccount),
    storageBucket: "YOURURL.appspot.com"
});

var bucket = admin.storage().bucket();

const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');

exports.uploadModel2s = functions.https.onRequest((req, res) => {
    res.set('Access-Control-Allow-Origin', '*');
    res.set('Access-Control-Allow-Credentials', 'true');
    if (req.method === 'OPTIONS'){
        console.log("send response 204"); 
        res.set('Access-Control-Allow-Methods', 'GET');
        res.set('Access-Control-Allow-Headers', 'Authorization,ModelId');
        res.set('Access-Control-Max-Age', '3600');
        return res.status(204).send();
    }

    console.log("method ---> "+req.method);
    const modelid = req.headers['modelid'];
    console.log("modelid ---> "+modelid);
    console.log(req.headers);

    const busboy = new Busboy({headers: req.headers});
    const tmpdir = os.tmpdir();

    const fields = {};
    const uploads = {};

    busboy.on('field', (fieldname, val) => {
        console.log(`Processed field ${fieldname}: ${val}.`);
        fields[fieldname] = val;
    });

    const fileWrites = [];

    busboy.on('file', (fieldname, file, filename) => {
        console.log(`Processed 2 file ${modelid} ${filename}`);
        const filepath = path.join(tmpdir,filename);
        uploads[fieldname] = filepath;
        console.log('PATH: '+filepath);
        const writeStream = fs.createWriteStream(filepath);
        file.pipe(writeStream);

        const promise = new Promise((resolve, reject) => {
            file.on('end', () => {
                writeStream.end();
            });
            writeStream.on('finish', resolve);
            writeStream.on('error', reject);
        });
        fileWrites.push(promise);
    });

    busboy.on('finish', () => {
        Promise.all(fileWrites).then(() => {
            for (const name in uploads){
                const file = uploads[name];
                const destpath = "/m_"+modelid+"/"+name;
                console.log("SAVE ME HERE: "+destpath);
                bucket.upload(file,{ destination: destpath,public: true }).then(() => {
                    if(name == "model.weights.bin"){
                        fs.unlinkSync(file);
                        console.log("save weights");
                        res.send();
                    }else{
                        console.log("save model");
                    }
                });
            }
        });
    });
    busboy.end(req.rawBody);
});

新的客户端代码

var api = 'https://yourproject.cloudfunctions.net/uploadModel2s';
var res = await model.save(tf.io.browserHTTPRequest(api,
                {method: 'POST', headers:{'Authorization':'test','ModelId':m}}));

安全风险

为了加快开发速度，我在上传时使用public: true，这样我就可以轻松访问文件进行下载，但这绝对不安全。