javascript - 如何在纯 Javascript webapp 中使用经过训练的 TensorFlow 模型

Question

我一直在学习本教程: https://docs.annotations.ai/workshops/object-detection/6.html

到了第 6 步，一旦我看到 webapp 示例，它是在 ReactJS 中完成的，我不知道如何将它转换为我们特定用例的纯 JS。我能够做到这一点:

脚本.js

var videoRef = document.getElementById("video");

if(navigator.mediaDevices.getUserMedia) {
  try {
    let stream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { facingMode: { exact: "environment" } } });
  } catch(error) {}

  navigator.mediaDevices.enumerateDevices()
  .then(function(devices) {

    // Remove non camera devices
    for(var i = devices.length - 1; i>=0; i--) {
      var device = devices[i];
      if(device.kind != 'videoinput') {
        devices.splice(i, 1);
      }
      if(!device.kind) {
        devices.splice(i, 1);
      }
    }

    // Force camera to back camera for mobile devices
    var activeDevice = devices[0];
    for(var i in devices) {
      var device = devices[i];
      if(device.label) {
        if(device.label.toLowerCase().indexOf('back') > -1) {
          activeDevice = device;
        }
      }
    }

    const constraints = {video: {deviceId: activeDevice.deviceId ? {exact: activeDevice.deviceId} : undefined}, audio: false};

    const webcamPromise = navigator.mediaDevices
    .getUserMedia(constraints)
    .then(stream => {
      window.stream = stream;
      videoRef.srcObject = stream;

      return new Promise(resolve => {
        videoRef.onloadedmetadata = () => {
          resolve();
        };
      });
    }, (error) => {
      console.error(error, 'camera error');
    });

    const loadlModelPromise = cocoSsd.load({modelUrl: 'https://nanonets.s3-us-west-2.amazonaws.com/uploadedfiles/87be4e38-b40d-4217-898b-fd619319c2e4/ssd/model.json'});

    Promise.all([loadlModelPromise, webcamPromise])
    .then(values => {
      detectFromVideoFrame(values[0], videoRef);
    })
    .catch(error => {
      console.error(error, 'error loading promises');
    })

  })
}


function detectFromVideoFrame(model, video) {
  model.detect(video).then(predictions => {

    console.log(predictions, 'predictions found');

    requestAnimationFrame(() => {
      detectFromVideoFrame(model, video);
    });
  }, (error) => {
    console.error(error, 'Tensorflow Error');
  });
};

在 HTML 中我包含一个 coco-ssd.js我也认为我需要修改的文件，但我不确定如何生成该文件:

<script src="/lib/coco-ssd.js"></script>
<script src="https://unpkg.com/babel-standalone@6.26.0/babel.min.js"></script>

该代码适用于 pre-defined coco-ssd model但是按照教程我无法弄清楚如何使用我自己的模型，这里是生成的文件:

现在我需要了解如何在上面的 Javascript 中使用这些文件。

我想我需要更改这些行:

const loadlModelPromise = cocoSsd.load({modelUrl: 'https://nanonets.s3-us-west-2.amazonaws.com/uploadedfiles/87be4e38-b40d-4217-898b-fd619319c2e4/ssd/model.json'});

并包含一个不同的 coco-ssd.js文件:

<script src="/lib/coco-ssd.js"></script>

但不清楚要从生成的文件夹结构中包含哪些文件，这就是我遇到的问题。

Answer 1

您可以将脚本用于 TensorFlow js 和 coco SSD

<!-- Load TensorFlow.js. This is required to use coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"> </script>
<!-- Load the coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd"> </script>

第 1 步:创建 index.html 文件
index.html
注意:您可以使用 modelUrl 功能加载您自己托管在某处的模型

<!-- Load TensorFlow.js. This is required to use coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"> </script>
<!-- Load the coco-ssd model. -->
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd"> </script>

<!-- Replace this with your image. Make sure CORS settings allow reading the image! -->
<img id="img" src="https://images.pexels.com/photos/20787/pexels-photo.jpg?auto=compress&cs=tinysrgb&dpr=1&w=500" crossorigin="anonymous"/>
<!-- Place your code in the script tag below. You can also use an external .js file -->
<script>
  // Notice there is no 'import' statement. 'cocoSsd' and 'tf' is
  // available on the index-page because of the script tag above.

  const img = document.getElementById('img');

  // Load the model.
  //Note: cocoSsd.load() will also work on this without parameter. It will default to the coco ssd model
  cocoSsd.load({ modelUrl: 'PATH TO MODEL JSON FILE' }).then(model => {
    // detect objects in the image.
    model.detect(img).then(predictions => {
      console.log('Predictions: ', predictions);
    });
  });
</script>

第 2 步:假设您有一个 nodejs 服务器来运行索引文件并通过访问 localhost:3000 在本地提供它。
Server.js

const express = require('express');
app = express();

app.get('/',function(req,res) {
    res.sendFile('./index.html', { root: __dirname });
});
const port = 3000
app.listen(port, function(){
    console.log(`Listening at port ${port}`);
})
'''