Predict tensorflow handposes with Node.js, capturing pictures from mjpg-streamer url

README.md

Node.js (with websockets) + tensorflow/handpose

Requirements

• mjpg-streamer: https://gist.github.com/poulou0/06dc75cf28aca764e2ceb75710aa2cf9
• Node.js: https://github.com/nodesource/distributions#debinstall

Use

cd ~ && \
git clone https://gist.github.com/poulou0/0c492ceff9b017cee6e995aae145e702 node-tfhandpose-mjpgstreamer && \
cd node-tfhandpose-mjpgstreamer && \
npm install && \
node index.js

Go to: http://127.0.0.1:8081

index.js

import * as tf from "@tensorflow/tfjs";
import * as handpose from '@tensorflow-models/handpose';
import * as jpeg from "jpeg-js";
import '@tensorflow/tfjs-backend-wasm';
import request from 'request';
import * as http from 'http';
import WebSocket, { WebSocketServer } from 'ws';

(async () => {
  console.time("init");
  await tf.ready();
  await tf.setBackend('wasm'); //redundant
  const model = await handpose.load();
  console.timeEnd("init");

  console.time("setup web and websocket servers");
  const server = http.createServer(function (req, res) {
    res.writeHead(200);
    res.end(`
    <style>body{margin:0}#hand{position: absolute; border: 3px solid green; display: none}</style>
    <script>
      var conn = new WebSocket('ws://' + location.hostname + ':8082');
      conn.onmessage = (e) => {
        const data = JSON.parse(e.data);
        console.log(data.boundingBox)
        document.getElementById("img").src=data.image;
        if (data.boundingBox) {
          document.getElementById("hand").style.display='block';
          document.getElementById("hand").style.top=data.boundingBox.topLeft[1];
          document.getElementById("hand").style.left=data.boundingBox.topLeft[0];
          document.getElementById("hand").style.width=data.boundingBox.bottomRight[0] - data.boundingBox.topLeft[0];
          document.getElementById("hand").style.height=data.boundingBox.bottomRight[1] - data.boundingBox.topLeft[1];
        } else {
          document.getElementById("hand").style.display='none';
        }
      };
    </script>
    <img id="img"/>
    <div id="hand"></div>
    `);
  });
  server.listen(8081);
  const wss = new WebSocketServer({ port: 8082 });
  console.timeEnd("setup web and websocket servers");

  wss.on('connection', function connection(ws) {
    console.log(wss.clients.size);
    const loop = () => {
      console.time("request");
      request('http://127.0.0.1:8080/?action=snapshot', {encoding: null}, async (error, result, body) => {
        console.timeEnd("request");
        console.time("decode");
        const jpegImageData = jpeg.decode(body);
        console.timeEnd("decode");
        console.time("analyse");
        const predictions = await model.estimateHands(jpegImageData);
        console.timeEnd("analyse");
        wss.clients.forEach(function each(client) {
          if (client.readyState === WebSocket.OPEN) {
            client.send(JSON.stringify({
              image: 'data:image/jpg;base64, ' + Buffer.from(body).toString('base64'),
              width: jpegImageData.width,
              height: jpegImageData.height,
              boundingBox: predictions[0]?.boundingBox,
            }));
          }
        });
        if (wss.clients.size > 0) loop();
      });
    }
    if (wss.clients.size == 1) loop();
  });

  // const loop = () => {
  //   console.time("request");
  //   request('http://localhost:8080/?action=snapshot', {encoding: null}, async (error, result, body) => {
  //     console.timeEnd("request");
  //     console.time("decode");
  //     const jpegImageData = jpeg.decode(body);
  //     console.timeEnd("decode");
  //     console.time("analyse");
  //     const predictions = await model.estimateHands(jpegImageData);
  //     console.timeEnd("analyse");
  //     console.log(predictions[0]?.boundingBox);
  //     loop();
  //   });
  // }
  // loop();
})();

package.json

{
  "name": "node-handpose-mjpgstreamer",
  "version": "1.0.0",
  "description": "Node.js + @tensorflow-models/handpose + mjpg-streamer working together",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "start": "node index.js"
  },
  "type": "module",
  "author": "poulou.0",
  "license": "ISC",
  "dependencies": {
    "@tensorflow-models/handpose": "^0.0.7",
    "@tensorflow/tfjs": "^3.20.0",
    "@tensorflow/tfjs-backend-wasm": "^3.20.0",
    "jpeg-js": "^0.4.4",
    "request": "^2.88.2",
    "ws": "^8.8.1"
  }
}