The goal:
I've got a functioning system that can detect hands with TensorFlowJS's [Hand Pose Detection][1]. However, each detection is slow, so I want to pull the model creation and inference into a Web Worker so that I can keep the UI responsive. However, I can't manage to figure out how to import everything I need.(The system also uses another model coco-ssd for object detection/identification, plus a global variable in index.js to hold the detected hands data, and a global function to update this data, which might be unusual, but that's separate to this issue, I think.)
Functional, no Web Worker code:
index.js
import {CreateHandDetector, CreateObjectDetector, DetectObjects} from './models';
let globalHands = [];
async function Init() {
await CreateHandDetector();
}
while(true) {
await DetectObjects();
/* At this point, I've got the hands data successfully */
}
models.js
import * as handPoseDetection from '@tensorflow-models/hand-pose-detection';
require('@tensorflow/tfjs-backend-cpu');
require('@tensorflow/tfjs-backend-webgl');
const cocoSsd = require('@tensorflow-models/coco-ssd');
export async function CreateHandDetector() {
const model = handPoseDetection.SupportedModels.MediaPipeHands;
const detectorConfig = {
runtime: 'mediapipe',
solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands',
modelType: 'lite'
};
detectors.handDetector = await handPoseDetection.createDetector(model, detectorConfig);
console.log("Hand detector made");
}
export async function DetectObjects() {
/* Some logic to cancel if hand detection turned off by user */
await DetectHands();
}
async function DetectHands() {
const video = document.querySelector("#liveVideo");
const hands = await detectors.handDetector.estimateHands(video, {flipHorizontal: false});
/* Some logic to modify the hands data and send it back */
}
My attempt to migrate to a Web Worker
*index.js* const modelWorker = new Worker(new URL('./modelsWorker.js', import.meta.url), {type: "module"});
modelWorker.onmessage = (e) => {
const modelData = e.data;
if(modelData.action === "ping") { console.log(`Ping reply: ${modelData.message}`); }
if(modelData.action === "init") { console.log(`Ran init, reply: ${modelData.message}`); }
}
modelWorker.postMessage({action: "ping"}); // output: "Ping reply: pong"
modelWorker.postMessage({action: "init"});
modelsWorker.js
//require('@tensorflow-models/hand-pose-detection');
//import * as handPoseDetection from '@tensorflow-models/hand-pose-detection';
//import * as handPoseDetection from './tensorflowImports/hand-pose-detection';
importScripts('https://cdn.jsdelivr.net/npm/@tensorflow-models/hand-pose-detection');
//importScripts('./tensorflowImports/hand-pose-detection.js');
//require('./tensorflowImports/hand-pose-detection');
//const handPoseDetection = require('./tensorflowImports/hand-pose-detection');
//require('./tensorflowImports/hands');
//importScripts('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu'); // TypeError: a.kernel_impls is undefinedtfjs-backend-cpu:17:3180
//importScripts('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl');
require('./tensorflowImports/tfjs-backend-cpu');
require('./tensorflowImports/tfjs-backend-webgl');
const cocoSsd = require('./tensorflowImports/coco-ssd');
const detectors = {handDetector: null, objectDetector: null, pauseAll: false}
onmessage = async (e) => {
const modelInputData = e.data;
const returnData = {action: modelInputData.action};
if(modelInputData.action === "ping") {
console.log("pong");
returnData.message = "pong";
postMessage(returnData);
return;
}
if(modelInputData.action === "init") {
returnData.success = await CreateDetectors();
postMessage(returnData);
}
}
async function CreateDetectors() {
await CreateHandDetector();
/* Some logic that runs when the detector is created successfully */
return true;
}
export async function CreateHandDetector() {
const model = handPoseDetection.SupportedModels.MediaPipeHands;
const detectorConfig = {
runtime: 'mediapipe',
solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands',
modelType: 'lite'
};
// Dies here. Error:
// TypeError: t.Hands is not a constructor hand-pose-detection:17:2509
detectors.handDetector = await handPoseDetection.createDetector(model, detectorConfig);
console.log("Hand detector made"); // Never printed
}
What I need
As you can see, I tried many different ways to import the hand detection model. All of them ended in tears. I assume I'm importing incorrectly, but who knows.This answer just uses importScripts
, but I can't replicate it.
This answer uses import {} from
which I'd love to use, but doesn't work either.