0

The goal:

I've got a functioning system that can detect hands with TensorFlowJS's [Hand Pose Detection][1]. However, each detection is slow, so I want to pull the model creation and inference into a Web Worker so that I can keep the UI responsive. However, I can't manage to figure out how to import everything I need.

(The system also uses another model coco-ssd for object detection/identification, plus a global variable in index.js to hold the detected hands data, and a global function to update this data, which might be unusual, but that's separate to this issue, I think.)


Functional, no Web Worker code:

index.js

import {CreateHandDetector, CreateObjectDetector, DetectObjects} from './models';

let globalHands = [];

async function Init() {
    await CreateHandDetector();
}

while(true) {
     await DetectObjects();
     /* At this point, I've got the hands data successfully */
}

models.js

import * as handPoseDetection from '@tensorflow-models/hand-pose-detection';
require('@tensorflow/tfjs-backend-cpu');
require('@tensorflow/tfjs-backend-webgl');
const cocoSsd = require('@tensorflow-models/coco-ssd');

export async function CreateHandDetector() {
    const model = handPoseDetection.SupportedModels.MediaPipeHands;
    const detectorConfig = {
        runtime: 'mediapipe',
        solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands',
        modelType: 'lite'
    };
    detectors.handDetector = await handPoseDetection.createDetector(model, detectorConfig);
    console.log("Hand detector made");
}

export async function DetectObjects() {
     /* Some logic to cancel if hand detection turned off by user */
     await DetectHands();
}

async function DetectHands() {
    const video = document.querySelector("#liveVideo");
    const hands = await detectors.handDetector.estimateHands(video, {flipHorizontal: false});
     /* Some logic to modify the hands data and send it back */
}

My attempt to migrate to a Web Worker

*index.js*
    const modelWorker = new Worker(new URL('./modelsWorker.js', import.meta.url), {type: "module"});
    modelWorker.onmessage = (e) => {
        const modelData = e.data;
        if(modelData.action === "ping")   { console.log(`Ping reply: ${modelData.message}`); }
        if(modelData.action === "init")   { console.log(`Ran init, reply: ${modelData.message}`); }
    }
    
    modelWorker.postMessage({action: "ping"}); // output: "Ping reply: pong"
    modelWorker.postMessage({action: "init"});

modelsWorker.js

//require('@tensorflow-models/hand-pose-detection');
//import * as handPoseDetection from '@tensorflow-models/hand-pose-detection';
//import * as handPoseDetection from './tensorflowImports/hand-pose-detection';
importScripts('https://cdn.jsdelivr.net/npm/@tensorflow-models/hand-pose-detection');
//importScripts('./tensorflowImports/hand-pose-detection.js');
//require('./tensorflowImports/hand-pose-detection');
//const handPoseDetection = require('./tensorflowImports/hand-pose-detection');
//require('./tensorflowImports/hands');
//importScripts('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu');  // TypeError: a.kernel_impls is undefinedtfjs-backend-cpu:17:3180
//importScripts('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl');
require('./tensorflowImports/tfjs-backend-cpu');
require('./tensorflowImports/tfjs-backend-webgl');
const cocoSsd = require('./tensorflowImports/coco-ssd');

const detectors = {handDetector: null, objectDetector: null, pauseAll: false}

onmessage = async (e) => {
    const modelInputData = e.data;
    const returnData = {action: modelInputData.action};

    if(modelInputData.action === "ping") {
        console.log("pong");
        returnData.message = "pong";
        postMessage(returnData);
        return;
    }

    if(modelInputData.action === "init") {
        returnData.success = await CreateDetectors();
        postMessage(returnData);
    }
}

async function CreateDetectors() {
    await CreateHandDetector();
    /* Some logic that runs when the detector is created successfully */
    return true;
}

export async function CreateHandDetector() {
    const model = handPoseDetection.SupportedModels.MediaPipeHands;
    const detectorConfig = {
        runtime: 'mediapipe',
        solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/hands',
        modelType: 'lite'
    };

    // Dies here. Error:
    // TypeError: t.Hands is not a constructor    hand-pose-detection:17:2509
    detectors.handDetector = await handPoseDetection.createDetector(model, detectorConfig);

    console.log("Hand detector made"); // Never printed
}


What I need

As you can see, I tried many different ways to import the hand detection model. All of them ended in tears. I assume I'm importing incorrectly, but who knows.

This answer just uses importScripts, but I can't replicate it.

This answer uses import {} from which I'd love to use, but doesn't work either.

cdehaan
  • 394
  • 5
  • 10

0 Answers0