in react-native how to use body-pix

Question

import { Camera, CameraType } from 'expo-camera';
import React, { useRef } from "react";
// import logo from './logo.svg';
import * as tf from "@tensorflow/tfjs";
import * as bodyPix from "@tensorflow-models/body-pix";
import { useState } from 'react';
import { Button, StyleSheet, Text, TouchableOpacity, View } from 'react-native';
//AFTER IMPORT 
export default function App() {
  const [type, setType] = useState(CameraType.back);
  const [permission, requestPermission] = Camera.useCameraPermissions();
  const canvasRef = useRef(null);
//next part
  const runBodysegment = async () => {
    const net = await bodyPix.load();
    console.log("BodyPix model loaded.");
    //  Loop and detect hands
    setInterval(() => {
      detect(net);
    }, 100)
  };
//next part

  const detect = async (net) => {
    const person = await net.segmentPersonParts(video);
    console.log(person);

    const coloredPartImage = bodyPix.toColoredPartMask(person);
    bodyPix.drawMask(
      canvasRef.current,
      video,
      coloredPartImage,
      0.7,
      0,
      false
    );
    runBodysegment();

  if (!permission) {
    // Camera permissions are still loading
    return <View />;
  }

  if (!permission.granted) {
    // Camera permissions are not granted yet
    return (
      <View style={styles.container}>
        <Text style={{ textAlign: 'center' }}>We need your permission to show the camera</Text>
        <Button onPress={requestPermission} title="grant permission" />
      </View>
    );
  }

  function toggleCameraType() {
    setType(current => (current === CameraType.back ? CameraType.front : CameraType.back));
  }

  return (
    <View style={styles.container}>
      <Camera style={styles.camera} type={type}>
        <View style={styles.buttonContainer}>
          <TouchableOpacity style={styles.button} onPress={toggleCameraType}>
            <Text style={styles.text}>Flip Camera</Text>
          </TouchableOpacity>
        </View>
      </Camera>
    </View>
  );
}

//next part
const styles = StyleSheet.create({
  container: {
    flex: 1,
    justifyContent: 'center',
  },
  camera: {
    flex: 1,
  },
  buttonContainer: {
    flex: 1,
    flexDirection: 'row',
    backgroundColor: 'transparent',
    margin: 64,
  },
  button: {
    flex: 1,
    alignSelf: 'flex-end',
    alignItems: 'center',
  },
  text: {
    fontSize: 24,
    fontWeight: 'bold',
    color: 'white',
  },
});
};

I want to use body-pix in my react-native app for android . can any one help me how to do it . I want my app to open a camera and in my camera there will be a body body-pix working in my android app I want my react-native app to work properly with body-pix I had try it many time time but I can't do it properly

Answer 1

import { Camera, CameraType } from 'expo-camera';
import { useState,useRef, useEffect } from 'react';
import { Button, StyleSheet, Text, TouchableOpacity, View } from 'react-native';
import * as tf from "@tensorflow/tfjs";
import * as bodyPix from "@tensorflow-models/body-pix";
import {cameraWithTensors} from '@tensorflow/tfjs-react-native'

export default function TensorCamera() {
  const [permission, requestPermission] = Camera.useCameraPermissions();
  const [video, setVideo] = useState(null)
  const cameraRef = useRef(null);
  const TensorCamera = cameraWithTensors(Camera);
  const textureDims = Platform.OS === 'ios' ?
{
  height: 1920,
  width: 1080,
} :
{
  height: 1200,
  width: 1600,
};
useEffect(() => {
  const loadTf = async ()=>  {
   await tf.ready()
  }

  return () => {
    loadTf()
  }
})


 

  const runBodysegment = async (images) => {
      const net = await bodyPix.load();
      console.log("BodyPix model loaded.");
      setVideo(images?.next().value)
      detect(net)
        
  };
   
  const detect = async (net) => {
    // Check data is available
    if(video){
      const person = await net.segmentPersonParts(video);
      const newArray = []
      person?.allPoses[0]?.keypoints.forEach(element => {
        if(element.score >= 0.6){

          newArray.push({part:element.part, score: element.score})
        }
      });
      console.log(newArray)}}

      useEffect(() => {
        const interval = setInterval(() => {
          runBodysegment();
        }, 10000);
        return () => clearInterval(interval);
      }, []);


  // runBodysegment()

  if (!permission) {
    // Camera permissions are still loading
    return <View />;
  }

  if (!permission.granted) {
    // Camera permissions are not granted yet
    return (
      <View style={styles.container}>
        <Text style={{ textAlign: 'center' }}>We need your permission to show the camera</Text>
        <Button onPress={requestPermission} title="grant permission" />
      </View>
    );
  }

  return (
    <View style={styles.container}>
      <TensorCamera 
      ref={cameraRef}
    style={styles.camera} 
    type={Camera.Constants.Type.back}
    onReady={runBodysegment}
    resizeHeight={200}
    resizeWidth={152}
    resizeDepth={3}
    autorender={true}
    cameraTextureHeight={textureDims.height}
    cameraTextureWidth={textureDims.width}
    ratio='4:3'
 />
    </View>
  );
}

const styles = StyleSheet.create({
  container: {
    flex: 1,
    justifyContent: 'center',
    alignItems:'center'
  },
  camera: {
    width:300,
    height:500
  },
  buttonContainer: {
    flex: 1,
    flexDirection: 'row',
    backgroundColor: 'transparent',
    margin: 64,
  },
  button: {
    flex: 1,
    alignSelf: 'flex-end',
    alignItems: 'center',
  },
  text: {
    fontSize: 24,
    fontWeight: 'bold',
    color: 'white',
  },
});