How can I drag an object in x and z constrained in y in react-three/fiber with an orthographic camera which may be moved via orbitcontrols?

Question

I would like to be able to drag an object across a plane (think of a piece on a chess board) in a canvas using React-three-fiber and an orthographic camera.

Here is an example (not mine) of that working with a fixed camera position: https://codepen.io/kaolay/pen/bqKjVz

But I would like to be able to move the camera too - so I have added Orbitcontrols which are disabled when the object is being dragged.

I have a code sandbox here with my attempt based many other's examples: https://codesandbox.io/s/inspiring-franklin-2r3ri?file=/src/Obj.jsx

The main code is in two files, App.jsx with the canvas, camera, and orbitcontrols. And Obj.jsx with the mesh that gets dragged as well as the dragging logic inside of a use-gesture useDrag function.

App.jsx

import React, { useState } from "react";
import { Canvas } from "@react-three/fiber";
import Obj from "./Obj.jsx";
import { OrthographicCamera, OrbitControls } from "@react-three/drei";
import * as THREE from "three";

export default function App() {
  const [isDragging, setIsDragging] = useState(false);

  return (
    <Canvas style={{ background: "white" }} shadows dpr={[1, 2]}>
      <ambientLight intensity={0.5} />
      <directionalLight
        intensity={0.5}
        castShadow
        shadow-mapSize-height={512}
        shadow-mapSize-width={512}
      />

      <mesh rotation={[-Math.PI / 2, 0, 0]} receiveShadow>
        <planeBufferGeometry attach="geometry" args={[10, 10]} receiveShadow />
        <meshPhongMaterial
          attach="material"
          color="#ccc"
          side={THREE.DoubleSide}
          receiveShadow
        />
      </mesh>

      <Obj setIsDragging={setIsDragging} />

      <OrthographicCamera makeDefault zoom={50} position={[0, 40, 200]} />

      <OrbitControls minZoom={10} maxZoom={50} enabled={!isDragging} />
    </Canvas>
  );
}

Obj.jsx (with the offending code in the Use Drag function)

import React, { useState } from "react";
import { useDrag } from "@use-gesture/react";
import { animated, useSpring } from "@react-spring/three";
import { useThree } from "@react-three/fiber";
import * as THREE from "three";

function Obj({ setIsDragging }) {
  const { camera } = useThree();
  const [pos, setPos] = useState([0, 1, 0]);
  const { size, viewport } = useThree();
  const aspect = size.width / viewport.width;
  const [spring, api] = useSpring(() => ({
    // position: [0, 0, 0],
    position: pos,
    scale: 1,
    rotation: [0, 0, 0],
    config: { friction: 10 }
  }));
  const bind = useDrag(
    ({ active, delta, movement: [x, y], velocity, timeStamp, memo = 0 }) => {
      if (active) {

 //// THIS IS THE CODE THAT I KNOW IS NOT WORKING /////

        let vDir = new THREE.Vector3();
        let vPos = new THREE.Vector3(
          (x / window.innerWidth) * 2 - 1,
          -(y / window.innerHeight) * 2 + 1,
          0.5
        ).unproject(camera);

        vDir.copy(vPos).sub(camera.position).normalize();
        let flDistance = -camera.position.z / vDir.z;
        vPos = vPos.copy(camera.position).add(vDir.multiplyScalar(flDistance));
        const arbitraryFactor = 1; // I suspect this has to reflect the distance from camera in all dims...
        setPos([vPos.x * arbitraryFactor, 1.5, -vPos.y * arbitraryFactor]);

 //// END /////
      }

      setIsDragging(active);

      api.start({
        // position: active ? [x / aspect, -y / aspect, 0] : [0, 0, 0],
        position: pos,
        scale: active ? 1.2 : 1,
        rotation: [y / aspect, x / aspect, 0]
      });
      return timeStamp;
    }
  );

  return (
    <animated.mesh {...spring} {...bind()} castShadow>
      <dodecahedronBufferGeometry
        castShadow
        attach="geometry"
        args={[1.4, 0]}
      />
      <meshNormalMaterial attach="material" />
    </animated.mesh>
  );
}

export default Obj;

A few references that have been helpful but have not got me there yet! Mouse / Canvas X, Y to Three.js World X, Y, Z

https://codesandbox.io/s/react-three-fiber-gestures-forked-lpfv3?file=/src/App.js:1160-1247

https://codesandbox.io/embed/react-three-fiber-gestures-08d22?codemirror=1

https://codesandbox.io/s/r3f-lines-capture-1gkvp

https://github.com/pmndrs/react-three-fiber/discussions/641

And finally relinking to my code sandbox example again: https://codesandbox.io/s/inspiring-franklin-2r3ri?file=/src/Obj.jsx:0-1848

Answer 1

I took a different approach after I realized that React-Three-Fiber passes event info into useDrag, which contains the coordinate and Ray information I needed.

https://codesandbox.io/s/musing-night-wso9v?file=/src/App.jsx

App.jsx

import React, { useState } from "react";
import { Canvas } from "@react-three/fiber";
import Obj from "./Obj.jsx";
import { OrthographicCamera, OrbitControls } from "@react-three/drei";
import * as THREE from "three";

export default function App() {
  const [isDragging, setIsDragging] = useState(false);
  const floorPlane = new THREE.Plane(new THREE.Vector3(0, 1, 0), 0);

  return (
    <Canvas style={{ background: "white" }} shadows dpr={[1, 2]}>
      <ambientLight intensity={0.5} />
      <directionalLight
        intensity={0.5}
        castShadow
        shadow-mapSize-height={512}
        shadow-mapSize-width={512}
      />

      <mesh
        rotation={[-Math.PI / 2, 0, 0]}
        position={[0, -0.1, 0]}
        receiveShadow
      >
        <planeBufferGeometry attach="geometry" args={[10, 10]} receiveShadow />
        <meshPhongMaterial
          attach="material"
          color="#ccc"
          side={THREE.DoubleSide}
          receiveShadow
        />
      </mesh>

      <planeHelper args={[floorPlane, 5, "red"]} />

      <gridHelper args={[100, 100]} />

      <Obj setIsDragging={setIsDragging} floorPlane={floorPlane} />

      <OrthographicCamera makeDefault zoom={50} position={[0, 40, 200]} />

      <OrbitControls minZoom={10} maxZoom={50} enabled={!isDragging} />
    </Canvas>
  );
}

Obj.jsx

import React, { useState, useRef } from "react";
import { useDrag } from "@use-gesture/react";
import { animated, useSpring } from "@react-spring/three";
import { useThree } from "@react-three/fiber";
import * as THREE from "three";

function Obj({ setIsDragging, floorPlane }) {
  const [pos, setPos] = useState([0, 1, 0]);
  const { size, viewport } = useThree();
  const aspect = size.width / viewport.width;

  let planeIntersectPoint = new THREE.Vector3();

  const dragObjectRef = useRef();

  const [spring, api] = useSpring(() => ({
    // position: [0, 0, 0],
    position: pos,
    scale: 1,
    rotation: [0, 0, 0],
    config: { friction: 10 }
  }));

  const bind = useDrag(
    ({ active, movement: [x, y], timeStamp, event }) => {
      if (active) {
        event.ray.intersectPlane(floorPlane, planeIntersectPoint);
        setPos([planeIntersectPoint.x, 1.5, planeIntersectPoint.z]);
      }

      setIsDragging(active);

      api.start({
        // position: active ? [x / aspect, -y / aspect, 0] : [0, 0, 0],
        position: pos,
        scale: active ? 1.2 : 1,
        rotation: [y / aspect, x / aspect, 0]
      });
      return timeStamp;
    },
    { delay: true }
  );

  return (
    <animated.mesh {...spring} {...bind()} castShadow>
      <dodecahedronBufferGeometry
        ref={dragObjectRef}
        attach="geometry"
        args={[1.4, 0]}
      />
      <meshNormalMaterial attach="material" />
    </animated.mesh>
  );
}

export default Obj;

That should not have taken me as long as it did, I hope that this is helpful for someone else!