How to correctly obtain screen coordinates of 3D shape after rotation

Question

I need to be able to select a number of shapes in my 3d model by drawing a rectangular region, and all shapes that lie in that region are selected.

I can draw the region and select the nodes for if there is only an x or y rotation. But most combinations of x and y give the incorrect result.

I thought it would be a simple matter of getting the mouse and node position in screen coordinates and comparing them, but that isn't working as expected.

In the application below you can draw a region using the right mouse button (you have to click on a sphere to start, i'm not sure why, they mouse events on the subscene are only triggered if you you click on a sphere?). Another right click (again on a sphere) clears the selection.

You can left-click drag to rotate the model (again you have to start on a sphere). After a rotation, of any amount, about the x-axis you can successfully select a region. Likewise a rotation about the y-axis. However a combination of x and y rotation gives the wrong result. For e.g. diagonally drag a node and you get a result as shown below.

Result of selection after x and y rotation

Any thoughts on what is going wrong? or suggestions for other ways to approach this? Thanks in advance

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;

import javafx.application.Application;
import javafx.geometry.Point2D;
import javafx.scene.DepthTest;
import javafx.scene.Group;
import javafx.scene.PerspectiveCamera;
import javafx.scene.Scene;
import javafx.scene.SceneAntialiasing;
import javafx.scene.SubScene;
import javafx.scene.control.Button;
import javafx.scene.control.Label;
import javafx.scene.control.Menu;
import javafx.scene.control.MenuBar;
import javafx.scene.control.Slider;
import javafx.scene.input.MouseButton;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.BorderPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Pane;
import javafx.scene.paint.Color;
import javafx.scene.paint.Material;
import javafx.scene.paint.PhongMaterial;
import javafx.scene.shape.Rectangle;
import javafx.scene.shape.Shape3D;
import javafx.scene.shape.Sphere;
import javafx.scene.transform.Rotate;
import javafx.scene.transform.Transform;
import javafx.scene.transform.Translate;
import javafx.stage.Stage;

public class ScreenSelection extends Application {


    private final PerspectiveCamera camera = new PerspectiveCamera(true);

    private final Group root = new Group();
    private final Group world = new Group();
    private final XFormWorld camPiv = new XFormWorld();

    private final Slider zoom = new Slider(-100, 0, -50);
    private final Button reset = new Button("Reset");

    private final Pane pane = new Pane();
    private final BorderPane main = new BorderPane();

    double mousePosX, mousePosY, mouseOldX, mouseOldY, mouseDeltaX, mouseDeltaY;
    double mouseFactorX, mouseFactorY;


public void start(Stage stage) throws Exception {

    camera.setTranslateZ(zoom.getValue());
    reset.setOnAction(eh -> {
        camPiv.reset();
        zoom.setValue(-50);
    });
    camera.setFieldOfView(60);

    camPiv.getChildren().add(camera);
    Collection<Shape3D> world = createWorld();
    RectangleSelect rs = new RectangleSelect(main, world);

    this.world.getChildren().addAll(world);
    root.getChildren().addAll(camPiv, this.world);

    SubScene subScene = new SubScene(root, -1, -1, true, SceneAntialiasing.BALANCED);
    subScene.setDepthTest(DepthTest.ENABLE);
    subScene.setCamera(camera);

    subScene.heightProperty().bind(pane.heightProperty());
    subScene.widthProperty().bind(pane.widthProperty());

    zoom.valueProperty().addListener((o, oldA, newA) -> camera.setTranslateZ(newA.doubleValue()));


    HBox controls = new HBox();
    controls.getChildren().addAll(new HBox(new Label("Zoom: "), zoom), new HBox(reset));

    pane.getChildren().addAll(controls, subScene);

    MenuBar menu = new MenuBar(new Menu("File"));
    main.setTop(menu);

    main.setCenter(pane);

    Scene scene = new Scene(main);

    subScene.setOnMousePressed((MouseEvent me) -> {
        mousePosX = me.getSceneX();
        mousePosY = me.getSceneY();
    });

    subScene.setOnMouseDragged((MouseEvent me) -> {
        if (me.isSecondaryButtonDown()) {
            rs.onMouseDragged(me);
        } else if (me.isPrimaryButtonDown()) {
            mouseOldX = mousePosX;
            mouseOldY = mousePosY;
            mousePosX = me.getSceneX();
            mousePosY = me.getSceneY();
            mouseDeltaX = (mousePosX - mouseOldX);
            mouseDeltaY = (mousePosY - mouseOldY);
            camPiv.ry(mouseDeltaX * 180.0 / subScene.getWidth());
            camPiv.rx(-mouseDeltaY * 180.0 / subScene.getHeight());


        }
    });
    subScene.setOnMouseReleased((MouseEvent me) -> {
        rs.omMouseDragReleased(me);
    });
    subScene.setOnMouseClicked((MouseEvent me) -> {
        if (me.getButton() == MouseButton.SECONDARY) {
            rs.clearSelection();
        }
    });
    stage.setScene(scene);
    stage.setWidth(800);
    stage.setHeight(800);
    stage.show();

}

private Collection<Shape3D> createWorld() {

    List<Shape3D> shapes = new ArrayList<Shape3D>();

    Random random = new Random(System.currentTimeMillis());
    for (int i=0; i<4000; i++) {
        double x = (random.nextDouble() - 0.5) * 30;
        double y = (random.nextDouble() - 0.5) * 30 ;
        double z = (random.nextDouble() - 0.5) * 30 ;

        Sphere point = new Sphere(0.2);
        point.setMaterial(new PhongMaterial(Color.SKYBLUE));
        point.setPickOnBounds(false);
        point.getTransforms().add(new Translate(x, y, z));
        shapes.add(point);
    }

    return shapes;
}


public static void main(String[] args) {
    launch(args);
}

public class XFormWorld extends Group {
    Transform rotation = new Rotate();
    Translate translate = new Translate();

    public XFormWorld() {
        getTransforms().addAll(rotation, translate);
    }

    public void reset() {
        rotation = new Rotate();
        getTransforms().set(0, rotation);

    }

    public void rx(double angle) {
        rotation = rotation.createConcatenation(new Rotate(angle, Rotate.X_AXIS));
        getTransforms().set(0, rotation);
    }

    public void ry(double angle) {
        rotation = rotation.createConcatenation(new Rotate(angle, Rotate.Y_AXIS));
        getTransforms().set(0, rotation); 
    }

    public void tx(double amount) {
        translate.setX(translate.getX() + amount);
    }

}

public class RectangleSelect  {

    private static final int START_X = 0;
    private static final int START_Y = 1;
    private static final int END_X = 2;
    private static final int END_Y = 3;

    private double[] sceneCoords = new double[2]; //mouse drag x, y in scene coords 
    private double[] screenCoords = new double[2]; //mouse drag current x, y in screen coords
    private double[] boundsInScreenCoords = new double[4]; //top left x, y, bottom right x,y in screen coords
    private Collection<Shape3D> world;

    private PhongMaterial selected = new PhongMaterial(Color.YELLOW);
    private Rectangle rectangle;

    public RectangleSelect(Pane pane, Collection<Shape3D> world) {
        sceneCoords[START_X] = Double.MIN_VALUE;
        sceneCoords[START_Y] = Double.MIN_VALUE;
        rectangle = new Rectangle();
        rectangle.setStroke(Color.RED);
        rectangle.setOpacity(0.0);
        rectangle.setMouseTransparent(true);
        rectangle.setFill(null);

        this.world = world;
        pane.getChildren().add(rectangle);
    }


    public void onMouseDragged(MouseEvent me) {
        clearSelection();
        if (sceneCoords[START_X] == Double.MIN_VALUE) {
            sceneCoords[START_X] = me.getSceneX();
            sceneCoords[START_Y] = me.getSceneY();
            screenCoords[START_X] = me.getScreenX();
            screenCoords[START_Y] = me.getScreenY();
        }
        double sceneX = me.getSceneX();
        double sceneY = me.getSceneY();
        double screenX = me.getScreenX();
        double screenY = me.getScreenY();

        double topX = Math.min(sceneCoords[START_X], sceneX);
        double bottomX = Math.max(sceneCoords[START_X], sceneX);
        double leftY = Math.min(sceneCoords[START_Y], sceneY);
        double rightY = Math.max(sceneCoords[START_Y], sceneY);

        boundsInScreenCoords[START_X] = Math.min(screenCoords[START_X], screenX);
        boundsInScreenCoords[END_X]= Math.max(screenCoords[START_X], screenX);
        boundsInScreenCoords[START_Y] = Math.min(screenCoords[START_Y], screenY);
        boundsInScreenCoords[END_Y] = Math.max(screenCoords[START_Y], screenY);

        world.forEach(this::selectIfInBounds);

        rectangle.setX(topX);
        rectangle.setY(leftY);
        rectangle.setWidth(bottomX - topX);
        rectangle.setHeight(rightY - leftY);
        rectangle.setOpacity(1.0);
    }


    private void selectIfInBounds(Shape3D node) {
        Point2D screenCoods = node.localToScreen(0.0, 0.0, 0.0);
        if (screenCoods.getX() > boundsInScreenCoords[START_X] &&
            screenCoods.getY() > boundsInScreenCoords[START_Y] &&
            screenCoods.getX() < boundsInScreenCoords[END_X] &&
            screenCoods.getY() < boundsInScreenCoords[END_Y]) {
            Material m = node.getMaterial();
            node.getProperties().put("material", m);
            node.setMaterial(selected);
        }
    }

    private void unselect(Shape3D node) {
        Material m = (Material) node.getProperties().get("material");
        if (m != null) {
            node.setMaterial(m);
        }
    }

    public void omMouseDragReleased(MouseEvent me) {
        rectangle.setOpacity(0.0);
        sceneCoords[START_X]  = Double.MIN_VALUE;
        sceneCoords[START_Y] = Double.MIN_VALUE;
    }

    public void clearSelection() {
        world.forEach(this::unselect);
    }
}   

}

Answer 1

Thanks user1803551 for those links, the first one was a good use case for tracking down the bug, and it does appear to be a bug in GeneralTransform3D.transform(Vec3d)

The implementation of GeneralTransform3D.transform(Vec3d) (which is called by Camera.project in the course of calculating the mouse position) calls the two arg transform method with the same point object. i.e.

public Vec3d transform(Vec3d point) {
    return transform(point, point);
}

By calling it with the same object, the calculations are borked. You can probably see that if pointOut and point are the same object then the calculation of pointOut.y will be incorrect (this is from GeneralTransform3D.transform)

    pointOut.x = (float) (mat[0] * point.x + mat[1] * point.y
            + mat[2] * point.z + mat[3]);
    pointOut.y = (float) (mat[4] * point.x + mat[5] * point.y
            + mat[6] * point.z + mat[7]); 

All well and good, not sure how to work around it though

Answer 2

Iv'e verified that this is the bug(s) listed in the comments. For your case, I believe the simplest solution would be to rotate the world instead of the camera. Since these are the only 2 objects that move relative to each other, it doesn't matter which one is moved. You can also make the zoom apply to the world instead of the camera if you want to unify the transforms, but this doesn't matter.

Rotating the world

Make the world rotatable by having it be a XFormWorld, and remove camPiv altogether. Note that there was no reason to add camPiv to the scene because it's an empty group; a camera is only added via setCamera and then you can bind its transforms (see below).

You would need to change the math in 2 ways:

1. Flip the rotation values of rx and ry because rotating the world in +x is like rotating the camera in -x (same for y).
2. Correct the rotation pivot. If you rotate on the x axis, and then on the y axis, the y axis rotation would actually rotate it around z (because of rotation matrix rules). That means that the pivot for the new rotation depends on the current rotation. If you rotated on x, you would now need to rotate on z to get a y rotation. The math is easy, but you need to know what you're doing.

Transforming the camera directly

The reason you don't need a camPiv even when you transform the camera is because you can bind directly to its transforms. In your case, you could do

camera.translateZProperty().bind(zoom.valueProperty());

instead of the annoying combination

camera.setTranslateZ(zoom.getValue());
zoom.valueProperty().addListener((o, oldA, newA) -> camera.setTranslateZ(newA.doubleValue()));

And for any Transform, add it to the camera.getTransforms() and bind its values (angle, translation...) to DoublePropertys whose value is the one you change via input.

Mouse events and Picking on bounds

Your subScene (and world) contain many nodes with empty spaces between them. By default, when you click on the subScene, the event will be delivered to it only if you click on a (non-mouse transparent) node inside it. This is because pickOnBounds is false, which means that the click "passes through" until it hits something. If you add

subScene.setPickOnBounds(true);

the container (subScene) will receive any events within its box bounds, regardless of whether there is a node there or not.

After that is fixed, you will encounter a new problem: releasing the mouse after drawing the rectangle will cause it to disappear via clearSelection(). This is because you call that method in onMouseClicked, but a click event is generated at the end of the drag because there was a press and the a release. What you want is to clear the selection if it's a click without a drag. This is done with isStillSincePress():

subScene.setOnMouseClicked(me -> {
    if (me.getButton() == MouseButton.SECONDARY && me.isStillSincePress()) {
        rs.clearSelection();
    }
});

The reason you didn't encounter this is because the subScene didn't receive the release event if it happened on an empty space. To summarize:

• Pressed on an empty space: event not registered - nothing happened.
• Pressed on a sphere: event registered - triangle drawing started.
  • Release on an empty space: event not registered - rectangle was not cleared.
  • Release on a sphere: event registered - rectangle was cleared.

Layout

Don't use Pane unless you need absolute positioning (and you rarely do). Pick a subclass that does the job better. A StackPane allows you to put controls on top of a SubScene via use of layers. Setting setPickOnBounds to false allows the lower layers to receive events normally. Additionally, I used an AnchorPane to have the controls placed on the top left.

The working solution

Here's your modified code. I did some refactoring while working on it so that it was easier for me to work with. The whole RectangleSelect could also be heavily modified I believe, but the question is loaded enough as it is.

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;

import javafx.application.Application;
import javafx.geometry.Point2D;
import javafx.geometry.Point3D;
import javafx.scene.Group;
import javafx.scene.PerspectiveCamera;
import javafx.scene.Scene;
import javafx.scene.SceneAntialiasing;
import javafx.scene.SubScene;
import javafx.scene.control.Button;
import javafx.scene.control.Label;
import javafx.scene.control.Menu;
import javafx.scene.control.MenuBar;
import javafx.scene.control.Slider;
import javafx.scene.input.MouseButton;
import javafx.scene.input.MouseEvent;
import javafx.scene.layout.AnchorPane;
import javafx.scene.layout.BorderPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Pane;
import javafx.scene.layout.StackPane;
import javafx.scene.paint.Color;
import javafx.scene.paint.Material;
import javafx.scene.paint.PhongMaterial;
import javafx.scene.shape.Rectangle;
import javafx.scene.shape.Shape3D;
import javafx.scene.shape.Sphere;
import javafx.scene.transform.Rotate;
import javafx.scene.transform.Transform;
import javafx.scene.transform.Translate;
import javafx.stage.Stage;

public class ScreenSelectionNew extends Application {

    private final PerspectiveCamera camera = new PerspectiveCamera(true);

    private final XFormWorld world = new XFormWorld();

    private double mousePosX, mousePosY, mouseOldX, mouseOldY;

    @Override
    public void start(Stage stage) throws Exception {
        BorderPane main = new BorderPane();
        StackPane stackPane = new StackPane();

        SubScene subScene = setupSubScene(main);
        subScene.heightProperty().bind(stackPane.heightProperty());
        subScene.widthProperty().bind(stackPane.widthProperty());
        stackPane.getChildren().addAll(subScene, setupControls());

        MenuBar menu = new MenuBar(new Menu("File"));

        main.setTop(menu);
        main.setCenter(stackPane);
        Scene scene = new Scene(main);

        stage.setScene(scene);
        stage.setWidth(800);
        stage.setHeight(800);
        stage.show();
    }

    private SubScene setupSubScene(Pane parent) {
        Collection<Shape3D> worldContent = createWorld();
        world.getChildren().addAll(worldContent);

        SubScene subScene = new SubScene(world, -1, -1, true, SceneAntialiasing.BALANCED);
        subScene.setCamera(camera);
        subScene.setPickOnBounds(true);
        camera.setFieldOfView(60);

        RectangleSelect rs = new RectangleSelect(parent, worldContent);

        subScene.setOnMousePressed(me -> {
            mousePosX = me.getX();
            mousePosY = me.getY();
        });

        subScene.setOnMouseDragged(me -> {
            if (me.isSecondaryButtonDown()) {
                rs.onMouseDragged(me);
            } else if (me.isPrimaryButtonDown()) {
                mouseOldX = mousePosX;
                mouseOldY = mousePosY;
                mousePosX = me.getX();
                mousePosY = me.getY();
                double mouseDeltaX = (mousePosX - mouseOldX);
                double mouseDeltaY = (mousePosY - mouseOldY);
                world.rx(mouseDeltaY * 180.0 / subScene.getHeight());
                world.ry(-mouseDeltaX * 180.0 / subScene.getWidth());
            }
        });

        subScene.setOnMouseReleased(me -> rs.onMouseDragReleased(me));

        subScene.setOnMouseClicked(me -> {
            if (me.getButton() == MouseButton.SECONDARY && me.isStillSincePress()) {
                rs.clearSelection();
            }
        });

        return subScene;
    }

    private Pane setupControls() {
        Slider zoom = new Slider(-100, 0, -50);
        camera.translateZProperty().bind(zoom.valueProperty());

        Button reset = new Button("Reset");
        reset.setOnAction(eh -> {
            world.reset();
            zoom.setValue(-50);
        });

        HBox controls = new HBox(new Label("Zoom: "), zoom, reset);
        AnchorPane anchorPane = new AnchorPane(controls);
        anchorPane.setPickOnBounds(false);
        return anchorPane;
    }

    private Collection<Shape3D> createWorld() {

        List<Shape3D> shapes = new ArrayList<>();

        Random random = new Random(System.currentTimeMillis());
        for (int i = 0; i < 4000; i++) {
            double x = (random.nextDouble() - 0.5) * 30;
            double y = (random.nextDouble() - 0.5) * 30;
            double z = (random.nextDouble() - 0.5) * 30;

            Sphere point = new Sphere(0.2);
            point.setMaterial(new PhongMaterial(Color.SKYBLUE));
            point.getTransforms().add(new Translate(x, y, z));
            shapes.add(point);
        }

        return shapes;
    }

    public static void main(String[] args) {
        launch(args);
    }

    public class XFormWorld extends Group {
        Transform rotation = new Rotate();

        public XFormWorld() {
            getTransforms().addAll(rotation);
        }

        public void reset() {
            rotation = new Rotate();
            getTransforms().set(0, rotation);
        }

        public void rx(double angle) {
            Point3D axis = new Point3D(rotation.getMxx(), rotation.getMxy(), rotation.getMxz());
            rotation = rotation.createConcatenation(new Rotate(angle, axis));
            getTransforms().set(0, rotation);
        }

        public void ry(double angle) {
            Point3D axis = new Point3D(rotation.getMyx(), rotation.getMyy(), rotation.getMyz());
            rotation = rotation.createConcatenation(new Rotate(angle, axis));
            getTransforms().set(0, rotation);
        }
    }

    public class RectangleSelect {

        private static final int START_X = 0;
        private static final int START_Y = 1;
        private static final int END_X = 2;
        private static final int END_Y = 3;

        private double[] sceneCoords = new double[2]; //mouse drag x, y in scene coords 
        private double[] screenCoords = new double[2]; //mouse drag current x, y in screen coords
        private double[] boundsInScreenCoords = new double[4]; //top left x, y, bottom right x,y in screen coords
        private Collection<Shape3D> world;

        private PhongMaterial selected = new PhongMaterial(Color.YELLOW);
        private Rectangle rectangle;

        public RectangleSelect(Pane pane, Collection<Shape3D> world) {
            sceneCoords[START_X] = Double.MIN_VALUE;
            sceneCoords[START_Y] = Double.MIN_VALUE;
            rectangle = new Rectangle();
            rectangle.setStroke(Color.RED);
            rectangle.setOpacity(0.0);
            rectangle.setMouseTransparent(true);
            rectangle.setFill(null);

            this.world = world;
            pane.getChildren().add(rectangle);
        }

        public void onMouseDragged(MouseEvent me) {
            clearSelection();
            if (sceneCoords[START_X] == Double.MIN_VALUE) {
                sceneCoords[START_X] = me.getSceneX();
                sceneCoords[START_Y] = me.getSceneY();
                screenCoords[START_X] = me.getScreenX();
                screenCoords[START_Y] = me.getScreenY();
            }
            double sceneX = me.getSceneX();
            double sceneY = me.getSceneY();
            double screenX = me.getScreenX();
            double screenY = me.getScreenY();

            double topX = Math.min(sceneCoords[START_X], sceneX);
            double bottomX = Math.max(sceneCoords[START_X], sceneX);
            double leftY = Math.min(sceneCoords[START_Y], sceneY);
            double rightY = Math.max(sceneCoords[START_Y], sceneY);

            boundsInScreenCoords[START_X] = Math.min(screenCoords[START_X], screenX);
            boundsInScreenCoords[END_X] = Math.max(screenCoords[START_X], screenX);
            boundsInScreenCoords[START_Y] = Math.min(screenCoords[START_Y], screenY);
            boundsInScreenCoords[END_Y] = Math.max(screenCoords[START_Y], screenY);

            world.forEach(this::selectIfInBounds);

            rectangle.setX(topX);
            rectangle.setY(leftY);
            rectangle.setWidth(bottomX - topX);
            rectangle.setHeight(rightY - leftY);
            rectangle.setOpacity(1.0);
        }

        private void selectIfInBounds(Shape3D node) {
            Point2D screenCoods = node.localToScreen(0.0, 0.0, 0.0);
            if (screenCoods.getX() > boundsInScreenCoords[START_X] &&
                screenCoods.getY() > boundsInScreenCoords[START_Y] &&
                screenCoods.getX() < boundsInScreenCoords[END_X] &&
                screenCoods.getY() < boundsInScreenCoords[END_Y]) {
                Material m = node.getMaterial();
                node.getProperties().put("material", m);
                node.setMaterial(selected);
            }
        }

        private void unselect(Shape3D node) {
            Material m = (Material) node.getProperties().get("material");
            if (m != null) {
                node.setMaterial(m);
            }
        }

        public void onMouseDragReleased(MouseEvent me) {
            rectangle.setOpacity(0.0);
            sceneCoords[START_X] = Double.MIN_VALUE;
            sceneCoords[START_Y] = Double.MIN_VALUE;
        }

        public void clearSelection() {
            world.forEach(this::unselect);
        }
    }
}