Rendering an iframe onto an AR marker using CSS3DRenderer and jsartoolkit

Question

I would like to be able to overlay a html iframe on top of an augmented reality marker, however I cannot get the CSS3DRenderer to show the same result as the WebGLRenderer and I'm not sure where I'm going wrong.

The WebGL renders perfectly, with the mesh following the marker and it's all magic, the CSS3DRenderer however centers the iframe in the middle of the video, inversed and unscaled, and it rotates in the opposite direction.

Thanks to three.js and artoolkit, here is some test code using video input and both renderers.

new Promise(function(resolve,reject) {
    var source = document.createElement('video');

    source.autoplay = true;
    source.playsinline = true;
    source.controls = false;
    source.loop = true;
    source.onplay = function(event) {
        resolve(source);
    }
    source.src = 'data/output_4.ogg';

    document.body.appendChild(source);
}).then(function(source) {
    var scene = new THREE.Scene();

    var camera = new THREE.Camera();
    camera.matrixAutoUpdate = false;

    scene.add(camera);

    var material = new THREE.MeshNormalMaterial({
        transparent : true,
        opacity : 0.5,
        side : THREE.DoubleSide
    });

    var geometry = new THREE.PlaneGeometry(1,1);
    var mesh = new THREE.Mesh(geometry,material);
    // mesh.matrixAutoUpdate = false;

    scene.add(mesh);

    var renderer = new THREE.WebGLRenderer({
        antialias : true,
        alpha : true
    });

    renderer.setSize(source.videoWidth,source.videoHeight);
    renderer.setClearColor(new THREE.Color('lightgrey'),0);

    document.body.appendChild(renderer.domElement);

    /*\

    cssRenderer

    \*/

    var cssRenderer = new THREE.CSS3DRenderer();
    cssRenderer.setSize(source.videoWidth,source.videoHeight);

    var cssScene = new THREE.Scene();

    var iframe = document.createElement("iframe");
    iframe.src = "/data/index.html";
    iframe.style.background = "rgb(0,0,0)";

    var iframe3D = new THREE.CSS3DObject(iframe);
    // iframe3D.matrixAutoUpdate = false;

    cssScene.add(iframe3D);

    document.body.appendChild(cssRenderer.domElement);

    /*\

    arController

    \*/

    var cameraParameters = new ARCameraParam();

    var arController = null;

    cameraParameters.onload = function() {
        arController = new ARController(source.videoWidth,source.videoHeight,cameraParameters);

        arController.addEventListener("getMarker",function(event) {
            var modelViewMatrix = new THREE.Matrix4().fromArray(event.data.matrix);

            camera.matrix.getInverse(modelViewMatrix);
            // mesh.matrix.copy(modelViewMatrix);
            // iframe3D.matrix.copy(modelViewMatrix);
        });

        var cameraViewMatrix = new THREE.Matrix4().fromArray(arController.getCameraMatrix());

        camera.projectionMatrix.copy(cameraViewMatrix);
    }

    cameraParameters.load("data/camera_para.dat");

    /*\

    animate

    \*/

    requestAnimationFrame(function animate() {
        requestAnimationFrame(animate);

        if (!arController) {
            return;
        }

        arController.process(source);

        renderer.render(scene,camera);
        cssRenderer.render(cssScene,camera);
    });
});

I had hoped that rotating the camera instead of the object would provide a solution, alas. It's as if I was missing out some matrix transformation that needs to be applied.

score 1 · Answer 1 · answered Oct 09 '17 at 18:34

Perhaps there was something with the fov parameters of the camera, or maybe the transform matrix was getting overwritten, but anyway I couldn't locate the problem. So here's an alternative that doesn't use CSS3DRenderer, or THREE.js.

Essentially we can use the coordinates from the marker data itself to create a projection matrix. Many, many thanks to this post which provided most of the code I needed.

function adjugate(m) { // Compute the adjugate of m
    return [
        m[4]*m[8]-m[7]*m[5],m[7]*m[2]-m[1]*m[8],m[1]*m[5]-m[4]*m[2],
        m[6]*m[5]-m[3]*m[8],m[0]*m[8]-m[6]*m[2],m[3]*m[2]-m[0]*m[5],
        m[3]*m[7]-m[6]*m[4],m[6]*m[1]-m[0]*m[7],m[0]*m[4]-m[3]*m[1]
    ];
}

function multiply(a,b) { // multiply two matrices
    a = [
        a[0],a[3],a[6],
        a[1],a[4],a[7],
        a[2],a[5],a[8]
    ];

    b = [
        b[0],b[3],b[6],
        b[1],b[4],b[7],
        b[2],b[5],b[8]
    ];

    var m = Array(9);

    for (var i = 0; i != 3; ++i) {
        for (var j = 0; j != 3; ++j) {
            var mij = 0;
            for (var k = 0; k != 3; ++k) {
                mij += a[3*i + k]*b[3*k + j];
            }
            m[3*i + j] = mij;
        }
    }

    return [
        m[0],m[3],m[6],
        m[1],m[4],m[7],
        m[2],m[5],m[8]
    ];
}

function apply(m,v) { // multiply matrix and vector
    return [
        m[0]*v[0] + m[3]*v[1] + m[6]*v[2],
        m[1]*v[0] + m[4]*v[1] + m[7]*v[2],
        m[2]*v[0] + m[5]*v[1] + m[8]*v[2]
    ];
}

//

var iframe = document.createElement("iframe");
iframe.src = "data/index.html";
iframe.style.position = "absolute";
iframe.style.left = "0";
iframe.style.top = "0";
iframe.style.transformOrigin = "0 0";

document.querySelector("main").appendChild(iframe);

var s = [
    0,0,1,
    iframe.offsetWidth,0,1,
    0,iframe.offsetHeight,1
];

var v = apply(adjugate(s),[iframe.offsetWidth,iframe.offsetHeight,1]);

s = multiply(s,[
    v[0], 0, 0,
    0, v[1], 0,
    0, 0, v[2]
]);

arController.addEventListener("getMarker",function(event) {
    if (event.data.marker.id === marker) {
        var d = [
            event.data.marker.vertex[(4 - event.data.marker.dir) % 4][0],event.data.marker.vertex[(4 - event.data.marker.dir) % 4][1],1,
            event.data.marker.vertex[(5 - event.data.marker.dir) % 4][0],event.data.marker.vertex[(5 - event.data.marker.dir) % 4][1],1,
            event.data.marker.vertex[(7 - event.data.marker.dir) % 4][0],event.data.marker.vertex[(7 - event.data.marker.dir) % 4][1],1
        ];

        var v = apply(adjugate(d),[event.data.marker.vertex[(6 - event.data.marker.dir) % 4][0],event.data.marker.vertex[(6 - event.data.marker.dir) % 4][1],1]);

        d = multiply(d,[
            v[0],0,0,
            0,v[1],0,
            0,0,v[2]
        ]);

        var t = multiply(d,adjugate(s));

        for (i = 0; i < 9; ++i) {
            t[i] = t[i] / t[8];
            t[i] = Math.abs(t[i]) < Number.EPSILON ? 0 : t[i];
        }

        t = [
            t[0],t[1],0,t[2],
            t[3],t[4],0,t[5],
            0,0,1,0,
            t[6],t[7],0,t[8]
        ];

        iframe.style.transform = "matrix3d(" + t.join(", ") + ")";

    } else {
        // mesh.visible = false;
    }
});

Jonathan Neels · Answer 2 · 2022-08-14T09:09:06.367

1

Might be an answer 4 years overdue, but it might be useful for somebody.

I've created an alternative solution. You can check it here: https://github.com/jonathanneels/QrA

In essence; QrA uses tilt-js to simulate 3D-effects (parallax) and AR.js a-box as the startobject. The iframe gets pushed by the tilt.js functions and the AR cube's .position, .scale and .rotation is used as reference.

Enjoy!

edited Aug 14 '22 at 09:09

answered Aug 14 '22 at 09:07

Jonathan Neels

11
2

Rendering an iframe onto an AR marker using CSS3DRenderer and jsartoolkit

2 Answers2