ThreeJS rotate around axis

Question

I want to rotate this cube around the light blue axis. I works If I change the rotation around THREE.Vector3(0,0,0) instead of THREE.Vector3(0.4,0,0.9)

I don't know why the cubes shape changes and why it gets smaller with more iterations

An fiddle showing this problem (please ignore the crappy implementation. I just changed a old one)

So this is how I do the rotation:

function rotate(deg) {
  _initTranslation = new THREE.Vector3();
  _initRotation = new THREE.Quaternion();
  _initScale = new THREE.Vector3();
  rotateMatrix = new THREE.Matrix4();

  cube.matrix.decompose(_initTranslation, _initRotation, _initScale);

  cube.matrix = rotateMatrix.compose(_initTranslation, new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0.4,1,0.9), THREE.Math.degToRad(deg)), _initScale);

  cube.matrixAutoUpdate = false;
  cube.matrixWorldNeedsUpdate = true;

}

Maybe someone knows what I did wrong.

var renderer, scene, camera, controls;
var geometry, material, line, vertices, last, _initTranslation, _initRotation, initScale, rotateMatrix;

var deg = 0;

init();
animate();

function init() {

  document.body.style.cssText = 'margin: 0; overflow: hidden;' ;

  renderer = new THREE.WebGLRenderer( { alpha: 1, antialias: true, clearColor: 0xffffff }  );
  renderer.setSize( window.innerWidth, window.innerHeight );
  document.body.appendChild( renderer.domElement );
  scene = new THREE.Scene();

  camera = new THREE.PerspectiveCamera( 40, window.innerWidth / window.innerHeight, 1, 1000 );
  camera.position.set( 5, 5, 5 );
  controls = new THREE.OrbitControls( camera, renderer.domElement );
  
  geometry2 = new THREE.BoxGeometry( .5, .5, .5 );
  material2 = new THREE.MeshNormalMaterial();
  cube = new THREE.Mesh( geometry2, material2 );
  scene.add( cube );       
  

  material = new THREE.LineBasicMaterial({ color: 0x0077ff }); 
  geometry = new THREE.Geometry();
  geometry.vertices.push( new THREE.Vector3( 0, 0, 0) );
  line = new THREE.Line( geometry, material ) 
  scene.add( line );
  
  var sphereAxis = new THREE.AxesHelper(20);
  scene.add(sphereAxis);

  addStep();
  
  cube.lookAt(new THREE.Vector3(0.4,0,0.9));

}

function addStep() {

  vertices = geometry.vertices;
  last = vertices[ vertices.length - 1 ];
  vertices.push( 

    new THREE.Vector3(0.4,0,0.9) 

  );

  geometry = new THREE.Geometry();
  geometry.vertices = vertices;

  scene.remove( line );
  line = new THREE.Line( geometry, material )
  scene.add( line );

}

function animate() {
    
  rotate(deg)
  
  deg += 5
  
  requestAnimationFrame( animate ); 
  renderer.render(scene, camera);
  controls.update();

}

function rotate(deg) {
  _initTranslation = new THREE.Vector3();
  _initRotation = new THREE.Quaternion();
  _initScale = new THREE.Vector3();
  rotateMatrix = new THREE.Matrix4();
    
  cube.matrix.decompose(_initTranslation, _initRotation, _initScale);
  
  cube.matrix = rotateMatrix.compose(_initTranslation, new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0.4,0,0.9), THREE.Math.degToRad(deg)), _initScale);

    
    cube.matrixAutoUpdate = false;
    cube.matrixWorldNeedsUpdate = true;
  
  
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/102/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>

Answer 1

The vector component of the Quaternion has to be (normalize.). The length of a normalized vector (Unit vector) is 1.0.

In your case the length of the vector component (THREE.Vector3(0.4, 0, 0.9)) is less than 1.0:

sqrt(0.9*0.9 + 0.0*0.0 + 0.4*0.4) = sqrt(0.81 + 0.16) = sqrt(0.97) = 0.9409

This causes that the cube scales sown by time. This can be verified by logging the scaling component (console.log(_initScale)).
If you would use a vector component with a length greater than 1.0 (e.g. THREE.Vector3(0.5, 0, 0.9), then the cube will scale up.

Normalize the axis of the Quaternion, to solve the issue:

let axis = new THREE.Vector3(0.4, 0, 0.9);
let q = new THREE.Quaternion().setFromAxisAngle(axis.normalize(), THREE.Math.degToRad(deg)); 
cube.matrix = rotateMatrix.compose(_initTranslation, q, _initScale);

If you want that one side of the cube is aligned to the axis, in that way, that the axis is normal to the side, then this is something completely different.
You've to do 2 rotations. First rotate the cube (e.g.) continuously around the x-axis, then turn the x-axis to the target axis (0.4, 0, 0.9). Use .setFromAxisAngle` to initialize a quaternion which rotates the x-axis to the target axis:

let x_axis = new THREE.Vector3(1, 0, 0);
let axis = new THREE.Vector3(0.4, 0, 0.9);
let q_align = new THREE.Quaternion().setFromUnitVectors(x_axis, axis.normalize());
let q_rotate = new THREE.Quaternion().setFromAxisAngle(x_axis, THREE.Math.degToRad(deg));
let q_final = q_align.clone().multiply(q_rotate);
cube.matrix = rotateMatrix.compose(_initTranslation, q, _initScale);

See the example, which compares the 2 different behavior:

var renderer, scene, camera, controls;
var geometry, material, line, vertices, last, _initTranslation, _initRotation, initScale, rotateMatrix;

var deg = 0;

init();
animate();

function init() {

    document.body.style.cssText = 'margin: 0; overflow: hidden;' ;

    renderer = new THREE.WebGLRenderer( { alpha: 1, antialias: true, clearColor: 0xffffff }  );
    renderer.setSize( window.innerWidth, window.innerHeight );
    document.body.appendChild( renderer.domElement );
    scene = new THREE.Scene();

    camera = new THREE.PerspectiveCamera( 40, window.innerWidth / window.innerHeight, 1, 1000 );
    camera.position.set( 1, 3, 3 );
    controls = new THREE.OrbitControls( camera, renderer.domElement );
    
    geometry2 = new THREE.BoxGeometry( .5, .5, .5 );
    material2 = new THREE.MeshNormalMaterial();
    
    let shift = 0.5
    cube = new THREE.Mesh( geometry2, material2 );
    cube.matrix.makeTranslation(shift, 0, 0);
    scene.add( cube );
    cube2 = new THREE.Mesh( geometry2, material2 );
    cube2.matrix.makeTranslation(-shift, 0, 0);
    scene.add( cube2 );       
    
    material = new THREE.LineBasicMaterial({ color: 0x0077ff }); 
    geometry = new THREE.Geometry();
    geometry.vertices.push( new THREE.Vector3(-0.4, 0, -0.9), new THREE.Vector3(0.4, 0, 0.9) );
    line = new THREE.Line( geometry, material ) 
    line.position.set(shift, 0, 0);
    scene.add( line );
    line2 = new THREE.Line( geometry, material ) 
    line2.position.set(-shift, 0, 0);
    scene.add( line2 );

    var sphereAxis = new THREE.AxesHelper(20);
    scene.add(sphereAxis);
    
    window.onresize = function() {
        renderer.setSize(window.innerWidth, window.innerHeight);
        camera.aspect = window.innerWidth / window.innerHeight;
        camera.updateProjectionMatrix();
    }
}

function animate() {  
    rotate(deg)
    deg += 5
    
    requestAnimationFrame( animate ); 
    renderer.render(scene, camera);
    controls.update();
}

function rotate(deg) {
    _initTranslation = new THREE.Vector3();
    _initRotation = new THREE.Quaternion();
    _initScale = new THREE.Vector3();
    
    let x_axis = new THREE.Vector3(1, 0, 0);
    let axis = new THREE.Vector3(0.4, 0, 0.9);

    // cube

    cube.matrix.decompose(_initTranslation, _initRotation, _initScale);
    
    let q_align = new THREE.Quaternion().setFromUnitVectors(x_axis, axis.normalize());
    let q_rotate = new THREE.Quaternion().setFromAxisAngle(x_axis, THREE.Math.degToRad(deg));
    let q_final = q_align.clone().multiply(q_rotate);
    
    cube.matrix.compose(_initTranslation, q_final, _initScale);
    cube.matrixAutoUpdate = false;
    cube.matrixWorldNeedsUpdate = true;

    // cube2

    cube2.matrix.decompose(_initTranslation, _initRotation, _initScale);
    
    q = new THREE.Quaternion().setFromAxisAngle(axis.normalize(), THREE.Math.degToRad(deg));
    
    cube2.matrix.compose(_initTranslation, q, _initScale);
    cube2.matrixAutoUpdate = false;
    cube2.matrixWorldNeedsUpdate = true;
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/102/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>