Is there a way to play audio and video separate from each other in the same video

Question

<video id="video1" width="320" height="176" controls="controls">
  <source src="mov_bbb.mp4" type="video/mp4">
  
  <source src="mov_bbb.m4a" type="video/m4a">
  
</video>

I have a silent video and want to add his audio file in the video playback

Answer 1

The MediaSource API allows us to do exactly this.

Note though that for this to work you need to prepare your media assets in a way it's compatible with that API. Have a thorough read of this MDN article which explains pretty well how to do so.

Once you have your assets, the remaining is quite straight-forward:

(async() => {

  const fetching = Promise.all( [
    // the video "only" file
    fetchData( "https://dl.dropboxusercontent.com/s/u9ycdfwy8fig4dl/bbb_video.mp4" ),
    // the audio "only" file
    fetchData( "https://dl.dropboxusercontent.com/s/rj4dh32vxwi1iv5/bbb_audio.mp4" )
  ] );

  const video_mime = "video/mp4; codecs=avc1.64000c";
  const audio_mime = "audio/mp4; codecs=mp4a.40.2";
  if(
    !MediaSource.isTypeSupported( video_mime ) ||
    !MediaSource.isTypeSupported( audio_mime )
  ) {
    throw "unsupported codecs";
  }
  
  const source = new MediaSource();
  document.querySelector( "video" ).src = URL.createObjectURL( source );
  await waitForEvent( source, "sourceopen" );

  const video_buffer = source.addSourceBuffer( video_mime );
  const audio_buffer = source.addSourceBuffer( audio_mime );
  video_buffer.mode = audio_buffer.mode = "sequence";

  const [ video_data, audio_data ] = await fetching;
  // There is a 'variable' limit as to how much
  // data we can append in on go, 10MB seems quite safe
  const chunk_size = 10 * 1024 * 1024;
  let i = 0;
  while (
    i < video_data.length &&
    i < audio_data.length
  ) {
    const next_i = i + chunk_size;
    const events = Promise.all( [
      waitForEvent( video_buffer, "updateend" ),
      waitForEvent( audio_buffer, "updateend" )
    ] );
    video_buffer.appendBuffer( video_data.subarray( i, next_i ) );
    audio_buffer.appendBuffer( audio_data.subarray( i, next_i ) );
    await events;
    i = next_i;
  }
  
  source.endOfStream();

})().catch( console.error );

function fetchData( url ) {
  return fetch( url )
    .then( (resp) => resp.ok && resp.arrayBuffer() )
    // we return an Uint8 view to be able to do a zero-cost subarray()
    .then( (buf) => new Uint8Array( buf ) );
}
function waitForEvent( target, event_name ) {
  return new Promise( (res) => {
    target.addEventListener( event_name, res, { once: true } );
  } );
}

<video controls></video>

Answer 2

This works for me:

<meta charset="utf-8">
<video controls id="v" height="480" src="file.webm"></video>
<audio id="a" src="file.weba"></audio>

<script>
let v = document.getElementById('v');
let a = document.getElementById('a');
v.onpause = () => a.pause();
v.onplay = () => a.play();
v.onseeked = () => a.currentTime = v.currentTime;
</script>

Controls are on the video element, and audio element will respond to video play, pause and seek. The volume control doesn't work in this example, so you can either add that code, or just use the system volume control.