Extracting audio from a video file

Question

Edit: This post is no duplicate of mine. I am trying to extract the audio data as binary, got no problems with playing the audio file as separate as I mentioned before.

I am trying to extract audio from a video file on client-side by using Web Audio Api.

var audioContext = new(window.AudioContext || window.webkitAudioContext)();
fileData = new Blob([input.files[0]]);
var videoFileAsBuffer = new Promise(getBuffer);
videoFileAsBuffer.then(function (data) {
    audioContext.decodeAudioData(data).then(function (decodedAudioData) {
        mySoundBuffer = decodedAudioData;
        soundSource = audioContext.createBufferSource();
        soundSource.buffer = mySoundBuffer;
        // soundSource.connect(audioContext.destination);
        // soundSource.start();
    });

When I uncomment the two lines at the end, I am hearing the sound of the uploaded video file. Though, when I create a link to download the file with the help of getChannelData method, it's almost the same size as video file.

I was expecting decodedAudioData to have only audio binary data, and send that to my webservice, which is the only one I need. However, that didn't work out as I expected. Anyone knows a way to extract audio of a video file on client-side? Thanks in advance.

Here is the getBuffer method in case anyone wants to know:

function getBuffer(resolve) {
    var reader = new FileReader();
    reader.onload = function () {
        var arrayBuffer = reader.result;
        resolve(arrayBuffer);
    }
    reader.readAsArrayBuffer(fileData);
}

Edit: I was able to decode the video file and get audiobuffer by using OfflineAudioContext inside decodeAudioData function.

var offlineAudioContext = new OfflineAudioContext(2, 44100 * 100, 44100);
var soundSource = offlineAudioContext.createBufferSource();
...
soundSource.connect(offlineAudioContext.destination);
soundSource.start();
offlineAudioContext.startRendering().then(function (renderedBuffer) {
    console.log(renderedBuffer); // outputs audiobuffer
    var song = audioContext.createBufferSource();
    song.buffer = renderedBuffer;
    song.connect(audioContext.destination);
    song.start();
}).catch(function (err) {
    console.log('Rendering failed: ' + err);
});

renderedBuffer is an audiobuffer, had no problem outputting the data, tested with Audacity's import raw data option. But the problem is, the new file (filled with renderedBuffer.getChannelData(0)) has higher size than the original video has. Isn't that supposed to have lower size since it only contains audio of the video file?

Answer 1

Ok, I actually had the answer already. Raw audio data is huge, that's why it's even greater than the video itself in size.

var offlineAudioContext = new OfflineAudioContext(numberOfChannels, sampleRate * duration, sampleRate);
var soundSource = offlineAudioContext.createBufferSource();
...
reader.readAsArrayBuffer(blob); // video file
reader.onload = function () {
  var videoFileAsBuffer = reader.result; // arraybuffer
  audioContext.decodeAudioData(videoFileAsBuffer).then(function (decodedAudioData) {
    myBuffer = decodedAudioData;
    soundSource.buffer = myBuffer;
    soundSource.connect(offlineAudioContext.destination);
    soundSource.start();

    offlineAudioContext.startRendering().then(function (renderedBuffer) {
      console.log(renderedBuffer); // outputs audiobuffer
    }).catch(function (err) {
      console.log('Rendering failed: ' + err);
    });
  });
};

After that, I was able to convert the audiobuffer (renderedbuffer) into a wav file by using audiobuffer-to-wav library. OfflineAudioContext is just needed to modify the cropped audio.

Edit: Here is the js fiddle example. decodedAudioData method will suffice if you don't want to override the audio data.

Answer 2

I'm using this code to extract audio:

// initialize the audioContext
var audioContext = new webkitAudioContext();
var video = document.getElementById("myVideo");
var mediaSource = audioContext.createMediaElementSource(video);
var analyser = audioContext.createAnalyser();
mediaSource.connect(analyser);
analyser.connect(audioContext.destination);

---

// this will give you the sound data

video.play();

function getSoundData() {
   var sample = new Float32Array(analyser.frequencyBinCount);
   return analyser.getFloatFrequencyData(sample);  
}

EDIT:

I've not run this code so far but I think this should work also:

Fetching an XMLHttpRequest to your videourl (working also with local files) instead of using WebAudio API.

var xhr = new XMLHttpRequest();
xhr.open('GET', 'https://test.com/myvideo.mp4', true);
xhr.responseType = 'blob';

xhr.onload = function(e) {
    var binaryData = this.response;
    console.log(binaryData)
};

xhr.send();

EDIT 2:

var audioFileUrl = 'video.mp3';

fetch(audioFileUrl)
  .then(function(res) {
    res.blob().then(function(blob) {
      var size = blob.size;
      var type = blob.type;

      var reader = new FileReader();
      reader.addEventListener("loadend", function() {

        var base64FileData = reader.result.toString();

        var mediaFile = {
          fileUrl: audioFileUrl,
          size: blob.size,
          type: blob.type,
          src: base64FileData
        };

      });

      reader.readAsDataURL(blob);

    });
  });

or try using some kind of code like this:

<audio id="audio" src="http://yourvideo.com/video.mp4" controls autoplay></audio>


function getAudio() {
  var elem = document.getElementById("audio")

  // get Base 64 string from elem.src
}

I hope it is helpful because for me these few lines are working quite fine so far :)

Answer 3

I think you should change your code from

var soundSource = offlineAudioContext.createBufferSource();
......
soundSource.connect(offlineAudioContext.destination);

to these

 const soundSource = audioContext.createBufferSource()
 soundSource.buffer = decodedAudioData
 soundSource.connect(audioContext.destination)

in order to play the audio