1

server side extracting frame from video using ffmpeg library. Here is code:

    AVFormatContext *pFormatContext = avformat_alloc_context();
    avformat_open_input(&pFormatContext, "VID_20230219_171331_300.mp4.mov", NULL, NULL);
    avformat_find_stream_info(pFormatContext,  NULL);

    const AVCodec *pCodec = NULL;
    AVCodecParameters *pCodecParameters =  NULL;
    int video_stream_index = -1;

    // loop though all the streams and print its main information
    for (int i = 0; i < pFormatContext->nb_streams; i++)
    {
        AVCodecParameters *pLocalCodecParameters =  NULL;
        pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
        const AVCodec *pLocalCodec = NULL;
        pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);

        if(pLocalCodec==NULL)
            continue;

        // when the stream is a video we store its index, codec parameters and codec
        if(pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO)
        {
            if(video_stream_index == -1)
            {
                video_stream_index = i;
                pCodec = pLocalCodec;
                pCodecParameters = pLocalCodecParameters;

                const char *buffer = reinterpret_cast <const char*> (pFormatContext->streams[i]->codecpar->extradata);
                QFile file("extradata.txt"); //!!!!!!!!!!!!!!!! storing extradata for browser visualization
                file.open(QIODevice::WriteOnly);
                QDataStream out(&file);
                out.writeRawData(buffer, pFormatContext->streams[i]->codecpar->extradata_size);
                file.flush();
                file.close();
            }
        }
    }

    AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
    avcodec_parameters_to_context(pCodecContext, pCodecParameters);
    avcodec_open2(pCodecContext, pCodec, NULL);

    AVFrame *pFrame = av_frame_alloc();
    AVPacket *pPacket = av_packet_alloc();

    int response = 0;
    int how_many_packets_to_process = 800;

    int frame_c = 0, counter = 0;
    while(true)
    {
        frame_c = av_read_frame(pFormatContext, pPacket);

        if(frame_c < 0)
            break;

        if(pPacket->stream_index == video_stream_index)
        {
            const char *buffer = reinterpret_cast <const char*> (pPacket->data);
            response = decode_packet(pPacket, pCodecContext, pFrame);

            if(response < 0)
                break;

            counter++;
            if(--how_many_packets_to_process <= 0) break;

            if(counter > 1)
            {
                QFile file("frame.txt"); // !!!!!!!!!!!!!!!! storing single frame for browser visualization
                file.open(QIODevice::WriteOnly);
                QDataStream out(&file);
                out.writeRawData(buffer, arr.size());
                file.flush();
                file.close();
                break;
            }
        }

        av_packet_unref(pPacket);
    }

As the result i stored 2 txt files: 1 with extradata of codec and 2 with frame pixel data. The next step is to decode this data in browser and display picture. Here is how i try to do it:

<input type="file" id="file-input" />
<h3>Contents of the file:</h3>
<pre id="file-content"></pre>

<input type="file" id="file-input2" />
<h3>Contents of the file:</h3>
<pre id="file-content2"></pre>

<canvas id="myCanvas" width="720" height="1280"
style="border:1px solid #d3d3d3;">
Your browser does not support the HTML5 canvas tag.
</canvas>

<script>

let pendingFrames = [];
let extraData = new Uint8Array;

const init = {
  output: handleFrame,
  error: (e) => {
  console.log(e.message);
  },
};

const config = {
  codec: "avc1.64001E",
  codedWidth:  720,
  codedHeight: 1280,
  description: extraData.buffer,
};

function handleFrame(frame) {
  const canvas = document.getElementById("myCanvas");
  const ctx = canvas.getContext("2d");
  ctx.drawImage(frame, 0, 0);
  frame.close();
}

function readSingleFile(e) {
  var file = e.target.files[0];
  if (!file) {
    return;
  }
  var reader = new FileReader();
  reader.onload = function(e) {
    var contents = e.target.result;
    displayContents(contents);
  };
  reader.readAsText(file);
} //end

function displayContents(contents) {
  var element = document.getElementById('file-content');
  element.textContent = contents;
} // end

const readFile = e => {
  var file = e.target.files[0];
  if (!file) {
    return;
  }

  let reader = new FileReader();

  reader.onload = function(e) {
      let arrayBuffer = new Uint8Array(reader.result);
      console.log(arrayBuffer);

        const decoder = new VideoDecoder(init);
        decoder.configure(config);
        
        const chunk = new EncodedVideoChunk({
          timestamp: 20,
          type: "key",
          data: arrayBuffer,
        });

        decoder.decode(chunk);
  }

  reader.readAsArrayBuffer(file);
} //end

const readFile2 = e => {
  var file = e.target.files[0];
  if (!file) {
    return;
  }

  let reader = new FileReader();

  reader.onload = function(e) {
      extraData = new Uint8Array(reader.result);
      console.log(extraData);
  }

  reader.readAsArrayBuffer(file);
} //end

document.getElementById('file-input')
  .addEventListener('change', readFile, false);

document.getElementById('file-input2')
  .addEventListener('change', readFile2, false);

</script>

file-input2 - is for extradata and file-input - for frame data.

After i run this code in browser i got following error message: ---> Failed to execute 'configure' on 'VideoDecoder': description is detached. Also i had another type of error messages such as something wrong with codec config (codec description)

Please help to find what im doing wrong =) Thanks in advance.

mccat
  • 33
  • 4
  • 1
    **(1)** Try replacing `description: extraData.buffer` with just `description: extraData` since a Uint8Array does not have a `.buffer` property. **(2)** Also make sure you save binary data (why use `.txt` instead of using `.h264` to signify to a reader that it's not really a "text" file?). **(3)** Use a hex editor to check your bytes structure (for next step). Make sure your frame data begins with a **start code** (not size) and that the SPS and PPS metadata bytes exist in one of your `.txt` files. – VC.One Mar 25 '23 at 00:51
  • 1
    **(4)** For quicker testing... Make sure your H264 frame is a keyframe data (for immediate picture viewing). Make sure the H264 data is in **AnnexB** format (for setup simplicity, does not need **extraData** etc). Then your code should work (you still need to add SPS and PPS once with first keyframe then onwards can just feed in any other frame types to the decoder). **PS:** See if my [Answer about decoding](https://stackoverflow.com/a/74421476/2057709) helps you. – VC.One Mar 25 '23 at 00:54
  • Thank you so much. Your links helped me a lot. I have dataset of videos for testing. One of them is coded with AV_CODEC_ID_H264 codec (annex b format). Im able to reproduce frames in browser using webcodecs without specifying any codec description. But other videos giving me list of problems with displaying theit frames. I see two solutions: 1) convert each frame to annex b format before sending to browser 2) get extradata of codec in ffmpeg and send it to browser for webcodec descriprion Im new to ffmpeg and reading literature for few days didnt give good results. – mccat Mar 28 '23 at 22:05
  • Can i ask you please point me how to realise both scenarious. The seond one sims to be much simpler and have better performance, but i didnt find any good example or description how to put extradata to webcodec description. Thanks in advance! – mccat Mar 28 '23 at 22:08
  • **(1)** About _"One of them is coded with `AV_CODEC_ID_H264` codec (AnnexB format). I'm able to reproduce frames using WebCodecs... But other videos giving me list of problems with displaying their frames"_ What is "other videos" exactly? A web codec is either: **H264** or **VP9** or OGV (**Theora**). Any other formats will not decode by it. **(2)** Can you provide some testable bytes (can file share or paste bytes)? Share one small keyframe and also its related extradata. Then we can discuss options to decode what you have. – VC.One Mar 29 '23 at 15:51
  • 1
    today i did some tests and decided to simplify my task. Decided to reject all files encoded not with h264 codec. For annex B transformation i decided to use ffmpeg.exe with command "ffmpeg -i input.mp4 -codec copy -bsf:v h264_mp4toannexb output.ts". Works perfect. Maybe will attach code samples after finish my codeing. – mccat Mar 29 '23 at 16:12

0 Answers0