server side extracting frame from video using ffmpeg library. Here is code:
AVFormatContext *pFormatContext = avformat_alloc_context();
avformat_open_input(&pFormatContext, "VID_20230219_171331_300.mp4.mov", NULL, NULL);
avformat_find_stream_info(pFormatContext, NULL);
const AVCodec *pCodec = NULL;
AVCodecParameters *pCodecParameters = NULL;
int video_stream_index = -1;
// loop though all the streams and print its main information
for (int i = 0; i < pFormatContext->nb_streams; i++)
{
AVCodecParameters *pLocalCodecParameters = NULL;
pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
const AVCodec *pLocalCodec = NULL;
pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);
if(pLocalCodec==NULL)
continue;
// when the stream is a video we store its index, codec parameters and codec
if(pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO)
{
if(video_stream_index == -1)
{
video_stream_index = i;
pCodec = pLocalCodec;
pCodecParameters = pLocalCodecParameters;
const char *buffer = reinterpret_cast <const char*> (pFormatContext->streams[i]->codecpar->extradata);
QFile file("extradata.txt"); //!!!!!!!!!!!!!!!! storing extradata for browser visualization
file.open(QIODevice::WriteOnly);
QDataStream out(&file);
out.writeRawData(buffer, pFormatContext->streams[i]->codecpar->extradata_size);
file.flush();
file.close();
}
}
}
AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
avcodec_parameters_to_context(pCodecContext, pCodecParameters);
avcodec_open2(pCodecContext, pCodec, NULL);
AVFrame *pFrame = av_frame_alloc();
AVPacket *pPacket = av_packet_alloc();
int response = 0;
int how_many_packets_to_process = 800;
int frame_c = 0, counter = 0;
while(true)
{
frame_c = av_read_frame(pFormatContext, pPacket);
if(frame_c < 0)
break;
if(pPacket->stream_index == video_stream_index)
{
const char *buffer = reinterpret_cast <const char*> (pPacket->data);
response = decode_packet(pPacket, pCodecContext, pFrame);
if(response < 0)
break;
counter++;
if(--how_many_packets_to_process <= 0) break;
if(counter > 1)
{
QFile file("frame.txt"); // !!!!!!!!!!!!!!!! storing single frame for browser visualization
file.open(QIODevice::WriteOnly);
QDataStream out(&file);
out.writeRawData(buffer, arr.size());
file.flush();
file.close();
break;
}
}
av_packet_unref(pPacket);
}
As the result i stored 2 txt files: 1 with extradata of codec and 2 with frame pixel data. The next step is to decode this data in browser and display picture. Here is how i try to do it:
<input type="file" id="file-input" />
<h3>Contents of the file:</h3>
<pre id="file-content"></pre>
<input type="file" id="file-input2" />
<h3>Contents of the file:</h3>
<pre id="file-content2"></pre>
<canvas id="myCanvas" width="720" height="1280"
style="border:1px solid #d3d3d3;">
Your browser does not support the HTML5 canvas tag.
</canvas>
<script>
let pendingFrames = [];
let extraData = new Uint8Array;
const init = {
output: handleFrame,
error: (e) => {
console.log(e.message);
},
};
const config = {
codec: "avc1.64001E",
codedWidth: 720,
codedHeight: 1280,
description: extraData.buffer,
};
function handleFrame(frame) {
const canvas = document.getElementById("myCanvas");
const ctx = canvas.getContext("2d");
ctx.drawImage(frame, 0, 0);
frame.close();
}
function readSingleFile(e) {
var file = e.target.files[0];
if (!file) {
return;
}
var reader = new FileReader();
reader.onload = function(e) {
var contents = e.target.result;
displayContents(contents);
};
reader.readAsText(file);
} //end
function displayContents(contents) {
var element = document.getElementById('file-content');
element.textContent = contents;
} // end
const readFile = e => {
var file = e.target.files[0];
if (!file) {
return;
}
let reader = new FileReader();
reader.onload = function(e) {
let arrayBuffer = new Uint8Array(reader.result);
console.log(arrayBuffer);
const decoder = new VideoDecoder(init);
decoder.configure(config);
const chunk = new EncodedVideoChunk({
timestamp: 20,
type: "key",
data: arrayBuffer,
});
decoder.decode(chunk);
}
reader.readAsArrayBuffer(file);
} //end
const readFile2 = e => {
var file = e.target.files[0];
if (!file) {
return;
}
let reader = new FileReader();
reader.onload = function(e) {
extraData = new Uint8Array(reader.result);
console.log(extraData);
}
reader.readAsArrayBuffer(file);
} //end
document.getElementById('file-input')
.addEventListener('change', readFile, false);
document.getElementById('file-input2')
.addEventListener('change', readFile2, false);
</script>
file-input2 - is for extradata and file-input - for frame data.
After i run this code in browser i got following error message: ---> Failed to execute 'configure' on 'VideoDecoder': description is detached. Also i had another type of error messages such as something wrong with codec config (codec description)
Please help to find what im doing wrong =) Thanks in advance.