I can capture the frames from the canvas using readPixels (https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/readPixels) but then what?
I think there are a few wasm versions of ffmpeg but they're slow.
I found this web assembly mp4 encoder but it's no longer supported and doesn't work in Chrome on my phone (Android 13).
Any other suggestions?
"What's the most performant way to encode an mp4 video of frames from a webgl canvas using Javascript in a web browser?"
The most performant way would be to use the WebCodecs API. It is a built-in feature of Chrome browser (and is a supported feature on Android).
An MP4 is a container of frames for audio and video. You want to first encode your pixels in a compressed video format (eg: H.264) then either play that H.264 file with a desktop media player (eg: VLC), or if you need to display it in the browser then it will have to be muxed into an MP4 container...
If creating your own muxer then you need to look up the MP4 specifications (MOV specs are also usable due to both formats being similar from the same ISO specs).
MP4 is mostly integers so you can create a (normal) array, fill it with valid values for an MP4 then add your encoded frames to have a playable MP4 file.
//# write 32 bits in BIG Endian format to a position with an Array
//# params: ( target Array, write position Offset, Value to write )
function write_uint32_BE ( in_Array, in_Offset, in_val )
{
in_Array[ in_Offset + 0 ] = (in_val & 0xFF000000) >>> 24;
in_Array[ in_Offset + 1 ] = (in_val & 0x00FF0000) >>> 16;
in_Array[ in_Offset + 2 ] = (in_val & 0x0000FF00) >>> 8;
in_Array[ in_Offset + 3 ] = (in_val & 0x000000FF);
}
So to create different atoms (or boxes) of MP4 data you have to do the following...
These example bytes are for an stss box (tells which frame nums are keyframes.
For more info on stss atom/box then see Sample Table - Sync Samples):
00 00 00 18 73 74 73 73 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 08
where each 4 bytes means:
00 00 00 18 = Size 0x18 == 24 decimal
73 74 73 73 = text "stss"
00 00 00 00 = version etc
00 00 00 02 = total number of entries of Keyframes
00 00 00 01 = entry 1 == frame 1
00 00 00 08 = entry 2 == frame 8
We can see that frame 1 and frame 8 are both keyframes. The MP4 decoder will not crash so far.
To create the above bytes by code you would run the "write 32-bit uint" function 6 times:
write_uint32_BE ( myMP4_bufferArray, (someOffsetposition + 0), 0x00000018 );
write_uint32_BE ( myMP4_bufferArray, (someOffsetposition + 4), 0x73747373 );
write_uint32_BE ( myMP4_bufferArray, (someOffsetposition + 8), 0x00000000 );
write_uint32_BE ( myMP4_bufferArray, (someOffsetposition + 12), 0x00000002 );
write_uint32_BE ( myMP4_bufferArray, (someOffsetposition + 16), 0x00000001 );
write_uint32_BE ( myMP4_bufferArray, (someOffsetposition + 20), 0x00000008 );
Creating the bytes of an MP4 is awhole other topic. You need to first be able to encode a frame.
Try this code below for encoding a test video frame...
It will encode only to H264 (compressed picture) format. You will need to write more code to create bytes of a container format like MP4 (or even AVI, FLV, MOV, MKV, etc). This is called muxing and you can try a library like Mux.js, MP4Box.js, or else write your own muxing code to put audio/video frames into one container.
<!DOCTYPE html>
<html>
<body>
<canvas id="myCanvas" width="320" height="240" style="" >
</canvas>
<script>
var myCanvas = document.getElementById("myCanvas");
//## Encoder vars
var fileName;
var is_keyFrame = true;
var myConfigBytes;
var encoded_chunkData;
var frameFromCanvas;
////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////
//# Phase (1)
//# Setup an Encoder ...
const encoder_init = {
output: handleChunk,
error: (e) => { console.log("Encode error : " + e.message) },
};
const encoder_config = {
//# set codec
codec: "avc1.42C01E", //# works 420 Baseline profile
//# set AVC format to "annexb" to get START CODES added automatically
avc: { format: 'annexb' },
//# best to set when decoding, not when encoding
//hardwareAcceleration: 'prefer-hardware',
width: 320,
height: 240,
framerate: 30,
latencyMode: "realtime",
//latencyMode: "quality", //# (default)
//bitrateMode: "constant",
bitrateMode: "variable",
//# if you want to set it manually
//bitrate: 500_000, //# test 500 kbps
//bitrate: 2_000_000, //# test 2 Mbps
};
const encoder = new VideoEncoder( encoder_init );
encoder.configure( encoder_config );
console.log(">>> Encoder is now ready");
////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////
//# Phase (2)
//# Create Image data ...
var vid_width = 320; var vid_height = 240;
myCanvas.setAttribute( "width", vid_width );
myCanvas.setAttribute( "height", vid_height );
var ctx_frame = myCanvas.getContext("2d");
var ctx_frame_imageData = ctx_frame.createImageData( vid_width, vid_height );
//# //# fill canvas with some R-G-B values at each pixel component
for (let i = 0; i < ctx_frame_imageData.data.length; i += 4)
{
//# Modify pixel data (blue square on main canvas)
ctx_frame_imageData.data[i + 0] = 200; //# R value
ctx_frame_imageData.data[i + 1] = 190; //# G value
ctx_frame_imageData.data[i + 2] = 90; //# B value
ctx_frame_imageData.data[i + 3] = 255; // Alpha value
}
//# Draw image data to the canvas
ctx_frame.putImageData(ctx_frame_imageData, 0, 0);
////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////
//# Phase (3)
//# Encode the Image data as a Video Frame ...
frameFromCanvas = new VideoFrame( myCanvas, { timestamp: 0 });
encode_frame( frameFromCanvas );
function encode_frame ( frame )
{
if (encoder.encodeQueueSize > 2)
{
//# drop this frame (since 2 is too many frames in 1 queue)
frame.close();
}
else
{
is_keyFrame = true;
//# encode as "keyframe" (or false == a "delta" frame)
encoder.encode(frame, { keyFrame: is_keyFrame } );
}
}
let tmp_obj_metadata;
//# handle chunks (eg: could mux frames into fragmented MP4 container)
async function handleChunk( encoded_frame_data, metadata)
{
if (metadata.decoderConfig)
{
//# save the decoder description (is SPS and PPS for AVC / MP4 )
myConfigBytes = new Uint8Array( metadata.decoderConfig.description );
}
//# get actual bytes of encoded data
encoded_chunkData = new Uint8Array( encoded_frame_data.byteLength );
encoded_frame_data.copyTo( encoded_chunkData );
//# end the encoding session (eg: if no more frames are expected)
let temp = await encoder.flush();
//# test save of encoded bytes
myFile_name = "vc_test_encode_webcodecs_05.h264"
//saveArrayToFile( encoded_chunkData, myFile_name )
return temp;
}
////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////
//# Phase (4)
//# Save H.264 as file...
function saveArrayToFile( in_Array, in_fileName )
{
let temp_out_bytes = [...myConfigBytes, ...in_Array]
let myBlob = new Blob(
[ Uint8Array.from( temp_out_bytes ) ] ,
{type: "application/octet-stream"}
);
let myBlob_url = window.URL.createObjectURL( myBlob );
let tmpOutFile = document.createElement("a");
tmpOutFile.href = myBlob_url;
tmpOutFile.download = in_fileName;
tmpOutFile.click();
window.URL.revokeObjectURL( myBlob_url );
}
</script>
</body>
</html>
