How does the canvas drawImage function convert YUV to sRGB

Question

I work with webcodecs and have found that when I manually convert an NV12 format to an RGB format, I get slightly different values than if I simply write the VideoFrame to a canvas and read the imageData.

The ImageData from the Canvas [87, 87, 39, 255, 86, 86, 39, 255, 85, 85, 39, 255, 83, 84, 39, 255, 81, 83, 39, 255, 79, 82, 38, 255, 77, 81, 37, 255, 76, 80, 36, 255, 79, 83, ... ]

The ImageData from manuel converting the NV12-Format to the RGB-Format: [94, 101, 62, 255, 94, 101, 62, 255, 95, 102, 63, 255, 97, 103, 64, 255, 97, 103, 64, 255, 98, 104, 66, 255, 100, 106, 68, 255, 101, 108, 69, 25, ...]

Can anyone tell me how these differences come about or how the Canvas converts a YUV format to an RGB format?

Here is the code for the two different options:

1. draw the image on a canvas and get the ImageData from the Canvas

    this.context.drawImage(frame, 0,0, this.canvas.width, this.canvas.height);
    let imageData = this.context!.getImageData(0,0,this.canvas.width,this.canvas.height);

1. Write the content of a VideoFrame into a Array and convert the Array to an RGB-Format.

  ...
  let imageData = convertToImageData(videoFrame);
  ...

  public async convertToImageData(frame) {
    let buffer : Uint8ClampedArray = new Uint8ClampedArray(frame.allocationSize());
    await frame.copyTo(buffer) 
    let imageData = await this.convertNV12ToRGB(buffer)
    return imageData;
  }

  private convertNV12ToRGB(buffer : Uint8ClampedArray) {
    // Y should be from 0-921599 (1280x720)
    // Cr & Cb should be from 921600 - 1382399 interleaved
    // NOTE: Solution is slow: 1280 * 720; 1 Minute -> 44 Seconds full convert.
    
    let imageData : ImageData = this.context!.getImageData(0, 0, 1280, 720);
        
    let pixels : Uint8ClampedArray = imageData.data;

    let row : number = 0;
    let col : number = 0;
    const plane2Start = 1280 * 720; //Hier starten die Chroma-Werte
    for (row = 0; row < 720; row++) {
      const p2row = (row % 2 === 0 ? row / 2 : row / 2 - 0.5) * 1280; //Hälfte der Row 
      let cr = 0;
      let cb = 0;
      const rowOffset : number = row * 1280;
      for (col = 0; col < 1280; col++) {
        const indexY = rowOffset + col;
        let y = buffer[indexY];
        if (col % 2 === 0) {
          const indexCr = plane2Start + p2row + (col);
          const indexCb = indexCr + 1;
          cr = buffer[indexCr];
          cb = buffer[indexCb];
        }
        this.yuvToRgb(y, cr, cb, row, col, pixels);
      }
    }
    return imageData;
  }

  private yuvToRgb(y : number, u : number, v : number, row : number, col : number, pixels : Uint8ClampedArray) {
    y -= 16;
    u -= 128;
    v -= 128;
    let r = 1.164 * y             + 1.596 * v;
    let g = 1.164 * y - 0.392 * u - 0.813 * v;
    let b = 1.164 * y + 2.017 * u;

    const index = (row * 1280 + col) * 4;
    pixels[index] = r;
    pixels[index + 1] = g;
    pixels[index + 2] = b;
    pixels[index + 3] = 255;
  } ```

Answer 1

The conversion formula used by Canvas applies BT.709 "limited range" conversion coefficients.

Conversion YUV to RGB matrix coefficients (after subtracting [16, 128, 128]):

1.1644   -0.0000    1.7927
1.1644   -0.2132   -0.5329
1.1644    2.1124    0.0000

It looks like Canvas conversion uses some kind of approximation, because I can't get the exact results.

JavaScript pixel conversion code:

function yuvToRgb(y, u, v, row, col, pixels) {
  y -= 16;
  u -= 128;
  v -= 128;

  //let r = 1.164 * y             + 1.596 * v;
  //let g = 1.164 * y - 0.392 * u - 0.813 * v;
  //let b = 1.164 * y + 2.017 * u;

  //Use BT.709 "limited range" conversion formula:
  //1.1644   -0.0000    1.7927
  //1.1644   -0.2132   -0.5329
  //1.1644    2.1124    0.0000
  let r = 1.1644 * y              + 1.7927 * v;
  let g = 1.1644 * y - 0.2132 * u - 0.5329 * v;
  let b = 1.1644 * y + 2.1124 * u;

  const index = (row * 4 + col) * 4;
  pixels[index] = Math.max(Math.min(Math.round(r), 255), 0); //Round and clip range to [0, 255]
  pixels[index + 1] = Math.max(Math.min(Math.round(g), 255), 0);
  pixels[index + 2] = Math.max(Math.min(Math.round(b), 255), 0);
  pixels[index + 3] = 255;
}

Notes:

• Each element is rounded and clipped to range [0, 255].
• Your code has a naming convention issue.
  The naming convention is: u applies cb, and v applies cr.

---

Testing code:

function yuvToRgb(y, u, v, row, col, pixels) {
  y -= 16;
  u -= 128;
  v -= 128;

  //let r = 1.164 * y             + 1.596 * v;
  //let g = 1.164 * y - 0.392 * u - 0.813 * v;
  //let b = 1.164 * y + 2.017 * u;

  //Use BT.709 "limited range" conversion formula:
  //1.1644   -0.0000    1.7927
  //1.1644   -0.2132   -0.5329
  //1.1644    2.1124    0.0000
  let r = 1.1644 * y              + 1.7927 * v;
  let g = 1.1644 * y - 0.2132 * u - 0.5329 * v;
  let b = 1.1644 * y + 2.1124 * u;

  const index = (row * 4 + col) * 4;
  pixels[index] = Math.max(Math.min(Math.round(r), 255), 0); //Round and clip range to [0, 255]
  pixels[index + 1] = Math.max(Math.min(Math.round(g), 255), 0);
  pixels[index + 2] = Math.max(Math.min(Math.round(b), 255), 0);
  pixels[index + 3] = 255;
}

//read image
function convertNV12ToRGBExample() {
  let NV12 = new Array(16, 81, 145, 210, 16, 170, 106, 41, 16, 170, 106, 41, 81, 177, 170, 74, 128, 128, 128, 128, 105, 118, 207, 128);
  let RGBA = new Array(64); //Output

  let row = 0;
  let col = 0;
  const plane2Start = 4 * 4;
  for (row = 0; row < 4; row++) {
    const p2row = (row % 2 === 0 ? row / 2 : (row-1) / 2) * 4;
    let cr = 0;
    let cb = 0;
    const rowOffset = row * 4;
    for (col = 0; col < 4; col++) {
      const indexY = rowOffset + col;
      let y = NV12[indexY];
      if (col % 2 === 0) {
        //const indexCr = plane2Start + p2row + col;
        //const indexCb = indexCr + 1;
        const indexCb = plane2Start + p2row + col;
        const indexCr = indexCb + 1;
        cb = NV12[indexCb];
        cr = NV12[indexCr];
      }

      yuvToRgb(y, cb, cr, row, col, RGBA);
    }
  }

  console.log(RGBA);
}

---

MATLAB testing code (code is incomplete):

NV12 = uint8([16,81,145,210,16,170,106,41,16,170,106,41,81,177,170,74,128,128,128,128,105,118,207,128]);
convertNV12ToRGB = uint8([0, 0, 0, 255, 75, 75, 75, 255, 150, 150, 150, 255, 225, 225, 225, 255, 241, 17, 210, 255, 163, 196, 132, 255, 104, 73, 8, 255, 29, 255, 188, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]);
Canvas2D         = uint8([0, 0, 0, 255, 76, 76, 76, 255, 150, 150, 150, 255, 226, 226, 226, 255, 0, 0, 0, 255, 179, 179, 179, 255, 105, 105, 105, 255, 29, 29, 29, 255, 0, 10, 0, 255, 161, 190, 133, 255, 105, 87, 255, 255, 29, 12, 187, 255, 58, 86, 30, 255, 169, 198, 141, 255, 179, 162, 255, 255, 68, 50, 226, 255]);

Y = reshape(NV12(1:16), [4, 4])';
U = reshape(NV12(17:2:end), [2, 2])';
V = reshape(NV12(18:2:end), [2, 2])';
U = imresize(U, [4, 4], 'nearest');
V = imresize(V, [4, 4], 'nearest');
YUV = cat(3, Y, U, V);

T = [1.1644   -0.0000    1.7927
     1.1644   -0.2132   -0.5329
     1.1644    2.1124    0.0000];

yuv = double(YUV);
yuv(:,:,1) = yuv(:,:,1) - 16;
yuv(:,:,2) = yuv(:,:,2) - 128;
yuv(:,:,3) = yuv(:,:,3) - 128;

RGB = zeros(size(Y,1), size(Y,2), 3);
RGB(:,:,1) = T(1,1) * yuv(:,:,1) + T(1,2) * yuv(:,:,2) + T(1,3) * yuv(:,:,3);
RGB(:,:,2) = T(2,1) * yuv(:,:,1) + T(2,2) * yuv(:,:,2) + T(2,3) * yuv(:,:,3);
RGB(:,:,3) = T(3,1) * yuv(:,:,1) + T(3,2) * yuv(:,:,2) + T(3,3) * yuv(:,:,3);
RGB = uint8(round(RGB));

RGBA = cat(3, RGB(:,:,1), RGB(:,:,2), RGB(:,:,3), uint8(ones(4)*255)); % Convert to RGBA
rgba_data = reshape(permute(RGBA, [3, 2, 1]), [1, 64]);  % Convert to vector

C = permute(reshape(Canvas2D, [4, 4, 4]), [3, 2, 1]);
C = C(:, :, 1:3);