WebGL/OpenGL text labeling animated instanced shapes

Question

I'm rendering a variable number of circles in the plane with variable size, color, and position using instancing. I'm hoping to reach on the order of 10k-100k circles/labels.

    in float instanceSize;
    in vec3 instanceColor;
    in vec2 instanceCenter;

The buffer backing the instanceCenter attribute changes every frame, animating the circles, but the rest is mostly static.

I have a quad per circle and I'm creating the circle in the fragment shader.

Now I'm looking into labeling the shapes with labels with font size proportional to circle size, centered on the circle, moving with the circles. From what I've read the most performant way to do so is to use a glyph texture with a quad for every letter using either a bitmap texture atlas or a signed distance field texture atlas. The examples I've seen seem to do a lot of work on the Javascript side and then use a draw call for every string like: https://webgl2fundamentals.org/webgl/lessons/webgl-text-glyphs.html

Is there a way to render the text with one draw call (with instancing, or otherwise?), while reusing the Float32Array backing instanceCenter every frame? It seems like more work would need to be done in the shaders but I'm not exactly sure how. Because each label has a variable number of glyphs I'm not sure how to associate a single instanceCenter with a single label.

All that aside, more basically I'm wondering how one centers text at a point?

Any help appreciated

Answer 1

Off the top of my head you could store your messages in a texture and add a message texcoord and length per instance. You can then compute the size of the rectangle needed to draw the message in the vertex shader and use that to center as well.

attribute float msgLength;
attribute vec2 msgTexCoord;
...

widthOfQuad = max(minSizeForCircle, msgLength * glphyWidth)

In the fragment shader read the message from the texture and use it look up glyphs (image based or SDF based).

varying vec2 v_msgTexCoord;  // passed in from vertex shader
varying float v_msgLength;   // passed in from vertex shader
varying vec2 uv;             // uv that goes 0 to 1 across quad

float glyphIndex = texture2D(
     messageTexture,
     v_msgTexCoord + vec2(uv.x * v_msgLength / widthOfMessageTexture)).r;

// now convert glyphIndex to tex coords to look up glyph in glyph texture

glyphUV = (up to you)

textColor = texture2D(glyphTexture, 
   glyphUV + glyphSize * vec2(fract(uv.x * v_msgLength), uv.v) / glyphTextureSize);

Or something like that. I have no idea how slow it would be

async function main() {
  const gl = document.querySelector('canvas').getContext('webgl');
  twgl.addExtensionsToContext(gl);

  function convertToGlyphIndex(c) {
    c = c.toUpperCase();
    if (c >= 'A' && c <= 'Z') {
      return c.charCodeAt(0) - 0x41;
    } else if (c >= '0' && c <= '9') {
      return c.charCodeAt(0) - 0x30 + 26;
    } else {
      return 255;
    }
  }

  const messages = [
    'pinapple',
    'grape',
    'banana',
    'strawberry',
  ];
  
  const glyphImg = await loadImage("https://webglfundamentals.org/webgl/resources/8x8-font.png");

  const glyphTex = twgl.createTexture(gl, {
    src: glyphImg,
    minMag: gl.NEAREST,
  });
  // being lazy about size, making them all the same.
  const glyphsAcross = 8;

  // too lazy to pack these in a texture in a more compact way
  // so just put one message per row
  const longestMsg = Math.max(...messages.map(m => m.length));
  const messageData = new Uint8Array(longestMsg * messages.length * 4);
  messages.forEach((message, row) => {
    for (let i = 0; i < message.length; ++i) {
      const c = convertToGlyphIndex(message[i]);
      const offset = (row * longestMsg + i) * 4; 
      const u = c % glyphsAcross;
      const v = c / glyphsAcross | 0;
      messageData[offset + 0] = u;
      messageData[offset + 1] = v;
    }
  });

  const messageTex = twgl.createTexture(gl, {
    src: messageData,
    width: longestMsg,
    height: messages.length,
    minMag: gl.NEAREST,
  });

  const vs = `
  attribute vec4 position;  // a centered quad (-1 + 1)
  attribute vec2 texcoord;
  attribute float messageLength;  // instanced
  attribute vec4 center;          // instanced
  attribute vec2 messageUV;       // instanced

  uniform vec2 glyphDrawSize;

  varying vec2 v_texcoord;
  varying vec2 v_messageUV;
  varying float v_messageLength;

  void main() {
    vec2 size = vec2(messageLength * glyphDrawSize.x, glyphDrawSize.y);
    gl_Position = position * vec4(size, 1, 0) + center;
    v_texcoord = texcoord;
    v_messageUV = messageUV;
    v_messageLength = messageLength;
  }
  `;

  const fs = `
  precision highp float;

  varying vec2 v_texcoord;
  varying vec2 v_messageUV;
  varying float v_messageLength;

  uniform sampler2D messageTex;
  uniform vec2 messageTexSize;

  uniform sampler2D glyphTex;
  uniform vec2 glyphTexSize;

  uniform vec2 glyphSize;

  void main() {
    vec2 msgUV = v_messageUV + vec2(v_texcoord.x * v_messageLength / messageTexSize.x, 0);
    vec2 glyphOffset = texture2D(messageTex, msgUV).xy * 255.0;
    vec2 glyphsAcrossDown = glyphTexSize / glyphSize;
    vec2 glyphUVOffset = glyphOffset / glyphsAcrossDown;
    vec2 glyphUV = fract(v_texcoord * vec2(v_messageLength, 1)) * glyphSize / glyphTexSize;

    vec4 glyphColor = texture2D(glyphTex, glyphUVOffset + glyphUV);

    // do some math here for a circle
    // TBD

    if (glyphColor.a < 0.1) discard;

    gl_FragColor = glyphColor;
  }
  `;

  const prgInfo = twgl.createProgramInfo(gl, [vs, fs]);

  const bufferInfo = twgl.createBufferInfoFromArrays(gl, {
    position: {
      numComponents: 2,
      data: [
        -1, -1,
         1, -1,
        -1,  1,
        -1,  1,
         1, -1,
         1,  1,
      ],
    },
    texcoord: [
       0, 1,
       1, 1,
       0, 0,
       0, 0,
       1, 1,
       1, 0,
    ],
    center: {
      numComponents: 2,
      divisor: 1,
      data: [
        -0.4, 0.1,
        -0.3, -0.5,
         0.6, 0,
         0.1, 0.5,
      ],
    },
    messageLength: {
      numComponents: 1,
      divisor: 1,
      data: messages.map(m => m.length),
    },
    messageUV: {
      numComponents: 2, 
      divisor: 1,
      data: messages.map((m, i) => [0, i / messages.length]).flat(),
    },
  });
  
  gl.clearColor(0, 0, 1, 1);
  gl.clear(gl.COLOR_BUFFER_BIT);

  gl.useProgram(prgInfo.program);

  twgl.setBuffersAndAttributes(gl, prgInfo, bufferInfo);
  twgl.setUniformsAndBindTextures(prgInfo, {
    glyphDrawSize: [16 / gl.canvas.width, 16 / gl.canvas.height],
    messageTex,
    messageTexSize: [longestMsg, messages.length],
    glyphTex,
    glyphTexSize: [glyphImg.width, glyphImg.height],
    glyphSize: [8, 8],
  });
  // ext.drawArraysInstancedANGLE(gl.TRIANGLES, 0, 6, messages.length);
  gl.drawArraysInstanced(gl.TRIANGLES, 0, 6, messages.length);
}

function loadImage(url) {
  return new Promise((resolve, reject) => {
    const img = new Image();
    img.crossOrigin = "anonymous";
    img.onerror = reject;
    img.onload = () => resolve(img);
    img.src = url;
  });
}
main();

<canvas></canvas>
<script src="https://twgljs.org/dist/4.x/twgl.min.js"></script>

note that if the glyphs were different sizes it seems like it would get extremely slow, at least off the top of my head, the only way to find each glyph as you draw a quad would be to loop over all the glyphs in the message for every pixel.

On the other hand, you could build a mesh of glyphs similar to the article, for each message, for every glyph in that message, add a per vertex message id or message uv that you use to look up offsets or matrices from a texture. In this way you can move every message independently but make it all happen in a single draw call. This would allow non-monospaced glyphs. As an example of storing positions or matrices in a texture see this article on skinning. It stores bone matrices in a texture.

async function main() {
  const gl = document.querySelector('canvas').getContext('webgl');
  const ext = gl.getExtension('OES_texture_float');
  if (!ext) {
    alert('need OES_texture_float');
    return;
  }
  twgl.addExtensionsToContext(gl);

  function convertToGlyphIndex(c) {
    c = c.toUpperCase();
    if (c >= 'A' && c <= 'Z') {
      return c.charCodeAt(0) - 0x41;
    } else if (c >= '0' && c <= '9') {
      return c.charCodeAt(0) - 0x30 + 26;
    } else {
      return 255;
    }
  }

  const messages = [
    'pinapple',
    'grape',
    'banana',
    'strawberry',
  ];

  const glyphImg = await loadImage("https://webglfundamentals.org/webgl/resources/8x8-font.png");

  const glyphTex = twgl.createTexture(gl, {
    src: glyphImg,
    minMag: gl.NEAREST,
  });
  // being lazy about size, making them all the same.
  const glyphsAcross = 8;
  const glyphsDown = 5;
  const glyphWidth = glyphImg.width / glyphsAcross;
  const glyphHeight = glyphImg.height / glyphsDown;
  const glyphUWidth = glyphWidth / glyphImg.width;
  const glyphVHeight = glyphHeight / glyphImg.height;

  // too lazy to pack these in a texture in a more compact way
  // so just put one message per row
  const positions = [];
  const texcoords = [];
  const messageIds = [];
  const matrixData = new Float32Array(messages.length * 16);
  const msgMatrices = [];
  const quadPositions = [
     -1, -1,
      1, -1,
     -1,  1,
     -1,  1,
      1, -1,
      1,  1,
  ];
  const quadTexcoords = [
      0,  1,
      1,  1,
      0,  0,
      0,  0,
      1,  1,
      1,  0,
  ];
  messages.forEach((message, id) => {
    msgMatrices.push(matrixData.subarray(id * 16, (id + 1) * 16));
    
    for (let i = 0; i < message.length; ++i) {
      const c = convertToGlyphIndex(message[i]);
      const u = (c % glyphsAcross) * glyphUWidth;
      const v = (c / glyphsAcross | 0) * glyphVHeight;
      for (let j = 0; j < 6; ++j) {
        const offset = j * 2;
        positions.push(
          quadPositions[offset    ] * 0.5 + i - message.length / 2,
          quadPositions[offset + 1] * 0.5,
        );
        texcoords.push(
          u + quadTexcoords[offset    ] * glyphUWidth,
          v + quadTexcoords[offset + 1] * glyphVHeight,
        );
        messageIds.push(id);
      }
    }
  });

  const matrixTex = twgl.createTexture(gl, {
    src: matrixData,
    type: gl.FLOAT,
    width: 4,
    height: messages.length,
    minMag: gl.NEAREST,
    wrap: gl.CLAMP_TO_EDGE,
  });

  const vs = `
attribute vec4 position;
attribute vec2 texcoord;
attribute float messageId;

uniform sampler2D matrixTex;
uniform vec2 matrixTexSize;
uniform mat4 viewProjection;

varying vec2 v_texcoord;

void main() {
  vec2 uv = (vec2(0, messageId) + 0.5) / matrixTexSize;
  mat4 model = mat4(
    texture2D(matrixTex, uv),
    texture2D(matrixTex, uv + vec2(1.0 / matrixTexSize.x, 0)),
    texture2D(matrixTex, uv + vec2(2.0 / matrixTexSize.x, 0)),
    texture2D(matrixTex, uv + vec2(3.0 / matrixTexSize.x, 0)));
  gl_Position = viewProjection * model * position;
  v_texcoord = texcoord;
}
`;

  const fs = `
precision highp float;

varying vec2 v_texcoord;
uniform sampler2D glyphTex;

void main() {
  vec4 glyphColor = texture2D(glyphTex, v_texcoord);

  // do some math here for a circle
  // TBD

  if (glyphColor.a < 0.1) discard;

  gl_FragColor = glyphColor;
}
`;

  const prgInfo = twgl.createProgramInfo(gl, [vs, fs]);

  const bufferInfo = twgl.createBufferInfoFromArrays(gl, {
    position: {
      numComponents: 2,
      data: positions,
    },
    texcoord: texcoords,
    messageId: {
      numComponents: 1,
      data: messageIds
    },
  });

  gl.clearColor(0, 0, 1, 1);
  gl.clear(gl.COLOR_BUFFER_BIT);

  gl.useProgram(prgInfo.program);
  
  const m4 = twgl.m4;
  const viewProjection = m4.ortho(0, gl.canvas.width, 0, gl.canvas.height, -1, 1);
  msgMatrices.forEach((mat, i) => {
    m4.translation([80 + i * 30, 30 + i * 25, 0], mat);
    m4.scale(mat, [16, 16, 1], mat)
  });
  
  // update the matrices
  gl.bindTexture(gl.TEXTURE_2D, matrixTex);
  gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, 4, messages.length, gl.RGBA, gl.FLOAT, matrixData);
  
  twgl.setBuffersAndAttributes(gl, prgInfo, bufferInfo);
  twgl.setUniformsAndBindTextures(prgInfo, {
    viewProjection,
    matrixTex,
    matrixTexSize: [4, messages.length],
    glyphTex,
  });
  gl.drawArrays(gl.TRIANGLES, 0, positions.length / 2);
}

function loadImage(url) {
  return new Promise((resolve, reject) => {
    const img = new Image();
    img.crossOrigin = "anonymous";
    img.onerror = reject;
    img.onload = () => resolve(img);
    img.src = url;
  });
}
main();

<canvas></canvas>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Also see https://stackoverflow.com/a/54720138/128511