Simon is right: the putImageData
method does not pay any attention to compositing; it merely copies pixel values. In order to get compositing, we need to use drawing operations.
We need to mess with the channels (turn red into alpha) with the pixel data, put that changed pixel data into an image, and then use a composite operation to get the desired masking.

//copy from one channel to another
var assignChannel = function(imageData, channelTo, channelFrom) {
if(channelTo < 0 || channelTo > 3 || channelFrom < 0 || channelFrom > 3) {
throw new Error("bad channel number");
}
if(channelTo == channelFrom)
return;
var px = imageData.data;
for(var i = 0; i < px.length; i += 4) {
px[i + channelTo] = px[i + channelFrom];
}
};
/**============================================================================
* this function uses 3 or 4 canvases for clarity / pedagogical reasons:
* redCanvas has our mask image;
* maskCanvas will be used to store the alpha channel conversion of redCanvas' image;
* imageCanvas contains the image to be masked;
* ctx is the context of the canvas to which the masked image will be drawn.
============================================================================**/
var drawOnTopOfRed = function(redCanvas, maskCanvas, imageCanvas, ctx) {
var redImageData = redCanvas.getContext("2d").getImageData(0, 0, w, h);
//assign the alpha channel
assignChannel(redImageData, 3, 0);
//write the mask image
maskCanvas.getContext("2d").putImageData(redImageData, 0, 0);
ctx.save();
//draw the mask
ctx.globalCompositeOperation = "copy";
ctx.drawImage(maskCanvas, 0, 0);
//draw the image to be masked, but only where both it
//and the mask are opaque; see http://www.whatwg.org/specs/web-apps/current-work/multipage/the-canvas-element.html#compositing for details.
ctx.globalCompositeOperation = "source-in";
ctx.drawImage(imageCanvas, 0, 0);
ctx.restore();
};
A doodle with the example:
