OpenCV: augmenting video stream source .png not rendering transparency as expected

Question

I am trying to create my own face filtering augmented reality program in open-cv. The idea is it will map a beaver to the users face. Currently, I am unable to get proper transperency when loading this image with 'cv2.imread(...)'. It looks black in the background, and often shows partially in certain areas that are white. When I open this image in photoshop, I am fully capable of moving this image on top of a background with expected transparency results. I am wondering if the alpha is not getting rendered properly. Here is the relevent code where I am loading the image in.

import numpy
import cv2

def augment_stream(face: numpy.array, augment: numpy.array) -> numpy.array:
    face_h, face_w, _ = face.shape
    augment_h, augment_w, _ = augment.shape

    scalar = min(face_h / augment_h, face_w / augment_w)
    delta_augment_h = int(scalar * augment_h)
    delta_augment_w = int(scalar * augment_w)

    delta_augment_shape = (delta_augment_w, delta_augment_h)
    resized_augment = cv2.resize(augment, delta_augment_shape)

    augmented_face = face.copy()
    dark_pixels = (resized_augment < 250).all(axis=2)
    offset_x = int((face_w - delta_augment_w) / 2)
    offset_y = int((face_h - delta_augment_h) / 2)

    augmented_face[offset_y: offset_y+delta_augment_h, offset_x: offset_x+delta_augment_w][dark_pixels] = resized_augment[dark_pixels]

    return augmented_face

def main():
    stream = cv2.VideoCapture(0)
    cascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
    augment = cv2.imread('assets/normal.png')
    # tmp = cv2.cvtColor(augment, cv2.COLOR_BGR2GRAY)
    # _,alpha = cv2.threshold(tmp,0,255,cv2.THRESH_BINARY)
    # b, g, r = cv2.split(augment)
    # rgba = [b,g,r, alpha]
    # dst = cv2.merge(rgba,4)
    # cv2.imwrite("assets/normal.png", dst)


    while True:
        ret, border = stream.read()
        border_h, border_w, _ = border.shape

        bw = cv2.equalizeHist(cv2.cvtColor(border, cv2.COLOR_BGR2GRAY))

        rects = cascade.detectMultiScale(bw, scaleFactor=1.3, minNeighbors=4, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)

        for x, y, w, h in rects:
            y0 = int(y - 0.25*h)
            y1 = int(y + 0.75*h)
            x0 = x
            x1 = x + w

            if x0 < 0 or x1 > border_w or y0 < 0 or y1 > border_h:
                continue

            border[y0: y1, x0: x1] = augment_stream(border[y0: y1, x0: x1], augment)

        cv2.imshow('border', border)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    stream.release()
    cv2.destroyAllWindows()


if __name__ == '__main__':
    main()

Answer 1

Using example from question overlay a smaller image on a larger image python OpenCv

I reduced it to show only how to put image

• use cv2.IMREAD_UNCHANGED to load it as RGBA.
• split it to RGB and A
• use A to create masks for image and border
• use loop to add channels

import cv2

stream = cv2.VideoCapture(0)

# load RGBA
augment = cv2.imread('image.png', cv2.IMREAD_UNCHANGED) # load RGBA

# make it smaller then frame - only for test
W = 320
H = 240
augment = cv2.resize(augment, (W, H))

# split image and alpha
image   = augment[:,:,0:3]
alpha   = augment[:,:,3]
mask_image  = alpha / 255.0
mask_border = 1.0 - mask_image

# ROI - region of interest
x1 = 200
y1 = 100
x2 = x1 + W
y2 = y1 + H

while True:
    ret, border = stream.read()

    # copy only in some region (ROI) (don't assign to variable) but gives worse result
    #cv2.copyTo(image, alpha, border[y1:y2, x1:x2]) 

    for c in range(0, 3): # channels RGB
        border[y1:y2, x1:x2, c] = (image[:, :, c]*mask_image + border[y1:y2, x1:x2, c]*mask_border)

    cv2.imshow('border', border)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

stream.release()
cv2.destroyAllWindows()

---

BTW: I tried to use cv2.copyTo(image, mask_image, border) but it gives worse result - maybe mask/alpha needs 3 channels.

It seems it can be done in C/C++ - how to insert a small size image on to a big image