My OpenCV Live Webcam Demo Doesn't Show Accurate Emotions

Question

I developed a code by following a non-complete tutorial for emotion recognition with using PyTorch. I had so many errors but i fixed them thanks to the other questions here. But i simply stucked at this one. I am running my code, then my webcam opens but i see "Neutral" emotions all the time, even tough i perform different emotions. But on the console i am seeing other emotions as recorded, like:

tensor(0, device='cuda:0')
tensor(6, device='cuda:0')
tensor(4, device='cuda:0')
tensor(4, device='cuda:0')

You have any idea?

Here's my code:

import cv2 ### pip install opencv-python
##pip install open-cv-contrib-python fullpackage
#from deepface import DeepFace ##pip install deepface
import numpy as np

path = "haarcascade_frontalface_default.xml"
font_scale = 1.5
font = cv2.FONT_HERSHEY_PLAIN

#set the rectangle background to white
rectangle_bgr = (255, 255, 255)
# make a black image
img = np.zeros((500, 500))
#set some text
text = "VİDGA Projesi"
#get the width and height of the text box
(text_width, text_height) = cv2.getTextSize(text, font, fontScale=font_scale, thickness=1)[0]
#set the text start position
text_offset_x = 10
text_offset_y = img.shape[0] - 25
#make the coords of the box with a small padding of two pixels
box_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y - text_height -2))
cv2.rectangle(img, box_coords[0], box_coords[1], rectangle_bgr, cv2.FILLED)
cv2.putText(img, text, (text_offset_x, text_offset_y), font, fontScale=font_scale, color= (0,0,0), thickness=1)

cap = cv2.VideoCapture(1)
#Check if the webcam is opened correctly
if not cap.isOpened():
    cap = cv2.VideoCapture(0)
if not cap.isOpened():
    raise IOError("Cannot open webcam")
    
while True:
    ret, frame = cap.read()
    #eye_Cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')
    faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
    
    if ret == True:
    
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        #print(faceCascade.empty())
        faces = faceCascade.detectMultiScale(gray,1.1,4)
        for x,y,w,h in faces:
            roi_gray = gray[y:y+h, x:x+w]
            roi_color = frame[y:y+h, x:x+w]
            cv2.rectangle(frame, (x,y), (x+w, y+h), (255, 0, 0), 2)
            facess = faceCascade.detectMultiScale(roi_gray)
            if len(facess) == 0:
                print("Face not detected")
            else: 
                for(ex,ey,ew,eh) in facess:
                    face_roi = roi_color[ey: ey+eh, ex: ex+ew] ##cropping the face
                
    graytemp = cv2.cvtColor(face_roi, cv2.COLOR_BGR2GRAY)
    final_image = cv2.resize(graytemp, (48,48))
    final_image = np.expand_dims(final_image, axis =0) #add third dimension
    final_image = np.expand_dims(final_image, axis =0) #add fourth dimension
    final_image = final_image/255.0 # normalization
    dataa = torch.from_numpy(final_image)
    dataa = dataa.type(torch.FloatTensor)
    dataa = dataa.to(device)
    outputs = net(dataa)
    Pred = F.softmax(outputs, dim=1)
    Predictions = torch.argmax(Pred)
    print(Predictions)
    
    font = cv2.FONT_HERSHEY_SIMPLEX
    
    font_scale = 1.5
    font = cv2.FONT_HERSHEY_PLAIN
    
    if ((Predictions)==0):
        status = "Angry"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
        
    elif ((Predictions)==1):
        status = "Disgust"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
        
    elif ((Predictions)==2):
        status = "Fear"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
        
    elif ((Predictions)==3):
        status = "Happy"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
        
    elif ((Predictions)==4):
        status = "Sad"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
        
    elif ((Predictions)==5):
        status = "Surprised"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
        
    elif ((Predictions)==6):
        status = "Neutral"
        
        x1,y1,w1,h1, = 0,0,175,5
        # Draw black background rectangle
        cv2.rectangle(frame, (x1, x1), (x1 + w1, y1 + h1), (0,0,0), -1)
        # Add text
        cv2.putText(frame, status, (x1 + int(w1/10), y1 + int(h1/2)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
        
        cv2.putText(frame, status, (100,150), font, 3,(0,0,255), 2, cv2.LINE_4)
        
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
    
    
        if ret == True:
            cv2.imshow('VIDGA Emotion Recognition', frame) 
            if cv2.waitKey(2) & 0xFF == ord('q'):
                break
        
cap.release()
cv2.destroyAllWindows

is the print(Predictions) whats printing tensor(0, device='cuda:0'). Because if so, Predictions isn't a number its a tensor. — Emile, Mar 19 '21 at 12:30

score 2 · Accepted Answer · answered Mar 19 '21 at 12:34

2

Predictions is a tensor, so you need its value and not the tensor itself.

Change this line:

Predictions = torch.argmax(Pred)

with this:

Predictions = torch.argmax(Pred).item()

answered Mar 19 '21 at 12:34

Doch88

728
1
8
22

I made that change, but im still getting Neutral all the time on camera – Gökhan Uçar Mar 19 '21 at 14:40
@GökhanUçar Do you still see lines like "tensor(0, device='cuda:0')" in your console? – Doch88 Mar 19 '21 at 14:48
No, now it's like: 3 3 3 Face not detected 0 3 6 6 6 – Gökhan Uçar Mar 19 '21 at 14:48
1

You see only neutral because your imshow is only on the last branch of the if. You're showing the image with the text only when the emotion is neutral. Try de-indenting the if that checks ret == True – Doch88 Mar 19 '21 at 14:55
Thanks so much for your help! Now i only have one last question. When i click on close button on the camera, camera keeps opening again. I tried fixing indentations but couldn't make it. – Gökhan Uçar Mar 19 '21 at 15:08
1

@GökhanUçar that is not an easy job to do with OpenCV, try searching on SO for other similar questions, like [this one](https://stackoverflow.com/questions/35003476/opencv-python-how-to-detect-if-a-window-is-closed) – Doch88 Mar 19 '21 at 15:20

My OpenCV Live Webcam Demo Doesn't Show Accurate Emotions

1 Answers1