Is there a way to integrate the imshow() function of OpenCV into kivy or kv file in python

Question

I'm new here and hope for a little help and would be very happy about that. I write a small program in python, kivy and opencv. The problem is that I would like to integrate my webcam with opencv and not via the existing camera function from kivy. I have already found a similar problem here Integrate OpenCV webcam into a Kivy user interface but this does not solve my problem. In my OpenCV code, also runs a code for facial recognition (https://github.com/ageitgey/face_recognition/blob/master/examples/facerec_from_webcam_faster.py). So it is therefore important that the command imshow() is issued. How can I integrate the webcam version of imshow() from Opencv into kivy or into a kv file? Unfortunately, I don't know if something like that might work. Can one of you help me or has a idea. Thank you very much for your help.

Python file:

import cv2
from kivy.app import App
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen


class MainScreen(Screen):
    pass


class Manager(ScreenManager):
    pass


kv = Builder.load_file("file.kv")


class Main(App):
    def build(self):
        return kv


if __name__ == '__main__':
    Main().run()

OpenCV - Code:

import cv2

cam = cv2.VideoCapture(0)
while(True):
    ret, frame = cam.read()
    # ...
    # more code
    # ...
    cv2.imshow('frame',frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

My Kivy file (minimal):

MainScreen:
    MainScreen:

<MainScreen>:
    name: "Test"

    FloatLayout:
        Label:
            text: "Webcam from OpenCV?"
            pos_hint: {"x":0.0, "y":0.8}
            size_hint: 1.0, 0.2


        Button:
            text: 'Click me!!'
            pos_hint: {"x":0.0, "y":0.0}
            size_hint: 1.0, 0.2
            font_size: 50

Answer 1

Here is a hack that sort of does what I think you want:

import threading
from functools import partial
import cv2
from kivy.app import App
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen


class MainScreen(Screen):
    pass


class Manager(ScreenManager):
    pass


Builder.load_string('''
<MainScreen>:
    name: "Test"

    FloatLayout:
        Label:
            text: "Webcam from OpenCV?"
            pos_hint: {"x":0.0, "y":0.8}
            size_hint: 1.0, 0.2

        Image:
            # this is where the video will show
            # the id allows easy access
            id: vid
            size_hint: 1, 0.6
            allow_stretch: True  # allow the video image to be scaled
            keep_ratio: True  # keep the aspect ratio so people don't look squashed
            pos_hint: {'center_x':0.5, 'top':0.8}

        Button:
            text: 'Stop Video'
            pos_hint: {"x":0.0, "y":0.0}
            size_hint: 1.0, 0.2
            font_size: 50
            on_release: app.stop_vid()
''')


class Main(App):
    def build(self):

        # start the camera access code on a separate thread
        # if this was done on the main thread, GUI would stop
        # daemon=True means kill this thread when app stops
        threading.Thread(target=self.doit, daemon=True).start()

        sm = ScreenManager()
        self.main_screen = MainScreen()
        sm.add_widget(self.main_screen)
        return sm

    def doit(self):
        # this code is run in a separate thread
        self.do_vid = True  # flag to stop loop

        # make a window for use by cv2
        # flags allow resizing without regard to aspect ratio
        cv2.namedWindow('Hidden', cv2.WINDOW_NORMAL | cv2.WINDOW_FREERATIO)

        # resize the window to (0,0) to make it invisible
        cv2.resizeWindow('Hidden', 0, 0)
        cam = cv2.VideoCapture(0)

        # start processing loop
        while (self.do_vid):
            ret, frame = cam.read()
            # ...
            # more code
            # ...

            # send this frame to the kivy Image Widget
            # Must use Clock.schedule_once to get this bit of code
            # to run back on the main thread (required for GUI operations)
            # the partial function just says to call the specified method with the provided argument (Clock adds a time argument)
            Clock.schedule_once(partial(self.display_frame, frame))

            cv2.imshow('Hidden', frame)
            cv2.waitKey(1)
        cam.release()
        cv2.destroyAllWindows()

    def stop_vid(self):
        # stop the video capture loop
        self.do_vid = False

    def display_frame(self, frame, dt):
        # display the current video frame in the kivy Image widget

        # create a Texture the correct size and format for the frame
        texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt='bgr')

        # copy the frame data into the texture
        texture.blit_buffer(frame.tobytes(order=None), colorfmt='bgr', bufferfmt='ubyte')

        # flip the texture (otherwise the video is upside down
        texture.flip_vertical()

        # actually put the texture in the kivy Image widget
        self.main_screen.ids.vid.texture = texture


if __name__ == '__main__':
    Main().run()

This hides the imshow() window (by making its size 0x0), then displays the frame in an Image Widget. Not sure if a window size of 0x0 messes with your other code.

Answer 2

I used the code from above and modified it a bit. Problems arise when I use/return a kivy file instead of the ScreenManger. In my example there is no video output. What is the problem/fail in the code?

main.py

import threading
from functools import partial
import cv2
from kivy.app import App
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen


class Scan(Screen):
    pass


class WindowManager(ScreenManager):
    pass


kv = Builder.load_file("kivy.kv")


class Main(App):

def build(self):
    threading.Thread(target=self.doit, daemon=True).start()
    self.new_screen = Scan()  # ?
    return kv


def doit(self):
    self.do_vid = True
    cv2.namedWindow('Hidden', cv2.WINDOW_NORMAL | cv2.WINDOW_FREERATIO)
    cv2.resizeWindow('Hidden', 0, 0)
    cam = cv2.VideoCapture(0)

    while (self.do_vid):
        ret, frame = cam.read()
        # ...
        Clock.schedule_once(partial(self.display_frame, frame))
        cv2.imshow('Hidden', frame)
        cv2.waitKey(1)
    cam.release()
    cv2.destroyAllWindows()


def display_frame(self, frame, dt):
    texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt='bgr')
    texture.blit_buffer(frame.tobytes(order=None), colorfmt='bgr', bufferfmt='ubyte')
    texture.flip_vertical()

     # No output of the Video Stream
     # Scan().ids.vid.texture = texture also doesn't work
     self.new_screen.ids.vid.texture = texture

if __name__ == '__main__':
    Main().run()

kivy.kv

WindowManager:
    Scan:

<Scan>:
    name: 'scan'

    FloatLayout:

        Image:
            id: vid
            allow_stretch: True
            keep_ratio: True
            pos_hint: {'x':0.0, 'y':0.2}
            size_hint: 1.0, 0.8


        Button:
            id: button_start
            text: 'Button'
            pos_hint: {'x':0.0, 'y':0.0}
            size_hint: 0.7, 0.2
            background_color: 0.7, 0.9, 0.0, 1
            font_size: 50

        Image:
            id: folder
            source: 'pic/folder.png'
            pos_hint: {'x':0.7, 'y':0.0}
            size_hint: 0.3, 0.2