How to draw eye gaze data from eye tracker by PyQt while a video is playing

Question

I am working bulid a eye gaze visualization tool just like this by PyQt5, and I also checked this post. here is the code by modifing the above links. turn out it's worked, but the video always get stuck sometime (the audio is normal, the the frames stuck, both video content and ellipse,just like this video),anyone can help?

import os
import time

from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets
import tobii_research as tr
import numpy as np

class Widget(QtWidgets.QWidget):

    def __init__(self, parent=None):

        super(Widget, self).__init__(parent)

        #first window,just have a single button for play the video
        self.resize(256, 256)
        self.btn_play = QtWidgets.QPushButton(self)
        self.btn_play.setGeometry(QtCore.QRect(100, 100, 28, 28))
        self.btn_play.setObjectName("btn_open")
        self.btn_play.setText("Play")
        self.btn_play.clicked.connect(self.Play_video)#click to play video
        #

        self._scene = QtWidgets.QGraphicsScene(self)
        self._gv = QtWidgets.QGraphicsView(self._scene)
        #construct a videoitem for showing the video
        self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
        #add it into the scene
        self._scene.addItem(self._videoitem)

        # assign _ellipse_item is the gaze data, and embed it into videoitem,so it can show above the video.
        self._ellipse_item = QtWidgets.QGraphicsEllipseItem(QtCore.QRectF(0, 0, 40, 40), self._videoitem)
        self._ellipse_item.setBrush(QtGui.QBrush(QtCore.Qt.black))
        self._ellipse_item.setPen(QtGui.QPen(QtCore.Qt.red))
        #self._scene.addItem(self._ellipse_item)
        self._gv.fitInView(self._videoitem)

        self._player = QtMultimedia.QMediaPlayer(self, QtMultimedia.QMediaPlayer.VideoSurface)
        self._player.setVideoOutput(self._videoitem)
        file = os.path.join(os.path.dirname(__file__), "video.mp4")#video.mp4 is under the same dirctory
        self._player.setMedia(QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(file)))
        print(f"self._videoitem::{self._videoitem.size()}")

        #get eye tracker
        self.eyetrackers = tr.find_all_eyetrackers()
        self.my_eyetracker = self.eyetrackers[0]

    def gaze_data_callback(self, gaze_data_):

        #for now, I don't know the coordinate system,just randomly assign the gaze data to test the functionality
        self._ellipse_item.setPos(float(np.random.choice(range(0, 300))), float(np.random.choice(range(0, 240))))
        print("time.time()::{}".format(time.time()))


    def Play_video(self):
        self.my_eyetracker.subscribe_to(tr.EYETRACKER_GAZE_DATA, self.gaze_data_callback, as_dictionary=True)
        #size = QtCore.QSizeF(1920.0, 1080.0)#I hope it can fullscreen the video
        #self._videoitem.setSize(size)
        #self._gv.showFullScreen()
        self._gv.resize(720,720)
        self._gv.show()
        self._player.play()


if __name__ == '__main__':
    import sys
    app = QtWidgets.QApplication(sys.argv)
    w = Widget()
    w.show()
    sys.exit(app.exec_())

cmd printout information here

It seems that to run your application it is necessary to have some specific HW so I can not test it so my analysis will depend on what you tell me so I ask you to be detailed in the answers to my questions. Do not set in fullscreen but set an appropriate size so that with the mouse you can change the screen size. Can you move it freely or the window freezes? On the other hand, run your code in the console/CMD and tell me if you have any error messages. — eyllanesc, Feb 18 '20 at 08:12
Hi eyllanesc, really thank you for you reply. I modify the code slightly because I made some mistakes before. I try this code in cmd, and it works same. when I show the QGraphicsView non-fullscreen, the problem still there. like this[video](https://youtu.be/vQCDSHBmbW8) — M conrey, Feb 18 '20 at 09:25
Oh when I run this code on cmd, It do send some information,let me print it out — M conrey, Feb 18 '20 at 09:33

score 1 · Accepted Answer · answered Feb 18 '20 at 18:40

According to the warning message:

QObject::startTimer: Timers can only be used with threads started with QThread

it can be deduced that the callback associated with "my_eyetracker" is executed in a secondary thread, so the position of the item from a different thread to the main thread would be updated, which could generate the problem described by the OP.

The solution is to send the callback information to the guide through signals.

import os
import time

from PyQt5 import QtCore, QtGui, QtWidgets, QtMultimedia, QtMultimediaWidgets
import tobii_research as tr
import numpy as np


class EyeTracker(QtCore.QObject):
    positionChanged = QtCore.pyqtSignal(float, float)

    def __init__(self, tracker, parent=None):
        super(EyeTracker, self).__init__(parent)
        self._tracker = tracker

    @property
    def tracker(self):
        return self._tracker

    def start(self):
        self.tracker.subscribe_to(
            tr.EYETRACKER_GAZE_DATA, self._callback, as_dictionary=True
        )

    def _callback(self, gaze_data_):
        self.positionChanged.emit(
            float(np.random.choice(range(0, 300))),
            float(np.random.choice(range(0, 240))),
        )
        print("time.time()::{}".format(time.time()))


class Widget(QtWidgets.QWidget):
    def __init__(self, parent=None):

        super(Widget, self).__init__(parent)

        # first window,just have a single button for play the video
        self.resize(256, 256)
        self.btn_play = QtWidgets.QPushButton(self)
        self.btn_play.setGeometry(QtCore.QRect(100, 100, 28, 28))
        self.btn_play.setObjectName("btn_open")
        self.btn_play.setText("Play")
        self.btn_play.clicked.connect(self.Play_video)  # click to play video
        #

        self._scene = QtWidgets.QGraphicsScene(self)
        self._gv = QtWidgets.QGraphicsView(self._scene)
        # construct a videoitem for showing the video
        self._videoitem = QtMultimediaWidgets.QGraphicsVideoItem()
        # add it into the scene
        self._scene.addItem(self._videoitem)

        # assign _ellipse_item is the gaze data, and embed it into videoitem,so it can show above the video.
        self._ellipse_item = QtWidgets.QGraphicsEllipseItem(
            QtCore.QRectF(0, 0, 40, 40), self._videoitem
        )
        self._ellipse_item.setBrush(QtGui.QBrush(QtCore.Qt.black))
        self._ellipse_item.setPen(QtGui.QPen(QtCore.Qt.red))
        # self._scene.addItem(self._ellipse_item)
        self._gv.fitInView(self._videoitem)

        self._player = QtMultimedia.QMediaPlayer(
            self, QtMultimedia.QMediaPlayer.VideoSurface
        )
        self._player.setVideoOutput(self._videoitem)
        file = os.path.join(
            os.path.dirname(__file__), "video.mp4"
        )  # video.mp4 is under the same dirctory
        self._player.setMedia(
            QtMultimedia.QMediaContent(QtCore.QUrl.fromLocalFile(file))
        )
        print(f"self._videoitem::{self._videoitem.size()}")

        # get eye tracker
        eyetrackers = tr.find_all_eyetrackers()
        self.tracker = EyeTracker(eyetrackers[0])
        self.tracker.positionChanged.connect(self._ellipse_item.setPos)

    def Play_video(self):
        self.tracker.start()
        # size = QtCore.QSizeF(1920.0, 1080.0)#I hope it can fullscreen the video
        # self._videoitem.setSize(size)
        # self._gv.showFullScreen()
        self._gv.resize(720, 720)
        self._gv.show()
        self._player.play()


if __name__ == "__main__":
    import sys

    app = QtWidgets.QApplication(sys.argv)
    w = Widget()
    w.show()
    sys.exit(app.exec_())

The warning is gone and video didn't get stuck,thank you for you help,I really appriciate it. cheers! — M conrey, Feb 19 '20 at 04:46

How to draw eye gaze data from eye tracker by PyQt while a video is playing

1 Answers1