pyqt emit signal from threading thread

Question

I'm trying to update a pyqt QProgressBar from multiple threads, and from what I understand the best way to do this is by emitting signals back to the main GUI thread (I tried passing the QProgressBar object to the worker threads and though it did seem to work I got a ton of warnings in the interpreter). In the following code I set up a progressSignal signal and connect it to a thread which (for now) just prints whatever was emitted. I then emit from each thread the total percentage. I know this works outside the threads by just throwing out a random emit in line 47, which does come through. However the emit from line 36 does not trigger anything, so it never seems to make it through...

import Queue, threading
from PyQt4 import QtCore
import shutil
import profile

fileQueue = Queue.Queue()

class Communicate(QtCore.QObject):

    progressSignal = QtCore.pyqtSignal(int)

class ThreadedCopy:
    totalFiles = 0
    copyCount = 0
    lock = threading.Lock()

    def __init__(self, inputList, progressBar="Undefined"):
        self.totalFiles = len(inputList)

        self.c = Communicate()
        self.c.progressSignal.connect(self.updateProgressBar)

        print str(self.totalFiles) + " files to copy."
        self.threadWorkerCopy(inputList)


    def CopyWorker(self):
        while True:
            self.c.progressSignal.emit(2000)
            fileName = fileQueue.get()
            shutil.copy(fileName[0], fileName[1])
            fileQueue.task_done()
            with self.lock:
                self.copyCount += 1
                percent = (self.copyCount * 100) / self.totalFiles
                self.c.progressSignal.emit(percent)

    def threadWorkerCopy(self, fileNameList):

        for i in range(16):
            t = threading.Thread(target=self.CopyWorker)
            t.daemon = True
            t.start()
        for fileName in fileNameList:
            fileQueue.put(fileName)
        fileQueue.join()
        self.c.progressSignal.emit(1000)

    def updateProgressBar(self, percent):
        print percent

UPDATE:

Heres a sample with a gui. This one runs but is quite unstable, it crashes regularly and the UI does some weird stuff (progress bar not completing, etc.)

Main.py:

import sys, os
import MultithreadedCopy_5
from PyQt4 import QtCore, QtGui

def grabFiles(path):
    # gets all files (not folders) in a directory
    for file in os.listdir(path):
        if os.path.isfile(os.path.join(path, file)):
            yield os.path.join(path, file)

class MainWin(QtGui.QWidget):

    def __init__(self):
        super(MainWin, self).__init__()
        self.initUI()

    def initUI(self):
        self.progress = QtGui.QProgressBar()

        box = QtGui.QVBoxLayout()
        box.addWidget(self.progress)
        goBtn = QtGui.QPushButton("Start copy")
        box.addWidget(goBtn)

        self.setLayout(box)

        goBtn.clicked.connect(self.startCopy)

    def startCopy(self):
        files = grabFiles("folder/with/files")
        fileList = []
        for file in files:
            fileList.append([file,"folder/to/copy/to"])

        MultithreadedCopy_5.ThreadedCopy(fileList, self.progress)

def main():
    app = QtGui.QApplication(sys.argv)
    ex = MainWin()
    ex.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()

MultithreadedCopy_5.py:

import Queue, threading
from PyQt4 import QtCore
import shutil
import profile

fileQueue = Queue.Queue()

class Communicate(QtCore.QObject):

    progressSignal = QtCore.pyqtSignal(int)

class ThreadedCopy:
    totalFiles = 0
    copyCount = 0
    lock = threading.Lock()

    def __init__(self, inputList, progressBar="Undefined"):
        self.progressBar = progressBar
        self.totalFiles = len(inputList)

        self.c = Communicate()
        self.c.progressSignal.connect(self.updateProgressBar, QtCore.Qt.DirectConnection)

        print str(self.totalFiles) + " files to copy."
        self.threadWorkerCopy(inputList)


    def CopyWorker(self):
        while True:
            fileName = fileQueue.get()
            shutil.copy(fileName[0], fileName[1])
            fileQueue.task_done()
            with self.lock:
                self.copyCount += 1
                percent = (self.copyCount * 100) / self.totalFiles
                self.c.progressSignal.emit(percent)

    def threadWorkerCopy(self, fileNameList):
        for i in range(16):
            t = threading.Thread(target=self.CopyWorker)
            t.daemon = True
            t.start()
        for fileName in fileNameList:
            fileQueue.put(fileName)
        fileQueue.join()

    def updateProgressBar(self, percent):
        self.progressBar.setValue(percent)

#profile.run('ThreadedCopy()')

Answer 1

There are two main problems with your examples.

Firstly, the object that emits the signals is created in the main/gui thread, so any signals its emits will not be cross-thread, and hence not thread-safe. The obvious solution to this is to create the signalling object inside the target function of the worker thread - which means there must be a separate instance for each thread.

Secondly, the while-loop inside the target function is never terminated, which means every ThreadedCopy object will be kept alive after the current copy operation is complete. Since all these objects share the same queue, the behaviour will become unpredictable if any attempt is made to repeat the copy operation. The obvious solution to this is to break out of the while-loop once the queue is empty.

Below is a re-write of MultithreadedCopy_5.py which should solve these issues. However, as stated in the comments, I would still strongly recommend using QThread rather than python threads in this scenario, as it is likely to provide a much more robust and more easily maintainable solution.

import Queue, threading
from PyQt4 import QtCore
import shutil
import profile

fileQueue = Queue.Queue()

class Communicate(QtCore.QObject):
    progressSignal = QtCore.pyqtSignal(int)

class ThreadedCopy:
    totalFiles = 0
    copyCount = 0
    lock = threading.Lock()

    def __init__(self, inputList, progressBar="Undefined"):
        self.progressBar = progressBar
        self.totalFiles = len(inputList)
        print str(self.totalFiles) + " files to copy."
        self.threadWorkerCopy(inputList)

    def CopyWorker(self):
        c = Communicate()
        c.progressSignal.connect(self.updateProgressBar)
        while True:
            try:
                fileName = fileQueue.get(False)
            except Queue.Empty:
                break
            else:
                shutil.copy(fileName[0], fileName[1])
                with self.lock:
                    self.copyCount += 1
                    percent = (self.copyCount * 100) / self.totalFiles
                    c.progressSignal.emit(percent)
                fileQueue.task_done()

    def threadWorkerCopy(self, fileNameList):
        if fileQueue.empty():
            for i in range(16):
                t = threading.Thread(target=self.CopyWorker)
                t.daemon = True
                t.start()
            for fileName in fileNameList:
                fileQueue.put(fileName)
            fileQueue.join()

    def updateProgressBar(self, percent):
        self.progressBar.setValue(percent)

Answer 2

The main problem is the delay of time between sending the signal and receiving, we can reduce that time using processEvents():

You can call this function occasionally when your program is busy performing a long operation (e.g. copying a file).

def CopyWorker(self):
    while True:
        fileName = fileQueue.get()
        shutil.copy(fileName[0], fileName[1])
        fileQueue.task_done()
        with self.lock:
            self.copyCount += 1
            print(self.copyCount)
            percent = (self.copyCount * 100) / self.totalFiles
            self.c.progressSignal.emit(percent)
            QtCore.QCoreApplication.processEvents()