How to signal slots in a GUI from a different process?

Question

Context: In Python a main thread spawns a 2nd process (using multiprocessing module) and then launches a GUI (using PyQt4). At this point the main thread blocks until the GUI is closed. The 2nd process is always processing and ideally should emit signal(s) to specific slot(s) in the GUI in an asynchronous manner.

Question: Which approach/tools are available in Python and PyQt4 to achieve that and how? Preferably in a soft-interrupt manner rather than polling.

Abstractly speaking, the solution I can think of is a "tool/handler" instantiated in the main thread that grabs the available slots from the GUI instance and connects with the grabbed signals from the 2nd process, assuming I provide this tool some information of what to expect or hard coded. This could be instantiated to a 3rd process/thread.

Answer 1

This is an example Qt application demonstrating sending signals from a child process to slots in the mother process. I'm not sure this is right approach but it works.

I differentiate between process as mother and child, because the word parent is alread used in the Qt context.
The mother process has two threads. Main thread of mother process sends data to child process via multiprocessing.Queue. Child process sends processed data and signature of the signal to be sent to the second thread of mother process via multiprocessing.Pipe. The second thread of mother process actually emits the signal.

Python 2.X, PyQt4:

from multiprocessing import Process, Queue, Pipe
from threading import Thread
import sys
from PyQt4.QtCore import *
from PyQt4.QtGui import *

class Emitter(QObject, Thread):

    def __init__(self, transport, parent=None):
        QObject.__init__(self,parent)
        Thread.__init__(self)
        self.transport = transport

    def _emit(self, signature, args=None):
        if args:
            self.emit(SIGNAL(signature), args)
        else:
            self.emit(SIGNAL(signature))

    def run(self):
        while True:
            try:
                signature = self.transport.recv()
            except EOFError:
                break
            else:
                self._emit(*signature)

class Form(QDialog):

    def __init__(self, queue, emitter, parent=None):
        super(Form,self).__init__(parent)
        self.data_to_child = queue
        self.emitter = emitter
        self.emitter.daemon = True
        self.emitter.start()
        self.browser = QTextBrowser()
        self.lineedit = QLineEdit('Type text and press <Enter>')
        self.lineedit.selectAll()
        layout = QVBoxLayout()
        layout.addWidget(self.browser)
        layout.addWidget(self.lineedit)
        self.setLayout(layout)
        self.lineedit.setFocus()
        self.setWindowTitle('Upper')
        self.connect(self.lineedit,SIGNAL('returnPressed()'),self.to_child)
        self.connect(self.emitter,SIGNAL('data(PyQt_PyObject)'), self.updateUI)

    def to_child(self):
        self.data_to_child.put(unicode(self.lineedit.text()))
        self.lineedit.clear()

    def updateUI(self, text):
        text = text[0]
        self.browser.append(text)

class ChildProc(Process):

    def __init__(self, transport, queue, daemon=True):
        Process.__init__(self)
        self.daemon = daemon
        self.transport = transport
        self.data_from_mother = queue

    def emit_to_mother(self, signature, args=None):
        signature = (signature, )
        if args:
            signature += (args, )
        self.transport.send(signature)

    def run(self):
        while True:
            text = self.data_from_mother.get()
            self.emit_to_mother('data(PyQt_PyObject)', (text.upper(),))

if __name__ == '__main__':

    app = QApplication(sys.argv)
    mother_pipe, child_pipe = Pipe()
    queue = Queue()
    emitter = Emitter(mother_pipe)
    form = Form(queue, emitter)
    ChildProc(child_pipe, queue).start()
    form.show()
    app.exec_()

And as convenience also Python 3.X, PySide:

from multiprocessing import Process, Queue, Pipe
from threading import Thread

from PySide import QtGui, QtCore

class Emitter(QtCore.QObject, Thread):

    def __init__(self, transport, parent=None):
        QtCore.QObject.__init__(self, parent)
        Thread.__init__(self)
        self.transport = transport

    def _emit(self, signature, args=None):
        if args:
            self.emit(QtCore.SIGNAL(signature), args)
        else:
            self.emit(QtCore.SIGNAL(signature))

    def run(self):
        while True:
            try:
                signature = self.transport.recv()
            except EOFError:
                break
            else:
                self._emit(*signature)

class Form(QtGui.QDialog):

    def __init__(self, queue, emitter, parent=None):
        super().__init__(parent)
        self.data_to_child = queue
        self.emitter = emitter
        self.emitter.daemon = True
        self.emitter.start()
        self.browser = QtGui.QTextBrowser()
        self.lineedit = QtGui.QLineEdit('Type text and press <Enter>')
        self.lineedit.selectAll()
        layout = QtGui.QVBoxLayout()
        layout.addWidget(self.browser)
        layout.addWidget(self.lineedit)
        self.setLayout(layout)
        self.lineedit.setFocus()
        self.setWindowTitle('Upper')
        self.lineedit.returnPressed.connect(self.to_child)
        self.connect(self.emitter, QtCore.SIGNAL('data(PyObject)'), self.updateUI)

    def to_child(self):
        self.data_to_child.put(self.lineedit.text())
        self.lineedit.clear()

    def updateUI(self, text):
        self.browser.append(text[0])

class ChildProc(Process):

    def __init__(self, transport, queue, daemon=True):
        Process.__init__(self)
        self.daemon = daemon
        self.transport = transport
        self.data_from_mother = queue

    def emit_to_mother(self, signature, args=None):
        signature = (signature, )
        if args:
            signature += (args, )
        self.transport.send(signature)

    def run(self):
        while True:
            text = self.data_from_mother.get()
            self.emit_to_mother('data(PyQt_PyObject)', (text.upper(),))

if __name__ == '__main__':

    app = QApplication(sys.argv)
    mother_pipe, child_pipe = Pipe()
    queue = Queue()
    emitter = Emitter(mother_pipe)
    form = Form(queue, emitter)
    ChildProc(child_pipe, queue).start()
    form.show()
    app.exec_()

Answer 2

Hy all,

I hope this is not considered to much of a necro-dump however I thought it would be good to update Nizam's answer by adding updating his example to PyQt5, adding some comments, removing some python2 syntax and most of all by using the new style of signals available in PyQt. Hope someone finds it useful.

"""
Demo to show how to use PyQt5 and qt signals in combination with threads and
processes.

Description:
Text is entered in the main dialog, this is send over a queue to a process that 
performs a "computation" (i.e. capitalization) on the data. Next the process sends 
the data over a pipe to the Emitter which will emit a signal that will trigger 
the UI to update.

Note:
At first glance it seems more logical to have the process emit the signal that 
the UI can be updated. I tried this but ran into the error 
"TypeError: can't pickle ChildProc objects" which I am unable to fix.
"""

import sys
from multiprocessing import Process, Queue, Pipe

from PyQt5.QtCore import pyqtSignal, QThread
from PyQt5.QtWidgets import QApplication, QLineEdit, QTextBrowser, QVBoxLayout, QDialog


class Emitter(QThread):
    """ Emitter waits for data from the capitalization process and emits a signal for the UI to update its text. """
    ui_data_available = pyqtSignal(str)  # Signal indicating new UI data is available.

    def __init__(self, from_process: Pipe):
        super().__init__()
        self.data_from_process = from_process

    def run(self):
        while True:
            try:
                text = self.data_from_process.recv()
            except EOFError:
                break
            else:
                self.ui_data_available.emit(text.decode('utf-8'))


class ChildProc(Process):
    """ Process to capitalize a received string and return this over the pipe. """

    def __init__(self, to_emitter: Pipe, from_mother: Queue, daemon=True):
        super().__init__()
        self.daemon = daemon
        self.to_emitter = to_emitter
        self.data_from_mother = from_mother

    def run(self):
        """ Wait for a ui_data_available on the queue and send a capitalized version of the received string to the pipe. """
        while True:
            text = self.data_from_mother.get()
            self.to_emitter.send(text.upper())


class Form(QDialog):
    def __init__(self, child_process_queue: Queue, emitter: Emitter):
        super().__init__()
        self.process_queue = child_process_queue
        self.emitter = emitter
        self.emitter.daemon = True
        self.emitter.start()

        # ------------------------------------------------------------------------------------------------------------
        # Create the UI
        # -------------------------------------------------------------------------------------------------------------
        self.browser = QTextBrowser()
        self.lineedit = QLineEdit('Type text and press <Enter>')
        self.lineedit.selectAll()
        layout = QVBoxLayout()
        layout.addWidget(self.browser)
        layout.addWidget(self.lineedit)
        self.setLayout(layout)
        self.lineedit.setFocus()
        self.setWindowTitle('Upper')

        # -------------------------------------------------------------------------------------------------------------
        # Connect signals
        # -------------------------------------------------------------------------------------------------------------
        # When enter is pressed on the lineedit call self.to_child
        self.lineedit.returnPressed.connect(self.to_child)

        # When the emitter has data available for the UI call the updateUI function
        self.emitter.ui_data_available.connect(self.updateUI)

    def to_child(self):
        """ Send the text of the lineedit to the process and clear the lineedit box. """
        self.process_queue.put(self.lineedit.text().encode('utf-8'))
        self.lineedit.clear()

    def updateUI(self, text):
        """ Add text to the lineedit box. """
        self.browser.append(text)


if __name__ == '__main__':
    # Some setup for qt
    app = QApplication(sys.argv)

    # Create the communication lines.
    mother_pipe, child_pipe = Pipe()
    queue = Queue()

    # Instantiate (i.e. create instances of) our classes.
    emitter = Emitter(mother_pipe)
    child_process = ChildProc(child_pipe, queue)
    form = Form(queue, emitter)

    # Start our process.
    child_process.start()

    # Show the qt GUI and wait for it to exit.
    form.show()
    app.exec_()

Answer 3

One should first look how Signals/Slots work within only one Python process:

If there is only one running QThread, they just call the slots directly.

If the signal is emitted on a different thread it has to find the target thread of the signal and put a message/ post an event in the thread queue of this thread. This thread will then, in due time, process the message/event and call the signal.

So, there is always some kind of polling involved internally and the important thing is that the polling is non-blocking.

Processes created by multiprocessing can communicate via Pipes which gives you two connections for each side.

The poll function of Connection is non-blocking, therefore I would regularly poll it with a QTimer and then emit signals accordingly.

Another solution might be to have a Thread from the threading module (or a QThread) specifically just waiting for new messages from a Queue with the get function of the queue. See the Pipes and Queues part of multiprocessing for more information..

Here is an example starting a Qt GUI in another Process together with a Thread who listens on a Connection and upon a certain message, closes the GUI which then terminates the process.

from multiprocessing import Process, Pipe
from threading import Thread
import time
from PySide import QtGui

class MyProcess(Process):

    def __init__(self, child_conn):
        super().__init__()
        self.child_conn = child_conn

    def run(self):
        # start a qt application
        app = QtGui.QApplication([])
        window = QtGui.QWidget()
        layout = QtGui.QVBoxLayout(window)
        button = QtGui.QPushButton('Test')
        button.clicked.connect(self.print_something)
        layout.addWidget(button)
        window.show()

        # start thread which listens on the child_connection
        t = Thread(target=self.listen, args = (app,))
        t.start()

        app.exec_() # this will block this process until somebody calls app.quit

    def listen(self, app):
        while True:
            message = self.child_conn.recv()
            if message == 'stop now':
                app.quit()
                return

    def print_something(self):
        print("button pressed")

if __name__ == '__main__':
    parent_conn, child_conn = Pipe()
    s = MyProcess(child_conn)
    s.start()
    time.sleep(5)
    parent_conn.send('stop now')
    s.join()

Answer 4

A quite interesting topic. I guess having a signal that works between threads is a very useful thing. How about creating a custom signal based on sockets? I haven't tested this yet, but this is what I gathered up with some quick investigation:

class CrossThreadSignal(QObject):
    signal = pyqtSignal(object)
    def __init__(self, parent=None):
        super(QObject, self).__init__(parent)
        self.msgq = deque()
        self.read_sck, self.write_sck = socket.socketpair()
        self.notifier = QSocketNotifier(
                           self.read_sck.fileno(), 
                           QtCore.QSocketNotifier.Read
                        )
        self.notifier.activated.connect(self.recv)

    def recv(self):
        self.read_sck.recv(1)
        self.signal.emit(self.msgq.popleft())

    def input(self, message):
        self.msgq.append(message)
        self.write_sck.send('s')

Might just put you on the right track.

Answer 5

I had the same problem in C++. From a QApplication, I spawn a Service object. The object creates the Gui Widget but it's not its parent (the parent is QApplication then). To control the GuiWidget from the service widget, I just use signals and slots as usual and it works as expected. Note: The thread of GuiWidget and the one of the service are different. The service is a subclass of QObject.

If you need multi process signal/slot mechanism, then try to use Apache Thrift or use a Qt-monitoring process which spawns 2 QProcess objects.