0

I have a script which has been written to allow multiple processes to be carried out concurrently. In addition to running in parallel, these processes report their properties to the GUI in real time to allow the user to track the processes' progress. Also, I believe I have the system set up in order to allow the user to halt the process at any point in time, change the properties of that process, and then resume the process from where it was with the new properties.

However, it appears that the GUI is still not very responsive to the starting/stopping of the processes and does not keep the changes in properties that are fed to it.

I have received some help on this script here, but it appears that the question was not appropriate for that forum.

This is the code which I have at the moment, which I hope to change in order to achieve all of the goals I listed above:

from PyQt4 import QtCore, QtGui
import multiprocessing as mp
import sys
import time

def make_data():
    data = {'num': 3, 'num2':4}
    return data

num = 1000000

def some_complex_processing(data, can_run):
    for i in range(data['num'], data['num'] + num):
        can_run.wait()
        data['num'] = i
        data['num2'] = i+100
        time.sleep(0.1)

class Tab(QtGui.QTabWidget):
    def __init__(self, parent, data_init):
        QtGui.QTabWidget.__init__(self, parent)
        self.top_level_layout = QtGui.QGridLayout(self)
        self.frame = QtGui.QFrame(self)
        self.top_level_layout.addWidget(self.frame)
        self.step_label = dict()
        self.step_stacked = dict()
        self.step_text = dict()
        self.step_input = dict()
        for wdgts in data_init.keys():
            self.step_label[wdgts] = QtGui.QLabel(str(wdgts))
            self.step_stacked[wdgts] = QtGui.QStackedWidget()
            self.step_text[wdgts] = QtGui.QLabel(str(data_init[wdgts]))
            self.step_input[wdgts] = QtGui.QLineEdit()
            self.step_stacked[wdgts].addWidget(self.step_text[wdgts])
            self.step_stacked[wdgts].addWidget(self.step_input[wdgts])
            self.top_level_layout.addWidget(self.step_stacked[wdgts])
            self.top_level_layout.addWidget(self.step_label[wdgts])

        self.process_button = QtGui.QPushButton("Process")
        self.top_level_layout.addWidget(self.process_button, 1, 1)
        self.process_button.clicked.connect(self.start_process)

        self.manager = mp.Manager()
        self.data = self.manager.dict(make_data())
        self.Event = self.manager.Event()
        self.process = mp.Process(target=some_complex_processing, args=(self.data,self.Event,))

        self.timer = QtCore.QTimer()
        self.timer.timeout.connect(self.update_GUI)

    def update_GUI(self):
        try:
            for wdgt in self.data.keys():
                self.step_label[wdgt].setText(str(wdgt))
                self.step_text[wdgt].setText(str(self.data[wdgt]))
                self.step_input[wdgt].setText(str(self.data[wdgt]))
        except EOFError:
            QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.stop_process)
            QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.start_process)
            self.timer.stop()
            self.process.join()
        return

    def start_process(self):
        self.Event.set()
        for wdgt in self.step_stacked.keys():
            self.step_stacked[wdgt].setCurrentWidget(self.step_text[wdgt])
        self.process.start()
        self.timer.start()
        self.process_button.setText('Stop Processing - (manually adjust data)')
        QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.start_process)
        QtCore.QObject.connect(self.process_button, QtCore.SIGNAL("clicked()"), self.stop_process)
        return

    def stop_process(self):
        self.Event.clear()
        for wdgt in self.step_stacked.keys():
            self.step_stacked[wdgt].setCurrentWidget(self.step_input[wdgt])
        self.timer.stop()
        self.process_button.setText('Start Processing Again Using Data')
        QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.stop_process)
        QtCore.QObject.connect(self.process_button, QtCore.SIGNAL("clicked()"), self.start_process)
        return

# GUI stuff
class MainWindow(QtGui.QMainWindow):
    def __init__(self, parent = None):
        QtGui.QMainWindow.__init__(self)
        screen_height = app.desktop().screenGeometry().height()
        screen_width = app.desktop().screenGeometry().width()
        self.resize(int(screen_width*0.2), int(screen_height*0.2))
        self.tab_list = []
        self.setTabShape(QtGui.QTabWidget.Rounded)
        self.centralwidget = QtGui.QWidget(self)
        self.top_level_layout = QtGui.QGridLayout(self.centralwidget)
        self.tabWidget = QtGui.QTabWidget(self.centralwidget)
        self.top_level_layout.addWidget(self.tabWidget, 1, 0, 25, 25)
        self.setCentralWidget(self.centralwidget)
        self.centralwidget.setLayout(self.top_level_layout)
        self.process_all__button = QtGui.QPushButton("Start All Processes")
        self.top_level_layout.addWidget(self.process_all__button, 0, 0)
        QtCore.QObject.connect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.start_all_process)
        # Make Tabs in loop from button
        for i in range(0,10):
            super_cool_data = make_data()
            name = 'tab ' + str(i)
            self.tab_list.append(Tab(self.tabWidget, super_cool_data))
            self.tabWidget.addTab(self.tab_list[-1], name)

    def start_all_process(self):
        self.process_all__button.setText('Stop All Processing')
        QtCore.QObject.disconnect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.start_all_process)
        QtCore.QObject.connect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.stop_all_process)
        for i in self.tab_list:
            i.start_process()

    def stop_all_process(self):
        self.process_all__button.setText('Start All Processing')
        QtCore.QObject.disconnect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.stop_all_process)
        QtCore.QObject.connect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.start_all_process)
        for i in self.tab_list:
            i.stop_process()        

if __name__ == "__main__":
    app = QtGui.QApplication([])
    win = MainWindow()
    win.show()
    sys.exit(app.exec_())

UPDATE:

I made some changes to the code to reflect the suggestions made in the commentary. Now the loop in the some_complex_processing function will not refer to the loop iteration number, but to the data that was stored in it, which allows the user to see the changes they make.

In addition, I added a time interval to update the GUI with such that it won't block now.

However, there are still issues with zombie processes that are kept around, even though I added a process.daemon = True. Also, the QtCore.QObject.disconnect operations do not seem to be working, as seen by the output of several print statements I have included. It doesn't appear that the buttons ever get disconnected.

Here is the new code:

from PyQt4 import QtCore, QtGui
import multiprocessing as mp
import sys
import time

def make_data():
    data = {'num': 3, 'num2': 4}
    return data

num = 10**8

def some_complex_processing(data, can_run):
    for i in range(data['num'], data['num'] + num):
        can_run.wait()
        data['num'] += 1
        data['num2'] += 100

class Tab(QtGui.QTabWidget):
    def __init__(self, parent, data_init):
        QtGui.QTabWidget.__init__(self, parent)
        self.top_level_layout = QtGui.QGridLayout(self)
        self.frame = QtGui.QFrame(self)
        self.top_level_layout.addWidget(self.frame)
        self.step_label = dict()
        self.step_stacked = dict()
        self.step_text = dict()
        self.step_input = dict()
        for wdgts in data_init.keys():
            self.step_label[wdgts] = QtGui.QLabel(str(wdgts))
            self.step_stacked[wdgts] = QtGui.QStackedWidget()
            self.step_text[wdgts] = QtGui.QLabel(str(data_init[wdgts]))
            self.step_input[wdgts] = QtGui.QLineEdit()
            self.step_stacked[wdgts].addWidget(self.step_text[wdgts])
            self.step_stacked[wdgts].addWidget(self.step_input[wdgts])
            self.top_level_layout.addWidget(self.step_stacked[wdgts])
            self.top_level_layout.addWidget(self.step_label[wdgts])

        self.process_button = QtGui.QPushButton("Process")
        self.top_level_layout.addWidget(self.process_button, 1, 1)
        self.process_button.clicked.connect(self.start_process)

        self.manager = mp.Manager()
        self.data = self.manager.dict(make_data())
        self.Event = self.manager.Event()
        self.process = mp.Process(target=some_complex_processing, args=(self.data,self.Event,))
        self.process.daemon = True

        self.timer = QtCore.QTimer()
        self.timer.timeout.connect(self.update_GUI)
        self.first_start = True

    def update_GUI(self):
        try:
            for wdgt in self.data.keys():
                self.step_label[wdgt].setText(str(wdgt))
                self.step_input[wdgt].setText(str(self.data[wdgt]))
                self.step_text[wdgt].setText(str(self.data[wdgt]))       
        except EOFError:
            QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.stop_process)
            QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.start_process)
            self.timer.stop()
            self.process.join()
        return

    def start_process(self):
        self.Event.set()
        for wdgt in self.step_stacked.keys():
            self.step_stacked[wdgt].setCurrentWidget(self.step_text[wdgt])
        if self.first_start==True:
            print 'first start'
            self.process.start()
            self.first_start = False
        else:
            for wdgt in self.data.keys():
                self.data[wdgt] = int(self.step_input[wdgt].text())
            print 'start process again'
        self.timer.start(100)
        self.process_button.setText('Stop Processing - (manually adjust data)')
        QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.start_process)
        QtCore.QObject.connect(self.process_button, QtCore.SIGNAL("clicked()"), self.stop_process)
        return

    def stop_process(self):
        print 'stopping proccess'
        self.Event.clear()
        for wdgt in self.step_stacked.keys():
            self.step_stacked[wdgt].setCurrentWidget(self.step_input[wdgt])
        self.timer.stop()
        self.process_button.setText('Start Processing Again Using Data')
        QtCore.QObject.disconnect(self.process_button, QtCore.SIGNAL("clicked()"), self.stop_process)
        QtCore.QObject.connect(self.process_button, QtCore.SIGNAL("clicked()"), self.start_process)
        return

# GUI stuff
class MainWindow(QtGui.QMainWindow):
    def __init__(self, parent = None):
        QtGui.QMainWindow.__init__(self)
        screen_height = app.desktop().screenGeometry().height()
        screen_width = app.desktop().screenGeometry().width()
        self.resize(int(screen_width*0.2), int(screen_height*0.2))
        self.tab_list = []
        self.setTabShape(QtGui.QTabWidget.Rounded)
        self.centralwidget = QtGui.QWidget(self)
        self.top_level_layout = QtGui.QGridLayout(self.centralwidget)
        self.tabWidget = QtGui.QTabWidget(self.centralwidget)
        self.top_level_layout.addWidget(self.tabWidget, 1, 0, 25, 25)
        self.setCentralWidget(self.centralwidget)
        self.centralwidget.setLayout(self.top_level_layout)
        self.process_all__button = QtGui.QPushButton("Start All Processes")
        self.top_level_layout.addWidget(self.process_all__button, 0, 0)
        QtCore.QObject.connect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.start_all_process)
        # Make Tabs in loop from button
        for i in range(0,10):
            super_cool_data = make_data()
            name = 'tab ' + str(i)
            self.tab_list.append(Tab(self.tabWidget, super_cool_data))
            self.tabWidget.addTab(self.tab_list[-1], name)

    def start_all_process(self):
        self.process_all__button.setText('Stop All Processing')
        QtCore.QObject.disconnect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.start_all_process)
        QtCore.QObject.connect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.stop_all_process)
        for i in self.tab_list:
            i.start_process()

    def stop_all_process(self):
        self.process_all__button.setText('Start All Processing')
        QtCore.QObject.disconnect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.stop_all_process)
        QtCore.QObject.connect(self.process_all__button, QtCore.SIGNAL("clicked()"), self.start_all_process)
        for i in self.tab_list:
            i.stop_process()        

if __name__ == "__main__":
    app = QtGui.QApplication([])
    win = MainWindow()
    win.show()
    sys.exit(app.exec_())
Community
  • 1
  • 1
chase
  • 3,592
  • 8
  • 37
  • 58
  • Looking through this whole mess of code, I don't see even an attempt to pass progress or anything else back from the child process to the main process. There's no queue or pipe, the only sync object is an `Event` that the parent signals and the child waits on; the progress is implicit in the `i` variable, which is local to `some_complex_processing` and not passed to anything outside of it. – abarnert Nov 27 '13 at 21:11
  • Shouldn't the `multiprocessing.Manager().dict()` hold the dictionary in a shared memory between the Tab object and the process? [Other examples shown here](http://stackoverflow.com/questions/9436757/how-does-multiprocessing-manager-work-in-python) show that with `Manager()` objects the memory is shared between different processes, although it does not propagate into the datatype recursively. – chase Nov 27 '13 at 21:13
  • 1
    Sorry, I misread that; I thought you were replacing `data`, not mutating a `Manager.dict()` instance. Never mind. But is there any way you can strip this down to the essentials, just enough to reproduce the problem, instead of all this GUI code with the relevant stuff hidden within it? – abarnert Nov 27 '13 at 23:30
  • 2
    Meanwhile, you're just creating a timer with the default interval (0) and then running it in repeat mode. That means it will fire on every pass through the event loop. That means that you'll be burning 100% CPU just to poll the dict and update the values as fast as possible. That could explain why it's "not very responsive"—it's spending so much time doing nothing useful that it doesn't have time to respond smoothly. Use a `Condition` or `Event` to signal the parent process to check the values and/or put a smallish but nonzero timeout on the timer and that problem should go away. – abarnert Nov 27 '13 at 23:35

0 Answers0