2

I have long running file I/O tasks which I'd like to be able to move into a daemon/server process. A CLI tool would be used to queue new jobs to run, query the status of running jobs, and wait for individual jobs. Python's multiprocessing.managers looks like a nice simple way to handle the IPC. I'd like to be able to construct a SyncManager.Event for the client to wait on without blocking the server, but attempting to do so results in triggers a "server not yet started" assertion. Ironically this assertion gets sent from the server to the client, so obviously the server is started, somewhere.

Here's the minimal example:

#!/usr/bin/env python3
import time
import sys
import concurrent.futures
from multiprocessing.managers import SyncManager

def do_work(files):
    """Simulate doing some work on a set of files."""
    print(f"Starting work for {files}.")
    time.sleep(2)
    print(f"Finished work for {files}.")

# Thread pool to do work in.
pool = concurrent.futures.ProcessPoolExecutor(max_workers=1)

class Job:
    job_counter = 1

    def __init__(self, files):
        """Setup a job and queue work for files on our thread pool."""
        self._job_number = self.job_counter
        Job.job_counter += 1
        print(f"manager._state.value = {manager._state.value}")
        self._finished_event = manager.Event()

        print(f"Queued job {self.number()}.")
        future = pool.submit(do_work, files)
        future.add_done_callback(lambda f : self._finished_event.set())

    def number(self):
        return self._job_number

    def event(self):
        """Get an event which can be waited on for the job to complete."""
        return self._finished_event

class MyManager(SyncManager):
    pass

MyManager.register("Job", Job)

manager = MyManager(address=("localhost", 16000), authkey=b"qca-authkey")
if len(sys.argv) > 1 and sys.argv[1] == "server":

    manager.start()
    print(f"Manager listening at {manager.address}.")

    while True:
        time.sleep(1)
else:
    manager.connect()
    print(f"Connected to {manager.address}.")

    job = manager.Job(["a", "b", "c"])
    job.event().wait()
    print("Done")

If I run the client I see:

$ ./mp-manager.py
Connected to ('localhost', 16000).
Traceback (most recent call last):
  File "./mp-manager.py", line 54, in <module>
    job = manager.Job(["a", "b", "c"])
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 740, in temp
    token, exp = self._create(typeid, *args, **kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 625, in _create
    id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 91, in dispatch
    raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError: 
---------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 210, in handle_request
    result = func(c, *args, **kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 403, in create
    obj = callable(*args, **kwds)
  File "./mp-manager.py", line 24, in __init__
    self._finished_event = manager.Event()
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 740, in temp
    token, exp = self._create(typeid, *args, **kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 622, in _create
    assert self._state.value == State.STARTED, 'server not yet started'
AssertionError: server not yet started
---------------------------------------------------------------------------

The server output is:

$ ./mp-manager.py server
Manager listening at ('127.0.0.1', 16000).
manager._state.value = 0
toojays
  • 787
  • 6
  • 10

0 Answers0