0

I'm trying to change the code (from this example) to use pathos multiprocess instead of multiprocessing. The example code runs just fine. Replacing multiprocessing with multiprocess fails to change the array stored in shared memory and raises an assertion error. Below is the code with my modification.

import ctypes
import logging
import multiprocess as mp

from contextlib import closing

import numpy as np

info = mp.get_logger().info

def main():
    logger = mp.log_to_stderr()
    logger.setLevel(logging.INFO)

    # create shared array
    N, M = 100, 11
    shared_arr = mp.Array(ctypes.c_double, N)
    arr = tonumpyarray(shared_arr)

    # fill with random values
    arr[:] = np.random.uniform(size=N)
    print(arr)
    arr_orig = arr.copy()

    # write to arr from different processes
    with closing(mp.Pool(initializer=init, initargs=(shared_arr,))) as p:
        # many processes access the same slice
        stop_f = N // 10
        p.map_async(f, [slice(stop_f)]*M)

        # many processes access different slices of the same array
        assert M % 2 # odd
        step = N // 10
        p.map_async(g, [slice(i, i + step) for i in range(stop_f, N, step)])
    p.join()
    print(arr)
    assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)

def init(shared_arr_):
    global shared_arr
    shared_arr = shared_arr_ # must be inherited, not passed as an argument

def tonumpyarray(mp_arr):
    return np.frombuffer(mp_arr.get_obj())

def f(i):
    """synchronized."""
    with shared_arr.get_lock(): # synchronize access
        g(i)

def g(i):
    """no synchronization."""
    info("start %s" % (i,))
    arr = tonumpyarray(shared_arr)
    arr[i] = -1 * arr[i]
    info("end   %s" % (i,))

if __name__ == '__main__':
    mp.freeze_support()
    main()

Here is the output of script demonstrating undesired behavior (shared array unchanged).

[INFO/MainProcess] allocating a new mmap of length 4096
[7.53982236e-01 8.91817533e-01 3.42952411e-01 4.81562566e-01
 8.11524911e-01 9.39701276e-01 2.49035240e-01 9.81663723e-01
 8.08235711e-01 1.42681227e-01 3.45497140e-01 8.90562288e-01
 4.35241873e-01 4.60578813e-01 3.46264261e-01 4.93318426e-01
 3.22399114e-01 7.83147144e-01 9.47066783e-02 5.18534236e-01
 8.51086344e-01 6.27441844e-01 1.06383034e-01 2.38705882e-01
 7.97136179e-01 7.86004956e-01 8.66374084e-01 5.17888509e-01
 8.17624705e-01 9.49644794e-01 6.31860201e-01 5.36854341e-01
 3.04143232e-01 3.72530617e-01 2.30401250e-01 8.07846412e-02
 9.07925558e-01 2.83202034e-01 2.29512224e-01 1.10851201e-01
 7.72062029e-01 3.60300517e-01 5.54688922e-04 5.65488687e-01
 5.62561777e-01 6.34814941e-01 5.96339057e-01 9.79598924e-01
 2.09899979e-01 7.15104955e-01 6.55292670e-01 1.68393973e-01
 9.72467624e-01 8.35882816e-01 7.43016913e-01 4.94503316e-01
 4.64440104e-01 9.19258107e-01 8.77918280e-01 3.50650473e-02
 1.78977134e-02 8.37892866e-01 3.97096237e-01 3.98471290e-01
 1.50104963e-01 6.85465476e-01 8.44218457e-01 5.62325961e-01
 1.10385518e-01 3.60357701e-01 3.57917840e-01 8.63808991e-01
 8.39141018e-01 1.79879196e-01 8.02565416e-01 6.78710916e-01
 9.40750906e-01 6.95072638e-01 4.07945520e-01 8.04220531e-01
 9.36671866e-01 8.04467890e-01 2.71010169e-01 6.12931552e-02
 9.13496687e-01 9.94252838e-03 6.29349539e-01 7.82656394e-02
 6.58311027e-01 9.18628747e-01 9.21899148e-01 6.34954946e-03
 9.87928873e-01 2.35952899e-01 6.93470029e-01 9.41543773e-01
 2.27509027e-01 7.76581113e-01 5.76203661e-01 7.70004310e-01]
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[7.53982236e-01 8.91817533e-01 3.42952411e-01 4.81562566e-01
 8.11524911e-01 9.39701276e-01 2.49035240e-01 9.81663723e-01
 8.08235711e-01 1.42681227e-01 3.45497140e-01 8.90562288e-01
 4.35241873e-01 4.60578813e-01 3.46264261e-01 4.93318426e-01
 3.22399114e-01 7.83147144e-01 9.47066783e-02 5.18534236e-01
 8.51086344e-01 6.27441844e-01 1.06383034e-01 2.38705882e-01
 7.97136179e-01 7.86004956e-01 8.66374084e-01 5.17888509e-01
 8.17624705e-01 9.49644794e-01 6.31860201e-01 5.36854341e-01
 3.04143232e-01 3.72530617e-01 2.30401250e-01 8.07846412e-02
 9.07925558e-01 2.83202034e-01 2.29512224e-01 1.10851201e-01
 7.72062029e-01 3.60300517e-01 5.54688922e-04 5.65488687e-01
 5.62561777e-01 6.34814941e-01 5.96339057e-01 9.79598924e-01
 2.09899979e-01 7.15104955e-01 6.55292670e-01 1.68393973e-01
 9.72467624e-01 8.35882816e-01 7.43016913e-01 4.94503316e-01
 4.64440104e-01 9.19258107e-01 8.77918280e-01 3.50650473e-02
 1.78977134e-02 8.37892866e-01 3.97096237e-01 3.98471290e-01
 1.50104963e-01 6.85465476e-01 8.44218457e-01 5.62325961e-01
 1.10385518e-01 3.60357701e-01 3.57917840e-01 8.63808991e-01
 8.39141018e-01 1.79879196e-01 8.02565416e-01 6.78710916e-01
 9.40750906e-01 6.95072638e-01 4.07945520e-01 8.04220531e-01
 9.36671866e-01 8.04467890e-01 2.71010169e-01 6.12931552e-02
 9.13496687e-01 9.94252838e-03 6.29349539e-01 7.82656394e-02
 6.58311027e-01 9.18628747e-01 9.21899148e-01 6.34954946e-03
 9.87928873e-01 2.35952899e-01 6.93470029e-01 9.41543773e-01
 2.27509027e-01 7.76581113e-01 5.76203661e-01 7.70004310e-01]

Here is the full traceback.

Traceback (most recent call last):
  File "C:\Users\Fernando\AppData\Local\Continuum\anaconda3\envs\fountaintx\lib\site-packages\IPython\core\interactiveshell.py", line 3326, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-4f9235b548aa>", line 1, in <module>
    runfile('C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py', wdir='C:/Users/Fernando/.PyCharm2019.2/config/scratches')
  File "C:\Program Files\JetBrains\PyCharm Professional Edition with Anaconda plugin 2019.2.4\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm Professional Edition with Anaconda plugin 2019.2.4\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py", line 60, in <module>
    main()
  File "C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py", line 37, in main
    assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)
AssertionError
Fernando Martinez
  • 1
  • 1
  • Please [edit] to post your actual attempt at "_Replacing multiprocessing with multiprocess fails to change the array stored in shared memory_, and the full/complete error Traceback you get with your own code. – Gino Mempin Dec 19 '19 at 03:55
  • 1
    Edited with my modification and the error traceback. – Fernando Martinez Dec 19 '19 at 20:41

0 Answers0