Is it possible to run two python scripts parallel and plotting real time?

Question

I have two Python scripts: 1st is responsible for generating data and save them in .csv file and 2nd for plotting this data in real time. I have to run 1st script in console and then run 2nd script in new instance of console. I want to do all this things in one script but I can't do that. I tried to use mutliprocessing but I failed. I add my 2 scripts:

#THIS IS DATA GENERATING SCRIPT
import csv
import random
import time
import numpy as np



steps = 12000
shift = [-1, 0, 1]
start_corr_2D = [0, 0]
start_corr_3D = [0, 0, 0]

path_2D = np.zeros(shape=(steps,2)) 
path_3D = np.zeros(shape=(steps,3)) 

path_2D[0] = start_corr_2D
path_3D[0] = start_corr_3D

fieldnames = ["X", "Y"]


with open('data.csv', 'w') as csv_file:
    csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
    csv_writer.writeheader()

    info = {
        "X": start_corr_2D[0],
        "Y": start_corr_2D[1],
        }

    csv_writer.writerow(info)


for i in range(1, steps):

    with open('data.csv', 'a') as csv_file:
        csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

        shift_2D = [random.choice(shift), random.choice(shift)]
        shift_3D = [random.choice(shift), random.choice(shift), random.choice(shift)]

        next_corr_2D = list(map(sum, zip(start_corr_2D, shift_2D)))
        next_corr_3D = list(map(sum, zip(start_corr_3D, shift_3D)))
        print(next_corr_2D)

        path_2D[i] = next_corr_2D
        path_3D[i] = next_corr_3D

        start_corr_2D = next_corr_2D
        start_corr_3D = next_corr_3D

        info = {
        "X": next_corr_2D[0],
        "Y": next_corr_2D[1],
        }

        csv_writer.writerow(info)

    time.sleep(0.3)

#THIS SCRIPT IS RESPONSIBLE FOR PLOTTING IN REAL TIME
import matplotlib.pyplot as plt
import pandas as pd
from matplotlib.animation import FuncAnimation

j = 0

def animate(i):
    data = pd.read_csv('data.csv')
    xs = data['X']
    ys = data['Y']

    global j
    j += 1
#    print(j)

    plt.cla()
#    plt.scatter(xs, ys, c='red', marker='*')

    if j < 10:
        plt.plot(xs, ys, c='blue', lw=0.6)
    elif j >= 10 and j < 50:
        plt.plot(xs, ys, c='red', lw=0.6)
    else:
        plt.plot(xs, ys, c='green', lw=0.6)

    plt.tight_layout()


ani = FuncAnimation(plt.gcf(), animate, interval=300)

plt.tight_layout()
plt.show()

I am doing this operations right now:

But I want to get results from running only 1 script. I would be grateful for any help.

Answer 1

1.Use GNU parallel:

 parallel -j -8 python ::: "$python_plot_scripts"

2.You can draw multiple fig: multiple_figs_demo

from matplotlib.path import Path
import matplotlib.patches as patches

3.Use: pyhdf

Answer 2

Multiprocessing seems the right way to do what you are trying to achieve. However, using the csv file to communicate between the two process is pretty dangerous. This can be avoided by using a queue between the two processes to send the new data to the plot.

Here's a possible implementation (you could put animate in a different file):

import csv
import random
import time
from multiprocessing import Process, Queue

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

def animate(q):
    xs = [0]
    ys = [1]
    def _animate(i):
        while not q.empty():
            x, y = q.get()
            xs.append(x)
            ys.append(y)
        plt.cla()
        #    plt.scatter(xs, ys, c='red', marker='*')

        if i < 10:
            plt.plot(xs, ys, c='blue', lw=0.6)
        elif 10 <= i < 50:
            plt.plot(xs, ys, c='red', lw=0.6)
        else:
            plt.plot(xs, ys, c='green', lw=0.6)

        plt.tight_layout()

    ani = FuncAnimation(plt.gcf(), _animate, interval=300)

    plt.tight_layout()
    plt.show()



def main():
    steps = 12000
    shift = [-1, 0, 1]
    start_corr_2D = [0, 0]
    start_corr_3D = [0, 0, 0]

    path_2D = np.zeros(shape=(steps,2))
    path_3D = np.zeros(shape=(steps,3))

    path_2D[0] = start_corr_2D
    path_3D[0] = start_corr_3D

    fieldnames = ["X", "Y"]
    data_queue = Queue()
    plot_process = Process(target=animate, args=(data_queue,))
    plot_process.start()


    with open('data.csv', 'w') as csv_file:
        csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
        csv_writer.writeheader()

        info = {
            "X": start_corr_2D[0],
            "Y": start_corr_2D[1],
            }

        csv_writer.writerow(info)


    for i in range(1, steps):

        with open('data.csv', 'a') as csv_file:
            csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

            shift_2D = [random.choice(shift), random.choice(shift)]
            shift_3D = [random.choice(shift), random.choice(shift), random.choice(shift)]

            next_corr_2D = list(map(sum, zip(start_corr_2D, shift_2D)))
            next_corr_3D = list(map(sum, zip(start_corr_3D, shift_3D)))

            path_2D[i] = next_corr_2D
            path_3D[i] = next_corr_3D

            start_corr_2D = next_corr_2D
            start_corr_3D = next_corr_3D

            info = {
            "X": next_corr_2D[0],
            "Y": next_corr_2D[1],
            }
            data_queue.put((next_corr_2D[0], next_corr_2D[1]))
            csv_writer.writerow(info)

        time.sleep(0.3)

    plot_process.join()

if __name__ == '__main__':
    main()