cannot parallelize (threads) telnet connections with python

Question

I have a code that will connect to several routers, using the telentlib, run some code within those, and finally write the output to a file. It runs smoothly.

However, when I had to access lots of routers (+50) the task consumes a lot of time (the code runs serially, one router at a time). I thought then on implementing threads in order to accelerate the process.

This is pretty much the code (just a snippet of it):

# We import the expect library for python
import telnetlib
import sys
import csv
import time
import threading

# --- Timers
TimeLogin = 10
TelnetWriteTimeout = 1
TelnetReadTimeout = 2
# --- CSV
FileCsv = "/home/lucas/Documents/script/file.csv"
# --- Extras
cr="\n"

# variables
IP = "A.B.C.D"
User = ["user","password"]
CliLogin = "telnet " + IP
Prompt = ["root@host.*]#"]
PromptLogin = ["login:"]
PromptLogout = ["logout"]
PromptPass = ["Password:"]
CliLine = "ls -l"

class MiThread(threading.Thread):  
    def __init__(self,num,datos):  
        threading.Thread.__init__(self)
        self.num = num
        self.datos = datos
        self.systemIP = self.datos[0]
        self.tn = telnetlib.Telnet(IP)
        self.tn.timeout = TimeLogin

        # File declaration
        self.FileOutGen = self.systemIP + "_commands"
        self.FileOutSend = self.systemIP + "_output"
        self.FileOutRx = self.systemIP + "_rx"      
        self.fg = open(self.FileOutGen,"w")
        self.fr = open(self.FileOutRx,"a")
        self.fs = open(self.FileOutSend,"w")

    def run(self):  
        print "Soy el hilo", self.num
        self.telnetLogin()
        self.runLs()
        self.telnetLogout()

    def telnetLogin(self):      
        i=self.tn.expect(PromptLogin)
        print i
        if i:
            writeTelnet(User[0],TelnetWriteTimeout)
            j=self.tn.expect(PromptPass)
            print j
            if j:
                writeTelnet(User[1],TelnetWriteTimeout)

    def telnetLogout(self):
        i=self.tn.expect(Prompt)
        if i:
            writeTelnet("exit",TelnetWriteTimeout)
            j=self.tn.expect(PromptLogout)
            if j:
                print "Logged out OK from SAM"

    def runLs(self):
        writeLog("Prueba de Ls " + self.systemIP)
        self.writeCsv(self.systemIP,1)
        i=self.tn.expect(Prompt,TimeLogin)
        print i
        if i:
            # Prompt
            CliLine = "ls -l "
            writeTelnet(CliLine,TelnetWriteTimeout)

    def writeCsv(self,inText,lastIn):
        lock.acquire(1)
        if lastIn==0:
            fc.write(inText + ",")
        elif lastIn==1:
            fc.write(inText + "\n")
        lock.release()

def writeTelnet(inText, timer):
    tn.write(inText + cr)
    time.sleep(timer)

def writeLog(inText):
    print (inText)
    t.fs.write("\n" + inText)

def printConsole(inText):
    print (inText)

oFile = csv.reader(open(FileCsv,"r"), delimiter=",", quotechar="|")

lock = threading.Lock()
routers = list(oFile)
threads_list = []

for i in range(len(routers)):
    FileOutCsv = "00_log.csv"

    # creating output file
    fc = open(FileOutCsv,"a")

    # running routine
    t = MiThread(i,routers[i])
    threads_list.append(t)
    t.start()

... everything runs nicely, but there is no gain in time since the t.join() will force a thread to complete before running the next one!

The fact of serializing the threads (meaning, using the t.join()) makes me think that some memory spaces are being shared because the problem arises when I do want to parallelize them (commenting out the t.join()).

Is there anything wrong that I'm doing? I can provide more information if needed but I really do not know what is it that I'm doing wrong so far ...

Answer 1

So after some digging, the error was found.

Before, the function writeTelnet() was declared outside the class. Once moved inside of it and properly being referenced by the rest (ie: self.writeTelnet()) everything worked so far as expected.

Here's the snippet of the new code:

class MiThread(threading.Thread):  
    def __init__(self,num,datos):  
        threading.Thread.__init__(self)
        self.num = num
        self.datos = datos
        self.systemIP = self.datos[0]
        self.tn = telnetlib.Telnet(IP)
        self.tn.timeout = TimeLogin

        # File declaration
        self.FileOutGen = self.systemIP + "_commands"
        self.FileOutSend = self.systemIP + "_output"
        self.FileOutRx = self.systemIP + "_rx"      
        self.fg = open(self.FileOutGen,"w")
        self.fr = open(self.FileOutRx,"a")
        self.fs = open(self.FileOutSend,"w")

    [ . . . ]

    def writeTelnet(self,inText, timer):
        self.tn.write(inText + ch_cr)
        time.sleep(timer)

oFile = csv.reader(open(FileCsv,"r"), delimiter=",", quotechar="|")
routers = list(oFile)

for i in range(len(routers)):   
    # creating output file
    fc = open(FileOutCsv,"a")

    # running routine
    t = MiThread(i,routers[i])
    t.start()

And it's now clear (at least to me): Since the different threads need at some point to write into their telnet's connection, they need unambiguously to identify it. The only way that I've found to do it is to include the function inside the class.

Thank you all,

Lucas

Answer 2

From past experience, you need to start all of the threads and have them join after they have all started.

thread_list = []
for i in range(routers):
    t = MiThread(i, routers[i])
    threadlist.append(t)
    t.start()

for i in thread_list:
    i.join()

This will start every thread and then wait for all threads to complete before moving on.

Answer 3

I think you can bunch them up. something like below. (not tried)

count=20
start=1
for i in range(len(routers)):
  if start == count :
    for _myth in range(1,i+1):
        thread_list[_myth].join()
    start+=1
FileOutCsv = "00_log.csv"

# creating output file
fc = open(FileOutCsv,"a")

# running routine
t = MiThread(i,routers[i])
threads_list.append(t)
t.start()
start+=1