How to run script in multithreading or multiprocessing

Question

This scripts takes 2 sec to complete but how to run it in many threads and complete in 50 ms

import urllib2                                                                                                                              
from threading import  Thread                                                                                       
def btl_test(url):                                                                                                                                                                                                          
    page = urllib2.urlopen(url) 
    print page                                                                                                            


url = ["http://google.com","http://example.com","http://yahoo.com","http://linkedin.com","http://orkut.com","http://quora.com","http://facebook.com","http://myspace.com","http://gmail.com","http://nltk.org","http://cyber.com"]
for i in url:                                                                                                                        
    t = Thread(target = btl_test,args=(i,))                                                                                                           
    t.start()

How to put results in order as well?

nehem · Accepted Answer · 2015-10-07T23:13:46.453

from contextlib import closing # http://stackoverflow.com/a/25968716/968442
from multiprocessing.pool import Pool

with closing(Pool(len(url))) as pool:
    pool.map(btl_test, url)

Should be the handy snippet. Regarding order you can assign a mapping using a tuple and print them accordingly.

Update:

As per this blog pool.map will return the output with the order preserved. Here is code which prints the list of tuples in (url, html_content) format without changing the order

urls = ["http://google.com","http://example.com","http://yahoo.com","http://linkedin.com","http://orkut.com","http://quora.com","http://facebook.com","http://myspace.com","http://gmail.com","http://nltk.org","http://cyber.com"]

def btl_test(url):
    import urllib2
    return url, urllib2.urlopen(url).read()

from contextlib import closing # http://stackoverflow.com/a/25968716/968442
from multiprocessing.pool import Pool

with closing(Pool(len(urls))) as pool:
    result = pool.map(btl_test, urls)

print result

could you please map this to get in order because i could not understand? — Mounarajan, Oct 07 '15 at 14:00

score 1 · Answer 2 · answered Oct 07 '15 at 14:43

Try to use Queue() and enumerate to store order.

import threading
import requests
import Queue

class UrlReader(threading.Thread):
    def __init__(self, queue, output):
        super(UrlReader, self).__init__()
        self.setDaemon = True
        self.queue = queue
        self.output = output

    def run(self):
        while True:
            try:
                target = self.queue.get(block=False)
                data = requests.get(target[1])
                print data.status_code
                if data.status_code == 200:
                    self.queue.task_done()
                    self.output.put((data.url, target[0]), block=False)
                else:
                    self.queue.task_done()
                    self.queue.put(target)
            except Queue.Empty:
                break
            except requests.exceptions.ConnectionError:
                self.queue.task_done()
                self.queue.put(target)


def load(urlrange, num_threads):
    mainqueue = Queue.Queue()
    outq = Queue.Queue()
    mythreads = []

    for url in urlrange:
        mainqueue.put(url)

    for j in xrange(num_threads):
        mythreads.append(UrlReader(mainqueue, outq))
        mythreads[-1].start()

    mainqueue.join()
    for j in xrange(num_threads):
        mythreads.append(UrlReader(mainqueue, outq))
        mythreads[j].join()
    return list(outq.__dict__['queue'])

urls = ["http://google.com","http://example.com","http://yahoo.com","http://linkedin.com","http://orkut.com","http://quora.com","http://facebook.com","http://myspace.com","http://gmail.com","http://nltk.org","http://cyber.com"]

print load(enumerate(urls), 10)

>>> [(6, 'http://facebook.com'), (9, 'http://nltk.org'), (0, 'http://google.com'), (1, 'http://example.com'), (2, 'http://yahoo.com'), (3, 'http://linkedin.com'), (4, 'http://orkut.com'), (5, 'http://quora.com'), (7, 'http://myspace.com'), (8, 'http://gmail.com'), (10, 'http://cyber.com')]

Hi could you please help me to understand this script by adding documentation(Comments) Please? — Mounarajan, Oct 08 '15 at 07:45

score 1 · Answer 3 · answered Oct 08 '15 at 08:16

This works

from urlparse import urlparse
from multiprocessing.pool import Pool 
import re
import urllib2 

def btl_test(url):                                                                                                                                                                                                          
    page = urllib2.urlopen(url).read()
    if (re.findall(r'<title>(.*?)<\/title>',page)):
        page1 =  (re.findall(r'<title>(.*?)<\/title>',page)[0])
        print page1

url = ["http://google.com","http://example.com","http://yahoo.com","http://linkedin.com","http://facebook.com","http://orkut.com","http://oosing.com","http://pinterets.com","http://orkut.com","http://quora.com","http://facebook.com","http://myspace.com","http://gmail.com","http://nltk.org","http://cyber.com"]


#for i in url:
#   print btl_test(i)
nprocs = 2 # nprocs is the number of processes to run
ParsePool = Pool(nprocs)
ParsePool.map(btl_test,url)
#ParsedURLS = ParsePool.map(btl_test,url)
#print ParsedURLS

Helps a lot

How to run script in multithreading or multiprocessing

3 Answers3