How to Update progress bar as well as Download File using Multi-Threading

Question

I'm trying to download files using the Request module using Multi threading and simultaneously need to Update a Progress bar as well. But while doing so, whenever I press submit button, my GUI stops working and in the background the files start to download and only after Downloading, my Progress bar updates and GUI continues to work Normal.

I have tried to apply multi-threading to the GUI as well so that the progress of downloaded files will be displayed, but it is only updating the progress after the whole Process gets over.

from datetime import datetime
from threading import Thread
import dask.dataframe as dd
from pubsub import pub
from queue import Queue
import pandas as pd
import requests
import shutil
import sqlite3
import time
import os
import csv
import wx


class MainPage(wx.Dialog):
    def __init__(self):
        wx.Dialog.__init__(self, None, title="File Selector")

        user_sizer = wx.BoxSizer(wx.HORIZONTAL)
        user_label = wx.StaticText(self, label='Number of Threads:')
        user_sizer.Add(user_label, 0, wx.ALL | wx.CENTER, 5)
        self.threads = wx.TextCtrl(self)
        user_sizer.Add(self.threads, 5, wx.ALL, 5)

        button_sizer = wx.BoxSizer(wx.HORIZONTAL)
        file_label = wx.StaticText(self, label='Select File:')
        button_sizer.Add(file_label, 0, wx.ALL | wx.CENTER, 5)
        self.select = wx.TextCtrl(self)
        button_sizer.Add(self.select, 0, wx.ALL | wx.CENTER, 10)
        file_select = wx.Button(self, label='Browse')
        file_select.Bind(wx.EVT_BUTTON, self.file)
        button_sizer.Add(file_select, 0, wx.ALL | wx.CENTER, 5)

        link_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
        link_choice_label = wx.StaticText(self, label='Link Column:')
        link_choice_sizer.Add(link_choice_label, 0, wx.ALL | wx.CENTER, 5)
        self.link_choice = wx.TextCtrl(self)
        link_choice_sizer.Add(self.link_choice, 0, wx.ALL | wx.CENTER, 5)

        file_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
        file_choice_label = wx.StaticText(self, label='File column:')
        file_choice_sizer.Add(file_choice_label, 0, wx.ALL | wx.CENTER, 5)
        self.file_choice = wx.TextCtrl(self)
        file_choice_sizer.Add(self.file_choice, 0, wx.ALL | wx.CENTER, 5)

        submit_sizer = wx.BoxSizer(wx.HORIZONTAL)
        submit_button = wx.Button(self, label='Submit')
        submit_button.Bind(wx.EVT_BUTTON, self.process)
        submit_sizer.Add(submit_button, 0, wx.TOP, 20)

        main_box = wx.BoxSizer(wx.VERTICAL)
        main_box.Add(user_sizer, 0, wx.ALL, 3)
        main_box.Add(button_sizer, 0, wx.LEFT, 43)
        main_box.Add(link_choice_sizer, 0, wx.LEFT, 39)
        main_box.Add(file_choice_sizer, 0, wx.LEFT, 43)
        main_box.Add(submit_sizer, 0, wx.LEFT, 135)

        self.user_sizer = user_sizer
        self.button_sizer = button_sizer
        self.SetSizer(main_box)
        self.Center()


    def file(self, event):
        dialog = wx.FileDialog(None, "Choose a File:", wildcard="CSV files (*.csv)|*.csv", style=wx.DD_DEFAULT_STYLE | wx.DD_NEW_DIR_BUTTON)
        if dialog.ShowModal() == wx.ID_OK:
            print(dialog.GetPath())
            path = dialog.GetPath()
            self.select.ChangeValue(path)
            print(path)
        dialog.Destroy()


    def process(self, event):
        selected_path = self.select.GetValue()
        thread = self.threads.GetValue()
        link = self.link_choice.GetValue()
        file = self.file_choice.GetValue()
        print(thread)
        print(link)
        print(file)
        if (thread != ""):
            if (selected_path != "") :
                if (link != "") :
                    if (file != "") :
                        if (thread.isdigit()) and (link.isdigit()) and (file.isdigit()) :
                            if (int(link) != int(file)):
                                if selected_path.endswith('.csv'):
                                    self.threads.Clear()
                                    self.select.Clear()
                                    self.link_choice.Clear()
                                    self.file_choice.Clear()
                                    print("out")
                                    print(file)
                                    wx.LogMessage("File processed Sucessfully")
                                    # self.csv_processing(selected_path, thread, link, file)
                                    self.progress_show(selected_path, thread, link, file)
                                else:
                                    wx.LogError("File Extension Not Valid")
                            else:
                                wx.LogError("Both Link Column and File Column Can't be Same")
                        else:
                            wx.LogError("Please Enter Valid Number")
                    else:
                        wx.LogError("Please Enter Number in File Column")
                else:
                    wx.LogError("Please Enter Number in Link Column")
            else:
                wx.LogError("Please Select File")
        else:
            wx.LogError("Please Enter Number in Thread")

    def progress_show(self, selected_path, thread, link, file):
        pub.sendMessage('frameListener', message='show', selected_path = selected_path, thread = thread, link = link, file = file)

class Download_file():
    def download_image(self, url, file_path, data_load):
        print("In Class")
        try:
            wx.CallAfter(self.new_update, data_load)
            response = requests.get(url)
            if response.status_code == 200:
                if (response.raw) != None:
                    with open(file_path, 'wb') as out_file:
                        out_file.write(response.content)
                    del response
            else:
                print("not found")
        except requests.exceptions.RequestException as e:
            print(e)



class DownloadWorker(Thread):
    def __init__(self, queue):
        Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            print("In queue")
            url, file_path, data_load = self.queue.get()
            download = Download_file
            try:
                # pub.sendMessage('progress', message='show', data_load=data_load)
                # print((url))
                download.download_image(self, url, file_path, data_load)
                # wx.CallAfter(self.new_update, data_load)
                time.sleep(1)
            finally:
                self.queue.task_done()
                print("Downloaded")

    def new_update(self, data_load):
        print("upload")
        print(data_load)
        pub.sendMessage('progress', message='show', data_load=data_load)


class Progressbar(wx.Dialog):
    def __init__(self, selected_path, thread, link, file):
        wx.Dialog.__init__(self, None, title='Progess Bar',  size=(300,100))
        print("INnnn")
        progress_sizer = wx.BoxSizer(wx.HORIZONTAL)
        self.progress_bar = wx.Gauge(self, range=100, size=(250,25), style= wx.GA_HORIZONTAL)
        progress_sizer.Add(self.progress_bar, 0, wx.CENTER)
        progress_box = wx.BoxSizer(wx.VERTICAL)
        progress_box.Add(progress_sizer, wx.ALIGN_CENTRE, 25)
        print(selected_path, thread, link, file)
        self.SetSizer(progress_box)
        self.Center()
        self.Show(True)
        self.call_progress(selected_path, thread, link, file)
        pub.subscribe(self.progressListener, 'progress')
        self.Destroy()

    def call_progress(self, selected_path, thread, link, file):
        start = datetime.now()
        count = 1
        thread = int(thread)
        dask_data = dd.read_csv(selected_path, sep=',', error_bad_lines=False, quoting=csv.QUOTE_NONE, dtype="unicode")
        data_length = len(dask_data)
        data = pd.read_csv(selected_path, delimiter=',', chunksize=thread)
        count = 1
        date = str(datetime.date(datetime.now()))
        mydir = os.getcwd()
        image_dir = os.path.join(mydir, "Downloaded_Images")
        if not os.path.exists(image_dir):
            os.mkdir(image_dir)
        date_dir = os.path.join(image_dir, date)
        if not os.path.exists(date_dir):
            os.mkdir(date_dir)
        link_name =  'Link_column' + '___' + str(link)
        link_column = os.path.join(date_dir, link_name)
        if not os.path.exists(link_column):
            os.mkdir(link_column)
        queue = Queue()
        for x in range(thread):
            worker = DownloadWorker(queue)
            worker.daemon = True
            worker.start()
        thread_length = thread
        for df in data:
            data_load = round((thread_length / data_length) * 100)
            if data_load > 100:
                data_load = 100
            for i in range(thread):
                try:
                    print(df.iloc[i][int(link) - 1])
                    url = (df.iloc[i][int(link) - 1])
                    print(df.iloc[i][int(file) - 1])
                    file_name = (df.iloc[i][int(file) - 1])
                    print(count)
                    # print(data_load)
                    file_path = os.path.join(link_column, file_name)
                    queue.put((url, file_path, data_load))
                    count += 1
                except IndexError:
                    pass
            print("-------------------------")

            queue.join()
            thread_length += thread

            # data_load = round((thread / data_length) * 100)
            # if data_load > 100:
            #     data_load = 100
            # self.progress_bar.SetValue(data_load)

        # queue.join()
        print((datetime.now()) - start)

    def progressListener(self, message, data_load):
        print("listener")
        self.progress_bar.SetValue(data_load)

class MyPanel(wx.Panel):
    def __init(self, parent):
        wx.Panel.__init__(self, parent)

class MainFrame(wx.Frame):
    def __init__(self):
        wx.Frame.__init__(self, None)
        MyPanel(self)
        pub.subscribe(self.myListener, 'frameListener')
        self.Center()
        dlg = MainPage()
        dlg.ShowModal()

    def myListener(self, message, selected_path, thread, link, file):
        print(selected_path, thread, link, file)
        # self.center()
        dlg = Progressbar(selected_path = selected_path, thread = thread, link = link, file = file)
        dlg.ShowModal()

def myapp():
    app = wx.App(False)
    MainFrame()
    # app.MainLoop()


if __name__ == '__main__':
    myapp()

Expected result is that the File Should be Downloaded and at the same time the Progress Bar should be Updated as well.

See `urlretrieve` https://stackoverflow.com/questions/13881092/download-progressbar-for-python-3 — Rolf of Saxony, Jul 09 '19 at 10:31
@Rolf of Saxony Thanks for Replying but above link is not helping me solve the problem. — Shubham, Jul 10 '19 at 04:26
You have to download the file in chunks not in one go. A way round this is to use `urlretrieve` in the `urllib` module, which allows you to declare a callable (`reporthook`) that will be called once on establishment of the network connection and once after each block read thereafter. The callable will be passed three arguments; a count of blocks transferred so far, a block size in bytes, and the total size of the file. Once you have calculated the progress, update the progress gauge, using either a `pubsub` or a `wx` event. — Rolf of Saxony, Jul 10 '19 at 17:01

score 1 · Answer 1 · answered Jul 10 '19 at 14:18

I wrote a tutorial on this topic a while back.

Here's the code:

import requests
import os
import wx
import wx.lib.scrolledpanel as scrolled

from threading import Thread
from wx.lib.pubsub import pub


class DownloadThread(Thread):
    """Downloading thread"""

    def __init__(self, gnum, url, fsize):
        """Constructor"""
        Thread.__init__(self)
        self.fsize = fsize
        self.gnum = gnum
        self.url = url
        self.start()

    def run(self):
        """
        Run the worker thread
        """
        local_fname = os.path.basename(self.url)
        count = 1
        while True:
            if os.path.exists(local_fname):
                tmp, ext = os.path.splitext(local_fname)
                cnt = "(%s)" % count
                local_fname = tmp + cnt + ext
                count += 1
            else:
                break
        req = requests.get(self.url, stream=True)
        total_size = 0
        print(local_fname)
        with open(local_fname, "wb") as fh:
            for byte in req.iter_content(chunk_size=1024):
                if byte:
                    fh.write(byte)
                    fh.flush()
                total_size += len(byte)
                if total_size < self.fsize:
                    wx.CallAfter(pub.sendMessage,
                                 "update_%s" % self.gnum,
                                 msg=total_size)
        wx.CallAfter(pub.sendMessage,
                     "update_%s" % self.gnum,
                     msg=self.fsize)



class MyGauge(wx.Gauge):
    """"""

    def __init__(self, parent, range, num):
        """Constructor"""
        wx.Gauge.__init__(self, parent, range=range)

        pub.subscribe(self.updateProgress, "update_%s" % num)


    def updateProgress(self, msg):
        """"""
        self.SetValue(msg)


class MyPanel(scrolled.ScrolledPanel):
    """"""

    def __init__(self, parent):
        """Constructor"""
        scrolled.ScrolledPanel.__init__(self, parent)

        self.data = []
        self.download_number = 1

        # create the sizers
        self.main_sizer = wx.BoxSizer(wx.VERTICAL)
        dl_sizer = wx.BoxSizer(wx.HORIZONTAL)

        # create the widgets
        lbl = wx.StaticText(self, label="Download URL:")
        self.dl_txt = wx.TextCtrl(self)
        btn = wx.Button(self, label="Download")
        btn.Bind(wx.EVT_BUTTON, self.onDownload)

        # layout the widgets
        dl_sizer.Add(lbl, 0, wx.ALL|wx.CENTER, 5)
        dl_sizer.Add(self.dl_txt, 1, wx.EXPAND|wx.ALL, 5)
        dl_sizer.Add(btn, 0, wx.ALL, 5)
        self.main_sizer.Add(dl_sizer, 0, wx.EXPAND)

        self.SetSizer(self.main_sizer)
        self.SetAutoLayout(1)
        self.SetupScrolling()

    def onDownload(self, event):
        """
        Update display with downloading gauges
        """
        url = self.dl_txt.GetValue()
        try:
            header = requests.head(url)
            fsize = int(header.headers["content-length"]) / 1024

            sizer = wx.BoxSizer(wx.HORIZONTAL)
            fname = os.path.basename(url)
            lbl = wx.StaticText(self, label="Downloading %s" % fname)
            gauge = MyGauge(self, fsize, self.download_number)

            sizer.Add(lbl, 0, wx.ALL|wx.CENTER, 5)
            sizer.Add(gauge, 0, wx.ALL|wx.EXPAND, 5)
            self.main_sizer.Add(sizer, 0, wx.EXPAND)

            self.Layout()

            # start thread
            DownloadThread(self.download_number, url, fsize)
            self.dl_txt.SetValue("")
            self.download_number += 1
        except Exception as e:
            print("Error: ", e)

class DownloaderFrame(wx.Frame):
    """"""

    def __init__(self):
        """Constructor"""
        wx.Frame.__init__(self, None, title="Downloader", size=(800, 400))
        panel = MyPanel(self)
        self.Show()

if __name__ == "__main__":
    app = wx.App(False)
    frame = DownloaderFrame()
    app.MainLoop()

Thanks for your Reply But I need to Download multiple files and for that also i need to use multi threading so how can i manage both file multiple file downloading and updating Progress bar. — Shubham, Jul 11 '19 at 03:24
This code supports doing multiple files. You just add the URL, hit Download and then add another URL. New Gauges will get added and updated by separate threads — Mike Driscoll, Jul 11 '19 at 20:26

score 1 · Accepted Answer · answered Jul 10 '19 at 18:41

With apologies to Mike for highjacking his code for a skeleton, here is a rough version using urlretrieve and a wxPython event to do the updating.

import os
import wx
import time
from threading import Thread
from urllib.request import FancyURLopener
from urllib.request import urlopen
import wx.lib.newevent

urlretrieve = FancyURLopener().retrieve
progress_event, EVT_PROGRESS_EVENT = wx.lib.newevent.NewEvent()

class DownloadThread(Thread):

    def __init__(self, parent, url, fsize):
        """Constructor"""
        Thread.__init__(self)
        self.fsize = fsize
        self.url = url
        self.parent = parent
        self.start()

    def run(self):
        local_fname = os.path.basename(self.url)
        urlretrieve(self.url, local_fname, self.reporthook)

    def reporthook(self,blocknum, blocksize, totalsize):
        readsofar = blocknum * blocksize
        if totalsize > 0:
            percent = int((readsofar / totalsize) * 100)
        else: # total size is unknown
            percent = 0
        evt = progress_event(count=percent)
        #Send back current count for the progress bar
        try:
            wx.PostEvent(self.parent, evt)
        except: # The parent frame has probably been destroyed
            pass

class MyPanel(wx.Panel):
    def __init__(self, parent):
        """Constructor"""
        wx.Panel.__init__(self, parent)

        self.lbl = wx.StaticText(self, label="Download URL:")
        self.dl_txt = wx.TextCtrl(self)
        btn = wx.Button(self, label="Download")
        self.gauge = wx.Gauge(self, -1, range=100)

        self.main_sizer = wx.BoxSizer(wx.VERTICAL)
        dl_sizer = wx.BoxSizer(wx.HORIZONTAL)

        dl_sizer.Add(self.dl_txt, 1, wx.EXPAND|wx.ALL, 5)
        dl_sizer.Add(btn, 0, wx.ALL, 5)

        self.main_sizer.Add(dl_sizer, 0, wx.EXPAND)
        self.main_sizer.Add(self.lbl, 0, wx.ALL|wx.CENTER, 5)
        self.main_sizer.Add(self.gauge, 0, wx.ALL|wx.EXPAND, 5)

        self.SetSizer(self.main_sizer)

        #Bind to the progress event issued by the thread
        self.Bind(EVT_PROGRESS_EVENT, self.OnProgress)
        btn.Bind(wx.EVT_BUTTON, self.OnDownload)

    def OnDownload(self, event):
        url = self.dl_txt.GetValue()
        self.lbl.SetLabel(os.path.basename(url))
        try:
            header = urlopen(url)
            fsize = int(header.headers['Content-Length'])
            # start thread
            d_thread = DownloadThread(self, url, fsize)
            while d_thread.isAlive():
                time.sleep(0.1)
                wx.GetApp().Yield()
                continue
            # Finished re-set
            self.dl_txt.SetValue("")
            self.lbl.SetLabel("Download URL:")
            self.gauge.SetValue(0)

        except Exception as e:
            print ("Error: ", e)

    def OnProgress(self, event):
        """"""
        self.gauge.SetValue(event.count)

class DownloaderFrame(wx.Frame):

    def __init__(self):
        """Constructor"""
        wx.Frame.__init__(self, None, title="Downloader", size=(800, 400))
        panel = MyPanel(self)
        self.Show()

if __name__ == "__main__":
    app = wx.App(False)
    frame = DownloaderFrame()
    app.MainLoop()

Thanks for your Reply But i need to Download multiple files and file size is also not that large and i am using multi-threading to download that files but while downloading those files my progress bar gets stuck. — Shubham, Jul 11 '19 at 03:25
@Shubham Find somewhere appropriate in the process to call `wx.GetApp().Yield()` I assume from what you say, that would be after each file has been downloaded. `Yield` releases control momentarily back to the `MainLoop` allowing your main gui to be a) responsive b) perform updates i.e. update the gauge. In the code above, you will see that I am doing this in a `while` loop when the `thread` is running. — Rolf of Saxony, Jul 11 '19 at 07:06
Thank you for your Reply your solution has helped me to solve this problem which i was facing from few days Thank you very much. I have read your other answers as well on wxpython topic on stack overflow you are really helpful Thank you once again for taking precious time and helping me. — Shubham, Jul 11 '19 at 08:49

How to Update progress bar as well as Download File using Multi-Threading

2 Answers2

Linked