如何使用多线程更新进度条以及下载文件

Question

我正在尝试使用多线程的请求模块下载文件，同时还需要更新进度条。但是在这样做的同时，每当我按下提交按钮时，我的 GUI 就会停止工作，并且在后台开始下载文件，并且只有在下载之后，我的进度条才会更新，并且 GUI 会继续正常工作。

我也尝试将多线程应用到GUI，以便显示下载文件的进度，但它只是在整个过程结束后才更新进度。

from datetime import datetime
from threading import Thread
import dask.dataframe as dd
from pubsub import pub
from queue import Queue
import pandas as pd
import requests
import shutil
import sqlite3
import time
import os
import csv
import wx


class MainPage(wx.Dialog):
    def __init__(self):
        wx.Dialog.__init__(self, None, title="File Selector")

        user_sizer = wx.BoxSizer(wx.HORIZONTAL)
        user_label = wx.StaticText(self, label='Number of Threads:')
        user_sizer.Add(user_label, 0, wx.ALL | wx.CENTER, 5)
        self.threads = wx.TextCtrl(self)
        user_sizer.Add(self.threads, 5, wx.ALL, 5)

        button_sizer = wx.BoxSizer(wx.HORIZONTAL)
        file_label = wx.StaticText(self, label='Select File:')
        button_sizer.Add(file_label, 0, wx.ALL | wx.CENTER, 5)
        self.select = wx.TextCtrl(self)
        button_sizer.Add(self.select, 0, wx.ALL | wx.CENTER, 10)
        file_select = wx.Button(self, label='Browse')
        file_select.Bind(wx.EVT_BUTTON, self.file)
        button_sizer.Add(file_select, 0, wx.ALL | wx.CENTER, 5)

        link_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
        link_choice_label = wx.StaticText(self, label='Link Column:')
        link_choice_sizer.Add(link_choice_label, 0, wx.ALL | wx.CENTER, 5)
        self.link_choice = wx.TextCtrl(self)
        link_choice_sizer.Add(self.link_choice, 0, wx.ALL | wx.CENTER, 5)

        file_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
        file_choice_label = wx.StaticText(self, label='File column:')
        file_choice_sizer.Add(file_choice_label, 0, wx.ALL | wx.CENTER, 5)
        self.file_choice = wx.TextCtrl(self)
        file_choice_sizer.Add(self.file_choice, 0, wx.ALL | wx.CENTER, 5)

        submit_sizer = wx.BoxSizer(wx.HORIZONTAL)
        submit_button = wx.Button(self, label='Submit')
        submit_button.Bind(wx.EVT_BUTTON, self.process)
        submit_sizer.Add(submit_button, 0, wx.TOP, 20)

        main_box = wx.BoxSizer(wx.VERTICAL)
        main_box.Add(user_sizer, 0, wx.ALL, 3)
        main_box.Add(button_sizer, 0, wx.LEFT, 43)
        main_box.Add(link_choice_sizer, 0, wx.LEFT, 39)
        main_box.Add(file_choice_sizer, 0, wx.LEFT, 43)
        main_box.Add(submit_sizer, 0, wx.LEFT, 135)

        self.user_sizer = user_sizer
        self.button_sizer = button_sizer
        self.SetSizer(main_box)
        self.Center()


    def file(self, event):
        dialog = wx.FileDialog(None, "Choose a File:", wildcard="CSV files (*.csv)|*.csv", style=wx.DD_DEFAULT_STYLE | wx.DD_NEW_DIR_BUTTON)
        if dialog.ShowModal() == wx.ID_OK:
            print(dialog.GetPath())
            path = dialog.GetPath()
            self.select.ChangeValue(path)
            print(path)
        dialog.Destroy()


    def process(self, event):
        selected_path = self.select.GetValue()
        thread = self.threads.GetValue()
        link = self.link_choice.GetValue()
        file = self.file_choice.GetValue()
        print(thread)
        print(link)
        print(file)
        if (thread != ""):
            if (selected_path != "") :
                if (link != "") :
                    if (file != "") :
                        if (thread.isdigit()) and (link.isdigit()) and (file.isdigit()) :
                            if (int(link) != int(file)):
                                if selected_path.endswith('.csv'):
                                    self.threads.Clear()
                                    self.select.Clear()
                                    self.link_choice.Clear()
                                    self.file_choice.Clear()
                                    print("out")
                                    print(file)
                                    wx.LogMessage("File processed Sucessfully")
                                    # self.csv_processing(selected_path, thread, link, file)
                                    self.progress_show(selected_path, thread, link, file)
                                else:
                                    wx.LogError("File Extension Not Valid")
                            else:
                                wx.LogError("Both Link Column and File Column Can't be Same")
                        else:
                            wx.LogError("Please Enter Valid Number")
                    else:
                        wx.LogError("Please Enter Number in File Column")
                else:
                    wx.LogError("Please Enter Number in Link Column")
            else:
                wx.LogError("Please Select File")
        else:
            wx.LogError("Please Enter Number in Thread")

    def progress_show(self, selected_path, thread, link, file):
        pub.sendMessage('frameListener', message='show', selected_path = selected_path, thread = thread, link = link, file = file)

class Download_file():
    def download_image(self, url, file_path, data_load):
        print("In Class")
        try:
            wx.CallAfter(self.new_update, data_load)
            response = requests.get(url)
            if response.status_code == 200:
                if (response.raw) != None:
                    with open(file_path, 'wb') as out_file:
                        out_file.write(response.content)
                    del response
            else:
                print("not found")
        except requests.exceptions.RequestException as e:
            print(e)



class DownloadWorker(Thread):
    def __init__(self, queue):
        Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            print("In queue")
            url, file_path, data_load = self.queue.get()
            download = Download_file
            try:
                # pub.sendMessage('progress', message='show', data_load=data_load)
                # print((url))
                download.download_image(self, url, file_path, data_load)
                # wx.CallAfter(self.new_update, data_load)
                time.sleep(1)
            finally:
                self.queue.task_done()
                print("Downloaded")

    def new_update(self, data_load):
        print("upload")
        print(data_load)
        pub.sendMessage('progress', message='show', data_load=data_load)


class Progressbar(wx.Dialog):
    def __init__(self, selected_path, thread, link, file):
        wx.Dialog.__init__(self, None, title='Progess Bar',  size=(300,100))
        print("INnnn")
        progress_sizer = wx.BoxSizer(wx.HORIZONTAL)
        self.progress_bar = wx.Gauge(self, range=100, size=(250,25), style= wx.GA_HORIZONTAL)
        progress_sizer.Add(self.progress_bar, 0, wx.CENTER)
        progress_box = wx.BoxSizer(wx.VERTICAL)
        progress_box.Add(progress_sizer, wx.ALIGN_CENTRE, 25)
        print(selected_path, thread, link, file)
        self.SetSizer(progress_box)
        self.Center()
        self.Show(True)
        self.call_progress(selected_path, thread, link, file)
        pub.subscribe(self.progressListener, 'progress')
        self.Destroy()

    def call_progress(self, selected_path, thread, link, file):
        start = datetime.now()
        count = 1
        thread = int(thread)
        dask_data = dd.read_csv(selected_path, sep=',', error_bad_lines=False, quoting=csv.QUOTE_NONE, dtype="unicode")
        data_length = len(dask_data)
        data = pd.read_csv(selected_path, delimiter=',', chunksize=thread)
        count = 1
        date = str(datetime.date(datetime.now()))
        mydir = os.getcwd()
        image_dir = os.path.join(mydir, "Downloaded_Images")
        if not os.path.exists(image_dir):
            os.mkdir(image_dir)
        date_dir = os.path.join(image_dir, date)
        if not os.path.exists(date_dir):
            os.mkdir(date_dir)
        link_name =  'Link_column' + '___' + str(link)
        link_column = os.path.join(date_dir, link_name)
        if not os.path.exists(link_column):
            os.mkdir(link_column)
        queue = Queue()
        for x in range(thread):
            worker = DownloadWorker(queue)
            worker.daemon = True
            worker.start()
        thread_length = thread
        for df in data:
            data_load = round((thread_length / data_length) * 100)
            if data_load > 100:
                data_load = 100
            for i in range(thread):
                try:
                    print(df.iloc[i][int(link) - 1])
                    url = (df.iloc[i][int(link) - 1])
                    print(df.iloc[i][int(file) - 1])
                    file_name = (df.iloc[i][int(file) - 1])
                    print(count)
                    # print(data_load)
                    file_path = os.path.join(link_column, file_name)
                    queue.put((url, file_path, data_load))
                    count += 1
                except IndexError:
                    pass
            print("-------------------------")

            queue.join()
            thread_length += thread

            # data_load = round((thread / data_length) * 100)
            # if data_load > 100:
            #     data_load = 100
            # self.progress_bar.SetValue(data_load)

        # queue.join()
        print((datetime.now()) - start)

    def progressListener(self, message, data_load):
        print("listener")
        self.progress_bar.SetValue(data_load)

class MyPanel(wx.Panel):
    def __init(self, parent):
        wx.Panel.__init__(self, parent)

class MainFrame(wx.Frame):
    def __init__(self):
        wx.Frame.__init__(self, None)
        MyPanel(self)
        pub.subscribe(self.myListener, 'frameListener')
        self.Center()
        dlg = MainPage()
        dlg.ShowModal()

    def myListener(self, message, selected_path, thread, link, file):
        print(selected_path, thread, link, file)
        # self.center()
        dlg = Progressbar(selected_path = selected_path, thread = thread, link = link, file = file)
        dlg.ShowModal()

def myapp():
    app = wx.App(False)
    MainFrame()
    # app.MainLoop()


if __name__ == '__main__':
    myapp()

预期的结果是文件应该被下载，同时进度条也应该被更新。

Answer 1

向Mike 为劫持他的骨架代码表示歉意，这里是使用urlretrieve 和wxPython event 进行更新的粗略版本。

import os
import wx
import time
from threading import Thread
from urllib.request import FancyURLopener
from urllib.request import urlopen
import wx.lib.newevent

urlretrieve = FancyURLopener().retrieve
progress_event, EVT_PROGRESS_EVENT = wx.lib.newevent.NewEvent()

class DownloadThread(Thread):

    def __init__(self, parent, url, fsize):
        """Constructor"""
        Thread.__init__(self)
        self.fsize = fsize
        self.url = url
        self.parent = parent
        self.start()

    def run(self):
        local_fname = os.path.basename(self.url)
        urlretrieve(self.url, local_fname, self.reporthook)

    def reporthook(self,blocknum, blocksize, totalsize):
        readsofar = blocknum * blocksize
        if totalsize > 0:
            percent = int((readsofar / totalsize) * 100)
        else: # total size is unknown
            percent = 0
        evt = progress_event(count=percent)
        #Send back current count for the progress bar
        try:
            wx.PostEvent(self.parent, evt)
        except: # The parent frame has probably been destroyed
            pass

class MyPanel(wx.Panel):
    def __init__(self, parent):
        """Constructor"""
        wx.Panel.__init__(self, parent)

        self.lbl = wx.StaticText(self, label="Download URL:")
        self.dl_txt = wx.TextCtrl(self)
        btn = wx.Button(self, label="Download")
        self.gauge = wx.Gauge(self, -1, range=100)

        self.main_sizer = wx.BoxSizer(wx.VERTICAL)
        dl_sizer = wx.BoxSizer(wx.HORIZONTAL)

        dl_sizer.Add(self.dl_txt, 1, wx.EXPAND|wx.ALL, 5)
        dl_sizer.Add(btn, 0, wx.ALL, 5)

        self.main_sizer.Add(dl_sizer, 0, wx.EXPAND)
        self.main_sizer.Add(self.lbl, 0, wx.ALL|wx.CENTER, 5)
        self.main_sizer.Add(self.gauge, 0, wx.ALL|wx.EXPAND, 5)

        self.SetSizer(self.main_sizer)

        #Bind to the progress event issued by the thread
        self.Bind(EVT_PROGRESS_EVENT, self.OnProgress)
        btn.Bind(wx.EVT_BUTTON, self.OnDownload)

    def OnDownload(self, event):
        url = self.dl_txt.GetValue()
        self.lbl.SetLabel(os.path.basename(url))
        try:
            header = urlopen(url)
            fsize = int(header.headers['Content-Length'])
            # start thread
            d_thread = DownloadThread(self, url, fsize)
            while d_thread.isAlive():
                time.sleep(0.1)
                wx.GetApp().Yield()
                continue
            # Finished re-set
            self.dl_txt.SetValue("")
            self.lbl.SetLabel("Download URL:")
            self.gauge.SetValue(0)

        except Exception as e:
            print ("Error: ", e)

    def OnProgress(self, event):
        """"""
        self.gauge.SetValue(event.count)

class DownloaderFrame(wx.Frame):

    def __init__(self):
        """Constructor"""
        wx.Frame.__init__(self, None, title="Downloader", size=(800, 400))
        panel = MyPanel(self)
        self.Show()

if __name__ == "__main__":
    app = wx.App(False)
    frame = DownloaderFrame()
    app.MainLoop()

Answer 2

不久前我就这个话题写了tutorial。

代码如下：

import requests
import os
import wx
import wx.lib.scrolledpanel as scrolled

from threading import Thread
from wx.lib.pubsub import pub


class DownloadThread(Thread):
    """Downloading thread"""

    def __init__(self, gnum, url, fsize):
        """Constructor"""
        Thread.__init__(self)
        self.fsize = fsize
        self.gnum = gnum
        self.url = url
        self.start()

    def run(self):
        """
        Run the worker thread
        """
        local_fname = os.path.basename(self.url)
        count = 1
        while True:
            if os.path.exists(local_fname):
                tmp, ext = os.path.splitext(local_fname)
                cnt = "(%s)" % count
                local_fname = tmp + cnt + ext
                count += 1
            else:
                break
        req = requests.get(self.url, stream=True)
        total_size = 0
        print(local_fname)
        with open(local_fname, "wb") as fh:
            for byte in req.iter_content(chunk_size=1024):
                if byte:
                    fh.write(byte)
                    fh.flush()
                total_size += len(byte)
                if total_size < self.fsize:
                    wx.CallAfter(pub.sendMessage,
                                 "update_%s" % self.gnum,
                                 msg=total_size)
        wx.CallAfter(pub.sendMessage,
                     "update_%s" % self.gnum,
                     msg=self.fsize)



class MyGauge(wx.Gauge):
    """"""

    def __init__(self, parent, range, num):
        """Constructor"""
        wx.Gauge.__init__(self, parent, range=range)

        pub.subscribe(self.updateProgress, "update_%s" % num)


    def updateProgress(self, msg):
        """"""
        self.SetValue(msg)


class MyPanel(scrolled.ScrolledPanel):
    """"""

    def __init__(self, parent):
        """Constructor"""
        scrolled.ScrolledPanel.__init__(self, parent)

        self.data = []
        self.download_number = 1

        # create the sizers
        self.main_sizer = wx.BoxSizer(wx.VERTICAL)
        dl_sizer = wx.BoxSizer(wx.HORIZONTAL)

        # create the widgets
        lbl = wx.StaticText(self, label="Download URL:")
        self.dl_txt = wx.TextCtrl(self)
        btn = wx.Button(self, label="Download")
        btn.Bind(wx.EVT_BUTTON, self.onDownload)

        # layout the widgets
        dl_sizer.Add(lbl, 0, wx.ALL|wx.CENTER, 5)
        dl_sizer.Add(self.dl_txt, 1, wx.EXPAND|wx.ALL, 5)
        dl_sizer.Add(btn, 0, wx.ALL, 5)
        self.main_sizer.Add(dl_sizer, 0, wx.EXPAND)

        self.SetSizer(self.main_sizer)
        self.SetAutoLayout(1)
        self.SetupScrolling()

    def onDownload(self, event):
        """
        Update display with downloading gauges
        """
        url = self.dl_txt.GetValue()
        try:
            header = requests.head(url)
            fsize = int(header.headers["content-length"]) / 1024

            sizer = wx.BoxSizer(wx.HORIZONTAL)
            fname = os.path.basename(url)
            lbl = wx.StaticText(self, label="Downloading %s" % fname)
            gauge = MyGauge(self, fsize, self.download_number)

            sizer.Add(lbl, 0, wx.ALL|wx.CENTER, 5)
            sizer.Add(gauge, 0, wx.ALL|wx.EXPAND, 5)
            self.main_sizer.Add(sizer, 0, wx.EXPAND)

            self.Layout()

            # start thread
            DownloadThread(self.download_number, url, fsize)
            self.dl_txt.SetValue("")
            self.download_number += 1
        except Exception as e:
            print("Error: ", e)

class DownloaderFrame(wx.Frame):
    """"""

    def __init__(self):
        """Constructor"""
        wx.Frame.__init__(self, None, title="Downloader", size=(800, 400))
        panel = MyPanel(self)
        self.Show()

if __name__ == "__main__":
    app = wx.App(False)
    frame = DownloaderFrame()
    app.MainLoop()