python多线程执行队列中的下载任务

一般先把下载任务先放到待处理队列中，也可以定时填充任务到待处理队列 ； 

 一个线程判断任务完成情况，完成后处理下载结果；

  开多个工作线程从队列中获取下载任务，完成后放到已完成队列中，失败要放回待处理；

# _*_ coding: utf-8 _*_
import requests
from threading import Thread
from queue import Queue
import time, json, math
import sys, os
import pymysql

req_queue  = Queue(maxsize=30000)
res_queue  = Queue()

class pagest():
    max_page   = 1
    username   = \'test\'
    password   = \'123\'
    date_start = \'2019-05-01\'
    date_end   = \'2019-05-05\'
    cookies    = {}

class maintd(Thread):
    def __init__(self):
        Thread.__init__(self)

    def run(self):
        while True:
            if res_queue.qsize() == pagest.max_page:
                bigfile()
                break
            time.sleep(10)


class worker(Thread):
    def __init__(self):
        Thread.__init__(self)

    def run(self):
        while True:
            if req_queue.empty():
                break
            page = req_queue.get(block=True, timeout=30)
            isok = parser(page)
            if isok == True:
                res_queue.put(page)
            else:
                req_queue.put(page)
                time.sleep(5)


def parser(page):
    sql = []
    try:
        url = \'http://a.b.c/log.html?page=%s&date_start=%s&date_end=%s\' % (page, pagest.date_start, pagest.date_end)
        res = requests.get(url, cookies=pagest.cookies)
        user = json.loads(res.text)
        f = open("dt/%s.txt" % page, "w")
                f.write(res.text+\'\n\')
                print(obj[\'phone\'])
        f.close()
    except Exception as e:
        print(e)
        return False
    return True


def login():
    url = \'http://a.b.c/login.html\'
    headers = {
        \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36\'
    }
    res = requests.get(url, headers=headers)
    sessid = res.cookies.get(\'PHPSESSID\')
    param = {
        \'username\': pagest.username,
        \'password\': pagest.password
    }
    res = requests.post(url, headers=headers, cookies={\'PHPSESSID\': sessid}, data=param)
    pagest.cookies = {
        \'PHPSESSID\': sessid
    }

    url = \'http://a.b.c/log.html?page=1&date_start=%s&date_end=%s\' % (pagest.date_start, pagest.date_end)
    res = requests.get(url, cookies=pagest.cookies)
    cnt = res.json().get(\'count\')
    pagest.max_page = math.ceil(int(cnt) / 100)

def init():
    path = os.path.realpath(sys.path[0])+\'\dt\'
    if not os.path.exists(path):
        os.mkdir(path)
    ls = os.listdir(path)
    for l in ls:
        fp = os.path.join(path, l)
        os.remove(fp)

def bigfile():
    i = 1
    bf = open("dt/all.txt", "a")
    while i <= pagest.max_page:
        f = open("dt/%s.txt" % i, "r")
        bf.write(f.read())
        f.close()
        i += 1
    bf.close()

def main():
    init()
    login()
    td = maintd()
    td.start()

    for i in range(pagest.max_page):
        req_queue.put(int(i+1))

    worker_list = []
    for i in range(30):
        item = worker()
        worker_list.append(item)

    for li in worker_list:
        li.start()
    for li in worker_list:
        li.join()

    td.join()

if __name__ == \'__main__\':
    main()