如何下载类python中的所有pdf

Question

我有大约 900 页，每页包含 10 个按钮（每个按钮都有 pdf）。我想下载所有的 pdf - 程序应该浏览所有页面并一一下载 pdf。 在此，一些pdf（随机）没有下载（代码一直到最后一页）但是一些页面中随机的一些pdf没有下载 大约 4521 个 pdf 正在下载 9000 个 pdf

仅搜索 .pdf 的代码，但我的 href 没有 .pdf page_no（1 到 900）。

https://bidplus.gem.gov.in/bidlists?bidlists&page_no=3

这是网站，下面是链接：

投标编号：GEM/2021/B/1804626

下面是我的代码

import requests
from bs4 import BeautifulSoup as bs

end_number = 900
current_page = 1
pdf_links = {}
path = '<your path>'

with requests.Session() as s:
    while True:
        r = s.get(f'https://bidplus.gem.gov.in/bidlists?bidlists&page_no={current_page}')
        soup = bs(r.content, 'lxml')
        for i in soup.select('.bid_no > a'):
            pdf_links[i.text.strip().replace('/', '_')] = 'https://bidplus.gem.gov.in' + i['href']
        #print(pdf_links)
        if current_page == 1:
            num_pages = int(soup.select_one('.pagination li:last-of-type > a')['data-ci-pagination-page'])
            print(num_pages)
        if current_page == num_pages or current_page > end_number:
            break
        current_page+=1
    
for k,v in pdf_links.items():
    with open(f'{path}/{k}.pdf', 'wb') as f:
        r = s.get(v)
        f.write(r.content)

Answer 1

未测试，但演示了如何使用 requests 模块流式传输文件：

import requests
from bs4 import BeautifulSoup as bs
from concurrent.futures import ThreadPoolExecutor


def writepdf(k, v):
    path = '<YOUR_PATH>'
    with requests.Session() as session:
        with open(f'{path}/{k}.pdf', 'wb') as f:
            with session.get(v, stream=True) as r:
                for data in r.iter_content():
                    f.write(data)

def main():
    with requests.Session() as s:
        current_page = 1
        end_number = 900
        threads = []
        with ThreadPoolExecutor() as executor:
            while current_page <= end_number:
                r = s.get(f'https://bidplus.gem.gov.in/bidlists?bidlists&page_no={current_page}')
                r.raise_for_status()
                soup = bs(r.content, 'lxml')
                for i in soup.select('.bid_no > a'):
                    k = i.text.strip().replace('/', '_')
                    v = f'https://bidplus.gem.gov.in{i["href"]}'
                    threads.append(executor.submit(writepdf, k, v))
                if current_page == 1:
                    num_pages = int(soup.select_one('.pagination li:last-of-type > a')['data-ci-pagination-page'])
                    end_number = min(end_number, num_pages)
                current_page += 1
            for t in threads:
                t.result()

if __name__ == '__main__':
    main()