add_done_call_back() 给出 AttributeError: 'Future' object has no attribute 'select'

Question

我尝试制作一个简单的网络爬虫。

我想用父链接和在父链接页面上找到的链接以及与之关联的文本创建一个数据框。

import requests
from bs4 import BeautifulSoup
from queue import Queue, Empty
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urljoin
import urllib
from urllib.error import HTTPError #used in the main function, to catch HTTPErrors
from requests.exceptions import InvalidURL #used in the main function, to catch invalidUrl errors
from urllib.parse import urlparse #used to parse the homepage url and get network location out of it
from urllib.parse import quote #used to correct incorrect urls
import pandas as pd


class MultiThreadScraper:

    global df
    df = pd.DataFrame(data=None, columns = ['parent_link','link', 'text'])
    
    def __init__(self, base_url):

        self.base_url = base_url
        self.root_url = '{}://{}'.format(urlparse(self.base_url).scheme, urlparse(self.base_url).netloc)
        self.pool = ThreadPoolExecutor(max_workers=20)
        self.scraped_pages = set([])
        self.to_crawl = Queue()
        self.to_crawl.put(self.base_url)
    
    #gets http.client.HTTPResponse from the server
    def get_http_response(self, url):
        header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.11',
                      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                      'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
                      'Accept-Encoding': 'none',
                      'Accept-Language': 'en-US,en;q=0.8',
                      'Connection': 'keep-alive'}
        request = urllib.request.Request(url, headers = header) #urllib.request.Request object
        response = urllib.request.urlopen(request)  #http.client.HTTPResponse
        return response

    #accepts the http.client.HTTPResponse from the server and fetches the html content using BeautifulSoup
    def get_html_content(self, httpResponse):
        httpResponse_content = httpResponse.read() #data in bytes
        httpResponse_htmlContent = BeautifulSoup(httpResponse_content, 'html.parser')
        return httpResponse_htmlContent
        
    def parse_links(self, html,df):
        all_tags_with_hrefs = html.select('[href]')
        for link in all_tags_with_hrefs:
            url = link['href']
            if url.startswith('/') or url.startswith(self.root_url):
                url = urljoin(self.root_url, url)
                if (link.text).strip()!='':
                    df.loc[-1,'parent_link']= self.base_url
                    df.loc[-1,'link'] = url
                    df.loc[-1,'text'] = (link.text).strip() 
                    df.drop_duplicates(subset='link', inplace=True)
                    df.reset_index(drop=True, inplace=True)
                if url not in self.scraped_pages:
                    self.to_crawl.put(url)

    def post_scrape_callback(self, html_content):
            self.parse_links(html_content, df)

    def scrape_page(self, url):
        try:
            url = urllib.parse.quote(url, safe='/,:,-,?,=,&')
            get_response = self.get_http_response(url)
            try: 
                html_content = self.get_html_content(get_response)
            except:
                print(f'{url} : ERROR READING RESPONSE !')
        except(HTTPError, InvalidURL):
            print(f'{url} : NO RESPONSE !')
        return html_content
        

    def run_scraper(self):
        while True:
            try:
                target_url = self.to_crawl.get(timeout=10)
                if target_url not in self.scraped_pages:
                    print("Scraping URL: {}".format(target_url))
                    self.scraped_pages.add(target_url)
                    job = self.pool.submit(self.scrape_page, target_url)
                    job.add_done_callback(self.post_scrape_callback)
            except Empty:
                return
            except Exception as e:
                print(e)
                continue
if __name__ == '__main__':
    s = MultiThreadScraper("https://www.nationalgrid.com/")
    s.run_scraper() 

它给了我以下错误： AttributeError: 'Future' 对象没有属性 'select' 从文档中我了解到 add_done_callback() 添加了一个回调以在 Future 完成时运行。 任何帮助将不胜感激！

Answer 1

您将需要一个池并将您的代码逻辑拆分为可以独立运行的较小函数。 例如，如果您编写一个接收 URL 作为参数的 crawl 函数，您可以执行以下操作：

import concurrent.futures
urls = [] # a url list you create
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
    for url in urls:
        executor.submit(crawl, url)

那么就可以使用多线程解析多个URL（这里限制为5个）