我的 BeautifulSoup 蜘蛛只爬 2 个页面而不是所有页面

Question

任何帮助将不胜感激，因为我是 python 新手。我创建了下面的网络爬虫，但它不会爬取所有页面，只有 2 个页面。需要进行哪些更改才能爬取所有页面？

请参阅 def trade_spider(max_pages) 循环，在底部我有 trade_spider(18) 应该循环所有页面。

感谢您的帮助。

import csv
import re
import requests
from bs4 import BeautifulSoup

f = open('dataoutput.csv','w', newline= "")
writer = csv.writer(f)

def trade_spider(max_pages):
    page = 1
    while page <= max_pages:
        url = 'http://www.zoopla.co.uk/for-sale/property/nottingham/?price_max=200000&identifier=nottingham&q=Nottingham&search_source=home&radius=0&pn=' + str(page) + '&page_size=100'
        source_code = requests.get(url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text)
        for link in soup.findAll('a', {'class': 'listing-results-price text-price'}):
            href = "http://www.zoopla.co.uk" + link.get('href')
            title = link.string 
            get_single_item_data(href) 
        page += 1
def get_single_item_data(item_url): 
    source_code = requests.get(item_url)
    plain_text = source_code.text
    soup = BeautifulSoup(plain_text)

    for item_name in soup.findAll('h2', {'itemprop': 'streetAddress'}):
     address = item_name.get_text(strip=True)
writer.writerow([address])
trade_spider(18)

Answer 1

您的代码运行良好，它会抓取所有页面（尽管只有 14 个页面而不是 18 个页面）。似乎您试图抓取街道地址，在这种情况下，第二个函数是不必要的，并且只会通过调用 requests.get() 太多次而使您的爬虫变慢。我稍微修改了代码，但这个更快。

import csv
import re
import requests
from bs4 import BeautifulSoup

f = open('dataoutput.csv','w', newline="")
writer = csv.writer(f)

def trade_spider(max_pages):
    page = 1
    while page <= max_pages:
        furl = 'http://www.zoopla.co.uk/for-sale/property/nottingham/?price_max=200000&identifier=nottingham&q=Nottingham&search_source=home&radius=0&pn=' + str(page) + '&page_size=100'
        source_code = requests.get(url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text)

        # Changed the class' value

        for link in soup.findAll('a', {'class': 'listing-results-address'}):     
            #href = "http://www.zoopla.co.uk" + link.get('href')
            #title = link.string 
            #get_single_item_data(href)
            address = link.get_text()
            print (address)               # Just to check it is working fine.
            writer.writerow([address])

        print (page)
        page += 1

# Unnecessary code

'''def get_single_item_data(item_url): 
source_code = requests.get(item_url)
plain_text = source_code.text
soup = BeautifulSoup(plain_text)

for item_name in soup.findAll('h2', {'itemprop': 'streetAddress'}):
    address = item_name.get_text(strip=True)
    writer.writerow([address])'''

trade_spider(18)