用 python 抓取多个页面（房地产网站）

Question

我似乎无法从房地产网站上抓取多个页面。我似乎只刮了第一页。任何帮助将不胜感激。下面的代码是我到目前为止收集到的，我在 stackoverflow 中尝试了各种解决方案，但我无法让它工作。

from bs4 import BeautifulSoup
import pandas as pd
import requests
import csv

def nonxt(target):
    if target is not None:
        spl_word = '>'
        sagent = str(target)
        p1 = sagent.split(spl_word, 1)
        p2 = p1[1]
        res = p2.split('<', 1)
        resf = res[0]
        cleanres = resf.strip().replace('\n', '')
        res = cleanres
        return res

def extract(page):
    URL = f'https://www.point2homes.com/MX/Real-Estate-Listings.html?LocationGeoId=&LocationGeoAreaId=240589&Location=San%20Felipe,%20Baja%20California,%20Mexico&page={page}'
    headers = {'User-Agent': "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1"}
    # Here the user agent is for Edge browser on windows 10. You can find your browser user agent from the above given link.
    r = requests.get(url=URL, headers=headers)
    soup = BeautifulSoup(r.content, 'html5lib')
    return soup

def transform(soup):
    listing = soup.findAll('article')
    with open('housing.csv', 'w', encoding = 'utf8', newline= '') as f:
        thewriter = csv.writer(f)
        header = ['Address', 'Beds', 'Baths', 'Size', 'Area', 'Acres', 'Price', 'Agent', 'Firm']
        thewriter.writerow(header)
        for ls in listing:
            address = ls.find('div', class_="address-container").text.replace('\n', "").strip()
            try:
                beds = ls.find('li', class_="ic-beds").text.replace('\n', "").strip()
            except:
                beds = ("Data Not Logged")
            try:
                baths = ls.find('li', class_="ic-baths").text.replace('\n', "").strip()
            except:
                baths = ("Data not logged")
            try:
                size = ls.find('li', class_="ic-sqft").text.replace('\n', "").strip()
            except:
                size = ("Data not logged")
            try:
                acre = ls.find('li', class_="ic-lotsize").text.replace('\n', "").strip()
            except:
                acre = ("Data not found")
            area = ls.find('li', class_="property-type ic-proptype").text.replace('\n', "").strip()
            price = ls.find('span', class_="green").text.replace('\n', "").strip()
            agentstr = ls.find('div', class_="agent-name")
            agent = ''
            agent1 = nonxt(agentstr)
            firmstr = ls.find('div', class_="agent-company")
            firm = ''
            if firmstr is not None:
              spl_word = '>'
              sagent = str(firmstr)
              p1 = sagent.split(spl_word, 1)
              p2 = p1[1]
              res = p2.split('<', 1)
              resf = res[0]
              cleanres = resf.strip().replace('\n', '')
              firm = cleanres
              r_list =[address, beds, baths, size, area, acre, price, agent1, firm]
              thewriter.writerow(r_list)
        return
page = 1
for i in range(1, 10):
    page = page+1
    webpage = f'https://www.point2homes.com/MX/Real-Estate-Listings.html?LocationGeoId=&LocationGeoAreaId=240589&Location=San%20Felipe,%20Baja%20California,%20Mexico&page={page}'
    webpage = webpage + str(page)
    print(f'Getting page {i}...')
    c = extract(webpage)
    transform(c)

我好像不能比第一页刮得更多，请帮忙

Answer 1

您的问题是您的 extract 函数采用 page 应该是页码。但是，在 for i in range(1, 10): 中，您将整个 url 作为参数传递，而不是页码。

要解决此问题，只需替换：

page = 1
for i in range(1, 10):
    page = page+1
    webpage = f'https://www.point2homes.com/MX/Real-Estate-Listings.html?LocationGeoId=&LocationGeoAreaId=240589&Location=San%20Felipe,%20Baja%20California,%20Mexico&page={page}'
    webpage = webpage + str(page)
    print(f'Getting page {i}...')
    c = extract(webpage)
    transform(c)

和：

for i in range(1, 10):
    print(f'Getting page {i}...')
    c = extract(i)
    transform(c)

作为 WebScrapingAPI 的一名工程师，我想出了这个脚本，它使用并发请求来为你加快抓取速度：

import requests
import json
from bs4 import BeautifulSoup
import concurrent.futures

API_KEY = '<YOUR_API_KEY>'
SCRAPER_URL = 'https://api.webscrapingapi.com/v1'

# Get the total number of pages by collecting the #bottom-list-no-results div and 
# dividing the totla number of results to 24
def get_total_pages():
    url = 'https://www.point2homes.com/MX/Real-Estate-Listings.html?LocationGeoId=&LocationGeoAreaId=240589&Location=San%20Felipe,%20Baja%20California,%20Mexico&page=1'
    params = {
        "api_key":API_KEY,
        "url": url,
        "render_js":"1",
        "wait_for_css":"#bottom-list-no-results",
        "extract_rules":'{"pages":{"selector":"#bottom-list-no-results","output":"text"}}',
    }
    res = requests.get(SCRAPER_URL, params=params)
    json_res = json.loads(res.text)
    return round(int(json_res['pages'][0].replace('Results','').partition(' of ')[2])/24)

# Scrape page and return properties
def get_properties_from_page(url):
    params = {
        "api_key":API_KEY,
        "url": url,
        "render_js":"1",
        "wait_for_css":"#bottom-list-no-results",
        "extract_rules":'{"articles":{"selector":"article","output":"html"}}',
    }
    res = requests.get(SCRAPER_URL, params=params)
    json_res = json.loads(res.text)
    return json_res['articles']


# Begin scraping
LISTINGS = []
RESULTS = []
URLS = []

total_pages = get_total_pages()
print(f'[i] Total number of pages: {total_pages}')
for i in range(1, total_pages+1):
    URLS.append(f'https://www.point2homes.com/MX/Real-Estate-Listings.html?LocationGeoId=&LocationGeoAreaId=240589&Location=San%20Felipe,%20Baja%20California,%20Mexico&page={i}')

with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
    res = executor.map(get_properties_from_page, URLS)
    for r in res:
        LISTINGS += r

for listing in LISTINGS:
    ls = BeautifulSoup(listing, 'html.parser')
    address = ls.find('div', class_='address-container').text.replace('
', "").strip()
    area = ls.find('li', class_="property-type ic-proptype").text.replace('
', "").strip()
    try:
        beds = ls.find('li', class_="ic-beds").text.replace('
', "").strip()
    except:
        beds = ("Data Not Logged")
    try:
        baths = ls.find('li', class_="ic-baths").text.replace('
', "").strip()
    except:
        baths = ("Data not logged")
    try:
        size = ls.find('li', class_="ic-sqft").text.replace('
', "").strip()
    except:
        size = ("Data not logged")
    try:
        acre = ls.find('li', class_="ic-lotsize").text.replace('
', "").strip()
    except:
        acre = ("Data not found")
    RESULTS.append({
        "address":address,
        "area":area,
        "beds":beds,
        "baths":baths,
        "size":size,
        "acre":acre
    })

print(RESULTS)