Pandas 仅将第一页抓取的数据保存到 CSV

Question

我正在尝试从该网站的前 5 页中抓取数据并将其保存为 CSV。一切似乎都运行良好，但只有网站的第一页被保存到 CSV。我认为这可能是一个缩进问题，但我无法弄清楚。任何帮助将不胜感激，谢谢！

from bs4 import BeautifulSoup
import requests
import pandas as pd
import numpy as np
import time
import random

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en-US,en;q=0.9',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Cookie': 'pis=3; ut=hB0ZZQzmgNnOh9lBQEu8bTmaWMULGZQUahdMlzzGd1k; SESSION_TOKEN=j-_KSJ8wscb0aeKUf-Ndr5wDcTCFdY3E2qwROIDEt5U; zjs_user_id=null; zjs_anonymous_id=%22hB0ZZQzmgNnOh9lBQEu8bTmaWMULGZQUahdMlzzGd1k%22; _pxvid=ef32ffea-7620-11eb-9542-0242ac12000d; _ga=GA1.2.845026454.1614116940; _gid=GA1.2.991776272.1614116940; _gcl_au=1.1.1819279014.1614116941; OptanonConsent=isIABGlobal=false&datestamp=Tue+Feb+23+2021+16%3A21%3A01+GMT-0600+(Central+Standard+Time)&version=5.9.0&landingPath=NotLandingPage&groups=1%3A1%2C0_172180%3A1%2C0_248632%3A1%2C0_172218%3A1%2C0_172151%3A1%2C0_172362%3A1%2C3%3A1%2C0_172152%3A1%2C0_172351%3A1%2C4%3A1%2C0_172338%3A1%2C0_172360%3A1%2C0_172153%3A1%2C0_172154%3A1%2C0_172343%3A1%2C0_177347%3A1%2C0_172331%3A1%2C0_172155%3A1%2C0_172156%3A1%2C0_248627%3A1%2C0_172157%3A1%2C0_248631%3A1%2C0_172158%3A1%2C0_172357%3A1%2C0_248633%3A1%2C0_172348%3A1%2C0_172159%3A1%2C0_172160%3A1%2C0_172161%3A1%2C0_172162%3A1%2C0_172163%3A1%2C0_172164%3A1%2C0_172165%3A1%2C0_172166%3A1%2C0_172167%3A1%2C0_172168%3A1%2C0_172169%3A1%2C0_172170%3A1%2C0_172171%3A1%2C0_172172%3A1%2C0_172173%3A1%2C0_172174%3A1%2C0_172175%3A1%2C0_172176%3A1%2C0_172177%3A1%2C0_172178%3A1%2C0_172179%3A1%2C0_172181%3A1%2C0_172182%3A1%2C0_172183%3A1%2C0_172184%3A1%2C0_172185%3A1%2C0_172186%3A1%2C0_172187%3A1%2C0_172188%3A1%2C0_172189%3A1%2C0_172190%3A1%2C0_172191%3A1%2C0_172192%3A1%2C0_172193%3A1%2C0_172195%3A1%2C0_172197%3A1%2C0_172198%3A1%2C0_172199%3A1%2C0_172200%3A1%2C0_172201%3A1%2C0_172202%3A1%2C0_172203%3A1%2C0_172204%3A1%2C0_172205%3A1%2C0_172206%3A1%2C0_172207%3A1%2C0_172208%3A1%2C0_172209%3A1%2C0_172210%3A1%2C0_172211%3A1%2C0_172212%3A1%2C0_172213%3A1%2C0_172214%3A1%2C0_172215%3A1%2C0_172216%3A1%2C0_172217%3A1%2C0_172219%3A1%2C0_172220%3A1%2C0_172221%3A1%2C0_172222%3A1%2C0_172223%3A1%2C0_172330%3A1%2C0_172333%3A1%2C0_172334%3A1%2C0_172335%3A1%2C0_172336%3A1%2C0_172337%3A1%2C0_172339%3A1%2C0_172340%3A1%2C0_172341%3A1%2C0_172342%3A1%2C0_172344%3A1%2C0_248628%3A1%2C0_172345%3A1%2C0_172346%3A1%2C0_172349%3A1%2C0_172350%3A1%2C0_172352%3A1%2C0_172353%3A1%2C0_172354%3A1%2C0_172355%3A1%2C0_172356%3A1%2C0_172358%3A1%2C0_172359%3A1%2C0_172361%3A1%2C0_248629%3A1%2C0_248630%3A1%2C0_248634%3A1&AwaitingReconsent=false; _px3=d6e4661fe0f89390bd501cf6d96d7c4ce6da6b629f038f745c417aec166457da:jkvtO/Et7fQoQ9uQjR7cLnpUmnMnTHJjbtYEYxtF8Af3XMaosoyoSH29Qf+5aiOY4Z/BqkATEDsYMrO6hKGNOQ==:1000:v1Auy0PIGkZc2wIJIcWfwOV3SoBz2sZHwNv/67LxTEKseVa/NakBSB7e9s397Ol/RCx/TcpBu3ZoJilwD/sP/3PIkNcxZXjbK+aHVEpfKf37sDvp8iNYyLqZ6QjNsa/0NXHrpVIWto2qgiaU21O2v9R9EgDeaTBEt4MCmMT87V4=',
    'Host': 'hotpads.com',
    'sec-ch-ua': '"Google Chrome";v="87", " Not;A Brand";v="99", "Chromium";v="87"',
    'sec-ch-ua-mobile': '?0',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
        }

params = {
            'lat: 41.7985 lon: -87.7117 z: 11'

        }


pages = np.arange(1, 6)

for page in pages:

    page = requests.get('https://hotpads.com/chicago-il/apartments-for-rent' + '?page=' + str(page), headers = headers)
    source = page.content
    soup = BeautifulSoup(source, 'lxml')
    postings = soup.find_all('div', class_="AreaListingsContainer")
    time.sleep(random.randint(2,10))

    for p in postings:
        
        url = ["https://hotpads.com" + u['href'] for u in p.find_all('a', href=True, text=True)]
        address = [a.get_text() for a in p.find_all('h4', class_="styles__ListingCardName-y78yl0-8 jQmZHq")]
        price = [p.get_text() for p in p.find_all('div', class_="styles__ListingCardPrice-y78yl0-17 cguwHc")]
        beds = [b.get_text() for b in p.find_all('div', class_="styles__ListingCardBedDisplay-y78yl0-7 iPqMa")]

    homes = list(zip(url, address, price, beds))

    df = pd.DataFrame(homes, columns = ['URL', 'Address', 'Price', 'Beds'])
    print(df)

    df.to_csv('Chicago_homes.csv')

Answer 1

看起来您在每次迭代中都覆盖了原始 df。有两种解决方案。

1- 在循环之前初始化数据帧，在循环完成后将整个帧写入文件。

df = pd.DataFrame()

for page in pages:
    #do stuff...
    df = pd.concat([df,pd.DataFrame(homes, columns = ['URL', 'Address', 'Price', 'Beds'])])

#put this outside of the loop.
df.to_csv('Chicago_homes.csv')

2- 将数据帧数据附加到循环内的文件中。这在执行许多循环时很有用，并且您不想将大型数据帧保存在内存中。

for page in pages:
    #do stuff..
    if os.path.exists('Chicago_homes.csv'):
        df.to_csv('Chicago_homes.csv',mode='a',header=False)
    else:
        df.to_csv('Chicago_homes.csv')