如何将从网站抓取的所有页面导出到 Excel

Question

我正在尝试将抓取的数据从网站导出到 Excel。但是我的代码用最后一次抓取的数据覆盖了 excel 文件中的先前数据。这是我第一次尝试抓取和 Pandas。请帮助我理解正确导出的逻辑。这是我的代码：

import requests
import lxml.html
import time
import sys
import pandas as pd

sys.stdin.reconfigure(encoding='utf-8')
sys.stdout.reconfigure(encoding='utf-8')


def parse_data(url):
    titles = []
    prices = []
    try:
        response = requests.get(url)
    except:
        return
    tree = lxml.html.document_fromstring(response.text)
    for item in tree.xpath('//*[contains(@class, "listing-item")]'):
            title = item.xpath(".//h2/a/text()")[0]
            price = item.xpath('.//*[contains(@class, "price")]/text()')
            price = price[0] if price else "N/A"
            titles.append(title)
            prices.append(price)
            
    return titles, prices


def output(titles, prices):
    output = pd.DataFrame({"Make": titles,
                           "Price": prices,
                           })
    writer = pd.ExcelWriter('avbuyer.com.xlsx', engine='xlsxwriter')
    output.to_excel(writer, sheet_name='Sheet1')

    output(titles, prices)


def main():
    for i in range(1, 3):
        url = 'https://www.avbuyer.com/aircraft/private-jets/page-' + str(i)
        print(url)
        parse_data(url)
        i += 1
        time.sleep(2)


if __name__ == "__main__":
    main()

Answer 1

在迭代期间，您总是会覆盖以前创建的列表（标题和价格），这就是为什么您总是只能从上次迭代中获得结果。 我建议你这样做：

dfFinal = pd.DataFrame()
def main():
    
    for i in range(1, 3):
        url = 'https://www.avbuyer.com/aircraft/private-jets/page-' + str(i)

        titles = []
        prices = []
        
        try:
            response = requests.get(url)
        except:
            return
        tree = lxml.html.document_fromstring(response.text)
        for item in tree.xpath('//*[contains(@class, "listing-item")]'):
            title = item.xpath(".//h2/a/text()")[0]
            price = item.xpath('.//*[contains(@class, "price")]/text()')
            price = price[0] if price else "N/A"
            titles.append(title)
            prices.append(price)

        output = pd.DataFrame({"Make": titles,
                               "Price": prices,
                               })
        dfFinal = dfFinal.appedn(output)    
        
        i += 1
        time.sleep(2)

dfFinal.to_excel(r'your path')