将数据抓取到 csv 文件 beautifulsoup

Question

正如标题所说，我使用 beautifulsoup scraper 从网站抓取数据工作正常，但是当我尝试将数据加载到 csv 文件中时，它只保存 1 个数据区域，而不是 500 个数据区域，scraper 给出的是我的代码：

#from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import csv



#launch url
url = "https://www.canlii.org/en/#search/type=decision&jId=bc,ab,sk,mb,on,qc,nb,ns,pe,nl,yk,nt,nu&startDate=1990-01-01&endDate=1992-01-14&text=non-pecuniary%20award%20&resultIndex=1"

# create a new Chrome session
driver = webdriver.Chrome('C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\selenium\webdriver\common\chromedriver.exe')
driver.implicitly_wait(30)
driver.get(url)


#Selenium hands the page source to Beautiful Soup
soup=BeautifulSoup(driver.page_source, 'lxml')

csv_file = open('test.csv', 'w')

csv_writer = csv.writer(csv_file, quoting=csv.QUOTE_ALL)
csv_writer.writerow(['Reference', 'case', 'link', 'province', 'keywords','snippets'])

#Scrape all
for scrape in soup.find_all('li', class_='result '):
    print(scrape.text)    
    
#Reference Index
    Reference = scrape.find('span', class_='reference')
    print(Reference.text)

#Case Name Index
    case = scrape.find('span', class_='name')
    print(case.text)
    
#Canlii Keywords Index
    keywords = scrape.find('div', class_='keywords')
    print(keywords.text)
    
#Province Index
    province = scrape.find('div', class_='context')
    print(province.text)
               
#snippet Index
    snippet = scrape.find('div', class_='snippet')
    print(snippet.text)
 
# Extracting URLs from the attribute href in the <a> tags.
    link = scrape.find('a', href=True)
    print(link)        
            
csv_writer.writerow([Reference.text, case.text,link.href, province.text, keywords.text, snippet.text])
csv_file.close()

Answer 1

你的 csv_writer.writerow() 函数在你的 for 循环之外。尝试缩进它，看看它是否有效。