Indeed 评论抓取

Question

我正在尝试使用以下代码从 indeed 中抓取评论。但是，我收到 403（访问被拒绝）状态代码。确实阻止用户抓取评论。请帮忙。

from bs4 import BeautifulSoup
import pandas as pd
import requests
import numpy as np
import pandas as pd

lst=[]
for i in range(0, 40, 20):
    print(i)
    url = (f'https://www.indeed.com/cmp/Meta-dd1502f2/reviews?start={i}')
    header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"}
    page = requests.get(url, headers = header)
    print(f'The status code is {page.status_code}')
    soup = BeautifulSoup(page.content, 'lxml')
    main_data = soup.find_all("div",attrs={"data-tn-section":"reviews"})
    for data in main_data:
        try:
            date=data.find("span",attrs={"itemprop":"author"}).get_text(strip=True).split("-")[2]
        except AttributeError:
            date=np.nan
        try:
            title=data.find("h2").get_text(strip=True)
        except AttributeError:
            title=np.nan
        try:
            status=data.find("span",attrs={"itemprop":"author"}).get_text(strip=True).split("-")[0]
        except AttributeError:
            status=np.nan
            
        try:
            location=data.find("span",attrs={"itemprop":"author"}).get_text(strip=True).split("-")[1]
        except AttributeError:
            location=np.nan
        try:
            review=data.find("span",attrs={"itemprop":"reviewBody"}).get_text(strip=True)
        except AttributeError:
            review=np.nan
            
        try:
            pros=data.find('h2',class_='css-6pbru9 e1tiznh50').next_sibling.get_text(strip=True)           
        except:
            pros=np.nan
        try:
            cons=data.find('h2',class_='css-cvf89l e1tiznh50').next_sibling.get_text(strip=True)
        except:
            cons=np.nan
            
        try:
            rating=data.find("div",attrs={"itemprop":"reviewRating"}).find("button")['aria-label'].split(" ")[0]
        except AttributeError:
            rating=np.nan
        lst.append([date, title, status, location, review, pros, cons, rating])

df_meta=pd.DataFrame(data=lst,columns=['date', 'title', 'status', 'location', 'review', 'pros', 'cons', 'rating'])
df_meta

我期待带有“日期”、“标题”、“状态”、“位置”、“评论”、“优点”、“缺点”、“评级”的数据框

Answer 1

https://pypi.org/project/cloudscraper/

cloudscraper 它对我有用。

import cloudscraper
url = "https://www.indeed.com/cmp/Meta-dd1502f2/reviews?start=0"
c_scraper = cloudscraper.create_scraper(delay=10, browser="chrome") 
res = c_scraper.get(url) 
 
print(res.text)