BeautifulSoup find.all() 网页抓取返回空

Question

当我试图抓取该网站的多个页面时，我没有得到任何内容作为回报。我通常会检查以确保我创建的所有列表长度相同，但所有列表都以len = 0 的形式返回。

我使用过类似的代码来抓取其他网站，为什么这段代码不能正常工作？

我尝试过的一些解决方案，但对我的目的不起作用：requests.Session() 解决方案，如 this answer、.json 中建议的 suggested here.

import requests
from requests import get
from bs4 import BeautifulSoup
import pandas as pd

from time import sleep
from random import randint
from googletrans import Translator

translator = Translator()

rg = []
ctr_n = []
ctr = []
yr = []
mn = []
sub = []
cst_n = []
cst = []
mag = []
pty_n = []
pty = []
can = []
pev1 = []
vot1 = []
vv1 = []
ivv1 = []
to1 = []
cv1 = []
cvs1 = []
pv1 = []
pvs1 = []
pev2 = []
vot2 = []
vv2 = []
ivv2 = []
to2 = []
cv2 = []
cvs2 =[]
pv2 = []
pvs2 = []
seat = []
no_info = []
manual = []

START_PAGE = 1
END_PAGE = 42


for page in range(START_PAGE, END_PAGE + 1):

    page = requests.get("https://sejmsenat2019.pkw.gov.pl/sejmsenat2019/en/wyniki/sejm/okr/" + str(page))

    page.encoding = page.apparent_encoding

    if not page:
        pass

    else:

        soup = BeautifulSoup(page.text, 'html.parser')
    
        tbody = soup.find_all('table', class_='table table-borderd table-striped table-hover dataTable no-footer clickable right2 right4')

        sleep(randint(2,10))
        
        for container in tbody:

            col1 = container.find_all('tr', {'data-id':'26079'})
            for info in col1:
                col_1 = info.find_all('td')
                for data in col_1:
                    party = data[0]
                    party_trans = translator.translate(party)
                    pty_n.append(party_trans)

                    pvotes = data[1]
                    pv1.append(pvotes)

                    pshare = data[2]
                    pvs1.append(pshare)

                    mandates = data[3]
                    seat.append(mandates)

            col2 = container.find_all('tr', {'data-id':'26075'})
            for info in col2:
                col_2 = info.find_all('td')
                for data in col_2:
                    party2 = data[0]
                    party_trans2 = translator.translate(party2)
                    pty_n.append(party_trans2)

                    pvotes2 = data[1]
                    pv1.append(pvotes2)

                    pshare2 = data[2]
                    pvs1.append(pshare2)

                    mandates2 = data[3]
                    seat.append(mandates2)

            col3 = container.find_all('tr', {'data-id':'26063'})
            for info in col3:
                col_3 = info.find_all('td')
                for data in col_3:
                    party3 = data[0].text
                    party_trans3 = translator.translate(party3)
                    pty_n.extend(party_trans3)

                    pvotes3 = data[1].text
                    pv1.extend(pvotes3)

                    pshare3 = data[2].text
                    pvs1.extend(pshare3)

                    mandates3 = data[3].text
                    seat.extend(mandates3)

            col4 = container.find_all('tr', {'data-id':'26091'})
            for info in col4:
                col_4 = info.find_all('td',recursive=True)
                for data in col_4:
                    party4 = data[0]
                    party_trans4 = translator.translate(party4)
                    pty_n.extend(party_trans4)

                    pvotes4 = data[1]
                    pv1.extend(pvotes4)

                    pshare4 = data[2]
                    pvs1.extend(pshare4)

                    mandates4 = data[3]
                    seat.extend(mandates4)

            col5 = container.find_all('tr', {'data-id':'26073'})
            for info in col5:
                col_5 = info.find_all('td')
                for data in col_5:
                    party5 = data[0]
                    party_trans5 = translator.translate(party5)
                    pty_n.extend(party_trans5)

                    pvotes5 = data[1]
                    pv1.extend(pvotes5)

                    pshare5 = data[2]
                    pvs1.extend(pshare5)

                    mandates5 = data[3]
                    seat.extend(mandates5)

            col6 = container.find_all('tr', {'data-id':'26080'})
            for info in col6:
                col_6 = info.find_all('td')
                for data in col_6:
                    party6 = data[0]
                    party_trans6 = translator.translate(party6)
                    pty_n.extend(party_trans6)

                    pvotes6 = data[1]
                    pv1.extend(pvotes6)

                    pshare6 = data[2]
                    pvs1.extend(pshare6)

                    mandates6 = data[3]
                    seat.extend(mandates6)
            
                
        #### TOTAL  VOTES ####
        tfoot = soup.find_all('tfoot')
        for data in tfoot:
            fvote = data.find_all('td')
            for info in fvote:
                votefinal = info.find(text=True).get_text()
                fvoteindiv = [votefinal]
                fvotelist = fvoteindiv * (len(pty_n) - len(vot1))
                vot1.extend(fvotelist)
            
        #### CONSTITUENCY NAMES ####
        constit = soup.find_all('a', class_='btn btn-link last')
        for data in constit:
            names = data.get_text()
            names_clean = names.replace("Sejum Constituency no.","")
            names_clean2 = names_clean.replace("[","")
            names_clean3 = names_clean2.replace("]","")
            namesfinal = names_clean3.split()[1]
            constitindiv = [namesfinal]
            constitlist = constitindiv * (len(pty_n) - len(cst_n))
            cst_n.extend(constitlist)

        #### UNSCRAPABLE INFO ####
        region = 'Europe'
        reg2 = [region]
        reglist = reg2 * (len(pty_n) - len(rg))
        rg.extend(reglist)

        country = 'Poland'
        ctr2 = [country]
        ctrlist = ctr2 * (len(pty_n) - len(ctr_n))
        ctr_n.extend(ctrlist)

        year = '2019'
        yr2 = [year]
        yrlist = yr2 * (len(pty_n) - len(yr))
        yr.extend(yrlist)

        month = '10'
        mo2 = [month]
        molist = mo2 * (len(pty_n) - len(mn))
        mn.extend(molist)

        codes = ''
        codes2 = [codes]
        codeslist = codes2 * (len(pty_n) - len(manual))
        manual.extend(codeslist)

        noinfo = '-990'
        noinfo2 = [noinfo]
        noinfolist = noinfo2 * (len(pty_n) - len(no_info))
        no_info.extend(noinfolist)

        print(len(rg), len(pty_n), len(pv1), len(pvs1), len(no_info), len(vot1), len(cst_n))

    

poland19 = pd.DataFrame({
'rg' : rg,
'ctr_n' : ctr_n,
'ctr': manual,
'yr' : yr,
'mn' : mn,
'sub' : manual,
'cst_n': cst_n,
'cst' : manual,
'mag': manual,
'pty_n': pty_n,
'pty': manual,
'can': can,
'pev1': no_info,
'vot1': vot1,
'vv1': vot1,
'ivv1': no_info,
'to1': no_info,
'cv1': no_info,
'cvs1': no_info,
'pv1': cv1,
'pvs1': cvs1,
'pev2': no_info,
'vot2': no_info,
'vv2': no_info,
'ivv2': no_info,
'to2': no_info,
'cv2': no_info,
'cvs2': no_info,
'pv2' : no_info,
'pvs2' : no_info,
'seat' : manual
})

print(poland19)

poland19.to_csv('poland_19.csv')

Answer 1

正如评论，您可能需要使用 Selenium。您可以替换请求库并将请求语句替换为如下所示：

from selenium import webdriver


wd = webdriver.Chrome('pathToChromeDriver') # or any other Browser driver
wd.get(url) # instead of requests.get()
soup = BeautifulSoup(wd.page_source, 'html.parser')

您需要按照此链接的说明安装和实现 selenium 库：https://selenium-python.readthedocs.io/

注意：我使用 selenium 测试了您的代码，并且能够获得您正在寻找的表，但是使用 class_=... 由于某种原因无法正常工作。 而是浏览抓取的数据，我发现它有一个属性 id。所以也许也可以试试这个：

tbody = soup.find_all('table', id="DataTables_Table_0")

再一次，通过使用 selenium 库执行 get 请求。 希望这有帮助:) 干杯