当 url 不变时在 python playwright 中处理分页

Question

我正在尝试用剧作家抓取这个网站https://franchisedisclosure.gov.au/Register，点击下一步按钮后 url 不会改变。我如何解决这个分页问题？ 这是我的代码 `

from bs4 import BeautifulSoup as bs
from playwright.sync_api import sync_playwright

url = 'https://franchisedisclosure.gov.au/Register'

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False, slow_mo=50)
    page = browser.new_page()
    page.goto(url)
    page.locator("text=I agree to the terms of use").click()
    page.locator("text=Continue").click()
    page.wait_for_load_state('domcontentloaded')
    page.is_visible('tbody')
    html = page.inner_html('table.table.table-hover')
    soup = bs(html, 'html.parser')
    table = soup.find('tbody')
    rows = table.findAll('tr')
    names = []
    industry = []
    Locations = []
    for row in rows:
        info = row.findAll('td')
        name = info[0].text.strip()
        industry = info[1].text.strip()
        Locations = info[2].text.strip()

`

我在网上查过，我看到的每个解决方案都涉及 url 更改。出于某种原因，您可以向网站的 api 发出请求。邮递员说了一些关于未发送参数的事情。

Answer 1

通过一些小的调整你可以得到它，让我们试试这个：

from bs4 import BeautifulSoup as bs
from playwright.sync_api import sync_playwright
import time

url = 'https://franchisedisclosure.gov.au/Register'

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False, slow_mo=100)
    page = browser.new_page()
    page.goto(url)
    page.locator("text=I agree to the terms of use").click()
    page.locator("text=Continue").click()
    page.wait_for_load_state('domcontentloaded')
    names = []
    industry = []
    Locations = []
    # When you click to next page, an element with text "Loading" appears in the screen, so we save that element
    loading_icon = "//strong[text()='Loading...']"
    # This is the "next page" button
    next_page_locator = "//ul[@class='pagination']/li[3]"
    # We select the option of 50 elements per page
    page.select_option("#perPageCount", value="50")
    # We wait for the selector of loading icon to be visible and then to be hidden, which means the new list is fully loaded
    page.wait_for_selector(loading_icon, state="visible")
    page.wait_for_selector(loading_icon, state="hidden")
    time.sleep(1)
    # We make a loop until the button "Next page" is disabled, which means there are no more pages to paginate
    while "disabled" not in page.get_attribute(selector=next_page_locator, name="class"):
        # We get the info you wanted
        page.is_visible('tbody')
        html = page.inner_html('table.table.table-hover')
        soup = bs(html, 'html.parser')
        table = soup.find('tbody')
        rows = table.findAll('tr')
        for row in rows:
            info = row.findAll('td')
            name = info[0].text.strip()
            industry = info[1].text.strip()
            Locations = info[2].text.strip()
        # Once we get the info we click in next page and we wait for the loading element to be visible and then to be hidden.
        page.click(next_page_locator)
        page.wait_for_selector(loading_icon, state="visible")
        page.wait_for_selector(loading_icon, state="hidden")
        time.sleep(1)

Answer 2

感谢伟大的问题......和答案。另外/与使用 loading_icon 相反，您还可以使用“networkidle”，因此扩展@Jaky Ruby 的答案添加 page.wait_for_load_state(state="networkidle")。我经常使用 networkidle 选项来检查下一页是否已完成加载，但是我在某处读到它不一定是最佳实践......但它经常起作用。

from bs4 import BeautifulSoup as bs
from playwright.sync_api import sync_playwright
import time

url = 'https://franchisedisclosure.gov.au/Register'

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False, slow_mo=100)
    page = browser.new_page()
    page.goto(url)
    page.locator("text=I agree to the terms of use").click()
    page.locator("text=Continue").click()
    page.wait_for_load_state('domcontentloaded')
    names = []
    industry = []
    Locations = []
    # When you click to next page, an element with text "Loading" appears in the screen, so we save that element
    loading_icon = "//strong[text()='Loading...']"
    # This is the "next page" button
    next_page_locator = "//ul[@class='pagination']/li[3]"
    # We select the option of 50 elements per page
    page.select_option("#perPageCount", value="50")
    # We wait for the selector of loading icon to be visible and then to be hidden, which means the new list is fully loaded
    page.wait_for_selector(loading_icon, state="visible")
    page.wait_for_selector(loading_icon, state="hidden")
    page.wait_for_load_state(state="networkidle")
    time.sleep(1)
    # We make a loop until the button "Next page" is disabled, which means there are no more pages to paginate
    while "disabled" not in page.get_attribute(selector=next_page_locator, name="class"):
        # We get the info you wanted
        page.is_visible('tbody')
        html = page.inner_html('table.table.table-hover')
        soup = bs(html, 'html.parser')
        table = soup.find('tbody')
        rows = table.findAll('tr')
        for row in rows:
            info = row.findAll('td')
            name = info[0].text.strip()
            industry = info[1].text.strip()
            Locations = info[2].text.strip()
        # Once we get the info we click in next page and we wait for the loading element to be visible and then to be hidden.
        page.click(next_page_locator)
        page.wait_for_selector(loading_icon, state="visible")
        page.wait_for_selector(loading_icon, state="hidden")
        time.sleep(1)