Python网页抓取页面循环

Question

感谢这里被问了很多次，但我似乎无法让它为我工作。

我写了一个刮板，它成功地从网站的第一页刮掉了我需要的一切。但是，我不知道如何让它循环浏览各个页面。

网址只是像这样 BLAH/3 + 'page=x' 递增

我学习编码的时间不长，所以任何建议都将不胜感激！

import requests
from bs4 import BeautifulSoup


url = 'http://www.URL.org/BLAH1/BLAH2/BLAH3'

soup = BeautifulSoup(r.content, "html.parser")

# String substitution for HTML
for link in soup.find_all("a"):
"<a href='>%s'>%s</a>" %(link.get("href"), link.text)

# Fetch and print general data from title class
general_data = soup.find_all('div', {'class' : 'title'})


for item in general_data:
    name = print(item.contents[0].text)
    address = print(item.contents[1].text.replace('.',''))
    care_type = print(item.contents[2].text)

更新：

r = requests.get('http://www.URL.org/BLAH1/BLAH2/BLAH3')

for page in range(10):

    r = requests.get('http://www.URL.org/BLAH1/BLAH2/BLAH3' + 'page=' + page)

soup = BeautifulSoup(r.content, "html.parser")
#print(soup.prettify())


# String substitution for HTML
for link in soup.find_all("a"):
    "<a href='>%s'>%s</a>" %(link.get("href"), link.text)

# Fetch and print general data from title class
general_data = soup.find_all('div', {'class' : 'title'})


for item in general_data:
    name = print(item.contents[0].text)
    address = print(item.contents[1].text.replace('.',''))
    care_type = print(item.contents[2].text)

更新 2！：

import requests
from bs4 import BeautifulSoup

url = 'http://www.URL.org/BLAH1/BLAH2/BLAH3&page='

for page in range(10):

r = requests.get(url + str(page))

soup = BeautifulSoup(r.content, "html.parser")

# String substitution for HTML
for link in soup.find_all("a"):
    print("<a href='>%s'>%s</a>" % (link.get("href"), link.text))

# Fetch and print general data from title class
general_data = soup.find_all('div', {'class' : 'title'})

for item in general_data:
    print(item.contents[0].text)
    print(item.contents[1].text.replace('.',''))
    print(item.contents[2].text)

Answer 1

要使用page=x 循环页面，您需要像这样使用for 循环>

import requests
from bs4 import BeautifulSoup

url = 'http://www.housingcare.org/housing-care/results.aspx?ath=1%2c2%2c3%2c6%2c7&stp=1&sm=3&vm=list&rp=10&page='

for page in range(10):

    print('---', page, '---')

    r = requests.get(url + str(page))

    soup = BeautifulSoup(r.content, "html.parser")

    # String substitution for HTML
    for link in soup.find_all("a"):
        print("<a href='>%s'>%s</a>" % (link.get("href"), link.text))

    # Fetch and print general data from title class
    general_data = soup.find_all('div', {'class' : 'title'})

    for item in general_data:
        print(item.contents[0].text)
        print(item.contents[1].text.replace('.',''))
        print(item.contents[2].text)

每个页面都可以不同，更好的解决方案需要更多关于页面的信息。有时您可以获得最后一页的链接，然后您可以使用此信息而不是 10 in range(10)

如果没有指向下一页的链接，您也可以使用while True 循环和break 离开循环。但首先你必须显示有问题的这个页面（真实页面的 URL）。

---

编辑：示例如何获取到下一页的链接，然后您将获取所有页面 - 而不是像以前的版本那样只有 10 个页面。

import requests
from bs4 import BeautifulSoup

# link to first page - without `page=`
url = 'http://www.housingcare.org/housing-care/results.aspx?ath=1%2c2%2c3%2c6%2c7&stp=1&sm=3&vm=list&rp=10'

# only for information, not used in url
page = 0 

while True:

    print('---', page, '---')

    r = requests.get(url)

    soup = BeautifulSoup(r.content, "html.parser")

    # String substitution for HTML
    for link in soup.find_all("a"):
        print("<a href='>%s'>%s</a>" % (link.get("href"), link.text))

    # Fetch and print general data from title class
    general_data = soup.find_all('div', {'class' : 'title'})

    for item in general_data:
        print(item.contents[0].text)
        print(item.contents[1].text.replace('.',''))
        print(item.contents[2].text)

    # link to next page

    next_page = soup.find('a', {'class': 'next'})

    if next_page:
        url = next_page.get('href')
        page += 1
    else:
        break # exit `while True`