从tripadvisor刮取酒店..如何从1到10页的所有页面中获取酒店并存储它？

Question

我的代码显示酒店的第一页。为什么没有显示更多？

import csv

import requests

from bs4 import BeautifulSoup

hotels=[]
i=0

url0 = 'https://www.tripadvisor.com/Hotels-g295424-Dubai_Emirate_of_Dubai-     Hotels.html#EATERY_LIST_CONTENTS'

r = requests.get(url0)

data = r.text
soup = BeautifulSoup(r.text, "html.parser")with open('hotels_Data.csv','wb') as file:

for link in soup.findAll('a', {'property_title'}):
    print('https://www.tripadvisor.com/Hotels-g295424-' + link.get('href'))
    print(link.string)


for i in range(20):
   while int(i) <= (20):
    i = str(i)

    url1 = 'https://www.tripadvisor.com/Hotels-g295424-oa' + i + '-  Dubai_Emirate_of_Dubai-Hotels.html#EATERY_LIST_CONTENTS'
    r1 = requests.get(url1)
    data1 = r1.text
    soup1 = BeautifulSoup(data1, "html.parser")
    for link in soup1.findAll('a', {'property_title','price'}):
        print('https://www.tripadvisor.com/Hotels-g294212-' +    link.get('href'))
        print(link.string)
        for link in soup.select("a.reference.internal"):
            url1 = link["href"]
            absolute_url = urljoin(base_url, url1)

            print(url1, absolute_url)       
        writer = csv.writer(file)
        for row in hotels:
            writer.writerow([s.encode("utf-8") for s in row])                                                
break

Answer 1

在页面底部检查指向下一页的链接 - 此门户不使用页码 - 1、2、3 等 - 但提供偏移量 - 0、30、 60、90 等。 （因为它在页面上显示 30 个优惠）

所以你必须在url中使用值0、30、60、90等

"...-oa" + offset + "-Dubai_Emirate..."

你可以使用 ie。 range(0, 250, 30) 获取值0、30、60、90。

import requests
from bs4 import BeautifulSoup

for offset in range(0, 250, 30):
    print('--- page offset:', offset, '---')

    url = 'https://www.tripadvisor.com/Hotels-g295424-oa' + str(offset) + '-Dubai_Emirate_of_Dubai-Hotels.html#EATERY_LIST_CONTENTS'

    r = requests.get(url)
    soup = BeautifulSoup(r.text, "html.parser")

    for link in soup.find_all('a', {'property_title'}):
          print(link.text)

但报价可能超过 250 个，因此您必须获得最后一页的链接才能获得正确的价值，而不是 250

import requests
from bs4 import BeautifulSoup

offset = 0
url = 'https://www.tripadvisor.com/Hotels-g295424-oa' + str(offset) + '-Dubai_Emirate_of_Dubai-Hotels.html#EATERY_LIST_CONTENTS'

r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser")

for link in soup.find_all('a', {'last'}):
    page_number = link.get('data-page-number')
    last_offset = int(page_number) * 30
    print('last offset:', last_offset)

在range(0, last_offset+1, 30)中使用last_offset+1

---

编辑：餐厅使用 JavaScript 和 AJAX 加载数据

import requests
from bs4 import BeautifulSoup

size = 30

# direct url - doesn't have expected information
#url = 'https://www.tripadvisor.com/Restaurants-g187791-Rome_Lazio.html'

# url used by AJAX
url = 'https://www.tripadvisor.com/RestaurantSearch?Action=PAGE&geo=187791&ajax=1&itags=10591&sortOrder=relevance&o=a' + str(size) + '&availSearchEnabled=true&eaterydate=2017_04_27&date=2017-04-28&time=20%3A00%3A00&people=2'

r = requests.get(url)
soup = BeautifulSoup(r.text, "html.parser")

link = soup.find_all('a')[-1]
page_number = link.get('data-page-number')
last_offset = int(page_number) * size # *30
print('last offset:', last_offset)

offset = link.get('data-offset')
offset = int(offset) + size # +30
print('offset:', offset)