【发布时间】：2021-08-10 05:51:58
【问题描述】：
这是我写的抓取代码。
import sys
import time
from time import sleep
from selenium import webdriver
from selenium.common import exceptions
from tqdm import tqdm
Max_Page = 2
def open_webdriver():
options = webdriver.ChromeOptions()
options.add_argument('headless')
driver = webdriver.Chrome(r'c:\chromedriver\chromedriver.exe', options=options)
return driver
def get_result(title, link, price):
result = (title, link, price)
return result
def get_url(search_term, page):
base_template = f'https://shopee.com.my/search?keyword={search_term}'
base_url = base_template.format(search_term)
url_template = base_url + '&page={}'
if page == 0:
return base_url
else:
return url_template.format(page)
return url_template.format(page)
def time_interval():
from random import random
time_in_seconds = random() * 2
sleep(time_in_seconds)
def spider(search_term):
num_records_scraped = 0
for page in range(0, Max_Page):
driver = open_webdriver()
search_url = get_url(search_term, page)
driver.get(search_url)
print(search_url)
# search_url = f'https://shopee.ph/search?keyword={search_term}&page={page}'
time.sleep(1)
# Page Scroll
y = 100
for timer in tqdm(range(0, 6), file=sys.stdout):
driver.execute_script("window.scrollTo(0, " + str(y) + ")")
y += 700
time.sleep(1)
items = driver.find_elements_by_xpath('//div[@data-sqe="item"]')
for item in items:
try:
title = item.find_element_by_xpath('.//a/div/div/div[2]/div[1]/div/div').text
except exceptions.NoSuchElementException:
continue
try:
link = item.find_element_by_xpath('.//a').get_attribute('href')
except exceptions.NoSuchElementException:
continue
try:
price_temp = item.find_element_by_xpath('.//a/div/div/div[2]/div[2]/div/span[2]').text
price = "₱" + price_temp
except exceptions.NoSuchElementException:
continue
record = get_result(title, link, price)
print(record)
if record:
get_records.append(record)
num_records_scraped += 1
time_interval()
driver.quit()
if __name__ == '__main__':
get_records = []
spider('ipad')
输出
上面的代码不会得到标题项。但是，如果我提供直接 URL，它会起作用。
def spider(search_term):
num_records_scraped = 0
for page in range(0, Max_Page):
driver = open_webdriver()
# search_url = get_url(search_term, page)
search_url = f'https://shopee.ph/search?keyword={search_term}&page={page}'
driver.get(search_url)
print(search_url)
time.sleep(1)
# Page Scroll
y = 100
for timer in tqdm(range(0, 6), file=sys.stdout):
driver.execute_script("window.scrollTo(0, " + str(y) + ")")
y += 700
time.sleep(1)
items = driver.find_elements_by_xpath('//div[@data-sqe="item"]')
for item in items:
try:
title = item.find_element_by_xpath('.//a/div/div/div[2]/div[1]/div/div').text
except exceptions.NoSuchElementException:
continue
try:
link = item.find_element_by_xpath('.//a').get_attribute('href')
except exceptions.NoSuchElementException:
continue
try:
price_temp = item.find_element_by_xpath('.//a/div/div/div[2]/div[2]/div/span[2]').text
price = "₱" + price_temp
except exceptions.NoSuchElementException:
continue
record = get_result(title, link, price)
print(record)
if record:
get_records.append(record)
num_records_scraped += 1
time_interval()
driver.quit()
区别是
# search_url = get_url(search_term, page)
search_url = f'https://shopee.ph/search?keyword={search_term}&page={page}'
所以获取url函数有问题，但不知道是什么问题。
def get_url(search_term, page):
base_template = f'https://shopee.com.my/search?keyword={search_term}'
base_url = base_template.format(search_term)
url_template = base_url + '&page={}'
if page == 0:
return base_url
else:
return url_template.format(page)
return url_template.format(page)
我已经验证了URL站点结果是一样的，
【问题讨论】：
标签： python web-scraping