网络抓取 webdriver 异常时出错

Question

!pip install selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
from datetime import datetime
import pandas as pd
errors = []
season = []

for id in range(46605, 46985):
my_url = f'https://www.premierleague.com/match/{id}'
option = Options()
#option.headless = True
driver = webdriver.Chrome(options=option)
driver.get(my_url)

代码运行良好，直到这里。

date = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="mainContent"]/div/section/div[2]/section/div[1]/div/div[1]/div[1]'))).text
date = datetime.strptime(date, '%a %d %b %Y').strftime('%m/%d/%Y')
home_team = driver.find_element_by_xpath('//*[@id="mainContent"]/div/section/div[2]/section/div[3]/div/div/div[1]/div[1]/a[2]/span[1]').text
away_team = driver.find_element_by_xpath('//*[@id="mainContent"]/div/section/div[2]/section/div[3]/div/div/div[1]/div[3]/a[2]/span[1]').text

执行这些行时会弹出一个错误。 Error Screenshot 1

Error Screenshot 2

Answer 1

为什么不使用英超网站使用的 api？

import requests

fixture = 66553

headers =   {
    'accept':'*/*',
    'accept-encoding':"gzip;q=1.0, identity; q=0.5",
    'accept-language':'en-ZA,en;q=0.9,en-GB;q=0.8,en-US;q=0.7,de;q=0.6',
    'content-type':'application/x-www-form-urlencoded; charset=UTF-8',
    'origin':'https://www.premierleague.com',
    'referer':'https://www.premierleague.com/',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
    }

url = f'https://footballapi.pulselive.com/football/broadcasting-schedule/fixtures/{fixture}'
data = requests.get(url,headers=headers).json()

print(data['fixture']['attendance'])
print(data['fixture']['kickoff']['label'])
print(data['fixture']['teams'])

url = f'https://footballapi.pulselive.com/football/fixtures/{fixture}/textstream/EN?pageSize=1000&sort=desc'
data = requests.get(url,headers=headers).json()

for message in data['events']['content']:
    print(message['text'])

Answer 2

@Kaustav，将其写在答案中，因为它是一个代码块，并且不知道如何将其放在评论部分。 尽管我赞同@Joe 的想法，但由于您想要一个精确的语法，我想我可以整理一段代码来向您展示。 所以，这里的代码，打开浏览器，获取详细信息（正如您在代码中输入的那样），然后将它们存储在一个列表中（用于此任务的显示目的，但实际上可能不需要它 - 取决于您的要求)。

我要再次重申，打开、使用和关闭浏览器（甚至是无头浏览器）进行如此多的迭代可能会在循环中的某个时间点影响性能，而您所有的时间都会浪费掉。我强烈建议您找到该网站的 API（如果可用）并使用它来成功。

话虽如此，这里是供您使用的代码。

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
from datetime import datetime
import pandas as p
import time
errors = []
season = []

url_ls=[] # used to show to query creator a demo
for id in range(46605, 46985):
    my_url = f'https://www.premierleague.com/match/{id}'
    option = Options()
    option.add_argument('--headless')
    option.add_argument('--disable-gpu')
    driver = webdriver.Chrome(options=option)
    driver.get(my_url)
    title = driver.current_url
    try:
        WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//*[text()='Accept All Cookies']"))).click()
    except:
        print("cookie modal not found")
        continue
    # time.sleep(10)
    match_date = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='matchInfo']//div[contains(@class, 'matchDate')]"))).text
    match_date_f = datetime.strptime(match_date, '%a %d %b %Y').strftime('%m/%d/%Y')
    home_team = driver.find_element(By.XPATH, "//div[@class='team home']").text
    away_team = driver.find_element(By.XPATH, "//div[@class='team away']").text
    tup = (title + "|" + match_date + "|" + home_team + "|" + away_team)
    url_ls.append(tup)
    driver.close()
print(url_ls)  # used to show to query creator a demo

这是输出（我只是在此代码中附加了当前 url 和元素的文本，但您可以进一步扩展此块并添加您的元素 - 您可以在导入 pandas 时将数据发送到一个数据框，然后到 excel 或 csv - 你喜欢它）

['https://www.premierleague.com/match/46605|Sat 10 Aug 2019|Liverpool|Norwich City', 'https://www.premierleague.com/match/46606|Sat 10 Aug 2019|AFC Bournemouth|Sheffield United', 'https://www.premierleague.com/match/46607|Sat 10 Aug 2019|Burnley|Southampton', 'https://www.premierleague.com/match/46608|Sat 10 Aug 2019|Crystal Palace|Everton', 'https://www.premierleague.com/match/46609|Sun 11 Aug 2019|Leicester City|Wolverhampton Wanderers']

Process finished with exit code 0