使用请求解析网页内容失败

Question

我正在尝试使用请求模块 (without using session) 创建一个脚本来解析来自 webpage 的两个字段，但该脚本严重失败。但是，当我使用会话创建 another script 时，我可以完美地从该站点获取内容。

以下是获取内容的手动步骤：

1. 从下拉列表中选择first item。
2. 获取the links 到详情页面。
3. 从详细信息页面获取这些two fields。

在使用普通请求创建脚本时，我尝试使用 cookie，但最终得到AttributeError。

没有会话的脚本：

import re
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

base = 'https://compranet.hacienda.gob.mx'
link = 'https://compranet.hacienda.gob.mx/web/login.html'
vigen_detail_page = 'https://compranet.hacienda.gob.mx/esop/toolkit/opportunity/current/{}/detail.si'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
}

def grab_first_link_from_dropdown(link):
    r = requests.get(link,headers=headers)
    soup = BeautifulSoup(r.text,"html.parser")
    category_link = urljoin(base,soup.select_one('ul.dropdown-menu > li > a:contains("Vigentes")').get("href"))
    return category_link

def fetch_detail_page_link(cat_link):
    res = requests.get(cat_link,headers=headers)
    str_cookie = f"JSESSIONID={res.cookies['JSESSIONID']}"
    soup = BeautifulSoup(res.text,"html.parser")
    for items in soup.select("table.list-table > tbody.list-tbody > tr"):
        target_link = items.select_one("a.detailLink").get("onclick")
        detail_num = re.findall(r"goToDetail\(\'(\d+?)\'",target_link)[0]
        inner_link = vigen_detail_page.format(detail_num)
        yield str_cookie,inner_link

def get_content(str_cookie,inner_link):
    headers['Cookie'] = str_cookie
    res = requests.get(inner_link,headers=headers)
    soup = BeautifulSoup(res.text,"html.parser")
    try:
        expediente = soup.select_one(".form_question:contains('Código del Expediente') + .form_answer").get_text(strip=True)
    except AttributeError: expediente = ""
    try:
        descripcion = soup.select_one(".form_question:contains('Descripción del Expediente') + .form_answer").get_text(strip=True)
    except AttributeError: descripcion = ""
    return expediente,descripcion

if __name__ == '__main__':
    category_link = grab_first_link_from_dropdown(link)
    for cookie,detail_page_link in fetch_detail_page_link(category_link):
        print(get_content(cookie,detail_page_link))

我应该进行哪些可能的更改才能使脚本正常运行？

Answer 1

fetch_detail_page_link 上发生了重定向。 Python 请求默认遵循重定向。当 你的 脚本获取 cookie 时，它​​只是为链中的最终请求获取 cookie。您必须访问响应的 history 字段才能查看所遵循的重定向。使用 Session 对象执行此操作是可行的，因为它为您保留了这些 cookie。

我必须同意其他人的评论，即为此使用Session 对象确实是个好主意。但是，如果您坚持不使用Session，您的脚本将如下所示：

import re
import requests
from requests.cookies import RequestsCookieJar
from bs4 import BeautifulSoup
from urllib.parse import urljoin

base = 'https://compranet.hacienda.gob.mx'
link = 'https://compranet.hacienda.gob.mx/web/login.html'
vigen_detail_page = 'https://compranet.hacienda.gob.mx/esop/toolkit/opportunity/current/{}/detail.si'

headers = {
    'User-Agent': "Scraping Your Vigentes 1.0",
}


def grab_first_link_from_dropdown(link):
    r = requests.get(link, headers=headers)
    soup = BeautifulSoup(r.text, "html.parser")
    category_link = urljoin(base, soup.select_one('ul.dropdown-menu > li > a:contains("Vigentes")').get("href"))
    return category_link


def fetch_detail_page_link(cat_link):
    res = requests.get(cat_link, headers=headers)
    cookies = RequestsCookieJar()  # create empty cookie jar
    for r in res.history:
        cookies.update(r.cookies)  # merge in cookies from each redirect response
    cookies.update(res.cookies)  # merge in cookies from the final response

    soup = BeautifulSoup(res.text, "html.parser")
    for items in soup.select("table.list-table > tbody.list-tbody > tr"):
        target_link = items.select_one("a.detailLink").get("onclick")
        detail_num = re.findall(r"goToDetail\(\'(\d+?)\'", target_link)[0]
        inner_link = vigen_detail_page.format(detail_num)
        yield cookies, inner_link


def get_content(cookies, inner_link):
    res = requests.get(inner_link, headers=headers, cookies=cookies)
    if not res.ok:
        print("Got bad response %s :(" % res.status_code)
        return "", ""
    soup = BeautifulSoup(res.text, "html.parser")
    try:
        expediente = soup.select_one(".form_question:contains('Código del Expediente') + .form_answer").get_text(strip=True)
    except AttributeError:
        expediente = ""
    try:
        descripcion = soup.select_one(".form_question:contains('Descripción del Expediente') + .form_answer").get_text(strip=True)
    except AttributeError:
        descripcion = ""
    return expediente, descripcion


if __name__ == '__main__':
    category_link = grab_first_link_from_dropdown(link)
    for cookie, detail_page_link in fetch_detail_page_link(category_link):
        print(get_content(cookie, detail_page_link))