用 Python 抓取 aspx 网页

Question

我是网络抓取的新手，并试图 scape 以下网页：http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx

这是我尝试从 Python 网页访问内容的内容。我只是尝试抓取内容，现在，解析将进行。

import requests
from bs4 import BeautifulSoup

r_obj = requests.Session()

url = 'http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx'

r_soup = r_obj.get(url)

soup = BeautifulSoup(r_soup.content,'lxml')

hidden_inputs = soup.find_all("input",type="hidden")

data = {
        hidden_inputs[0]['name']:hidden_inputs[0]['value'],
        hidden_inputs[1]['name']:hidden_inputs[1]['value'],
        # hidden_inputs[2]['name']:hidden_inputs[2]['value'],
        # hidden_inputs[3]['name']:hidden_inputs[3]['value'],
        hidden_inputs[4]['name']:hidden_inputs[4]['value'],
        hidden_inputs[5]['name']:hidden_inputs[5]['value'],
        'allmarket$ctl00$ctl03$ctl00': 'Tomato, Potato, Onion',
        'allmarket$ctl00$ctl05$ctl00': 'Lahore',
        'allmarket$ctl00$ctl07$ctl00': '08/13/2021',
        'allmarket$ctl00$ctl03$ctl03$ctl01': 'on',
        'allmarket$ctl00$ctl03$ctl03$ctl02': 'on',
        'allmarket$ctl00$ctl03$ctl03$ctl03': 'on',
        'allmarket$ctl00$ctl05$ctl03$ctl01': 'on',
        'allmarket$ctl04':'', 
        'allmarket$ctl05':'', 
        'allmarket$ctl06': '0',
        'allmarket$ctl07': '0',
        'allmarket$ctl00$ctl00': 'View Report',
        }

url_needed = 'http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx'

final = r_obj.post(url_needed,verify=False,data=data)
soup1 = BeautifulSoup(final.content,"lxml")
detail_tab = soup1.find_all("table")

上面的代码运行后，detail_tab 不包含相关的表数据，只包含最上面的表头。我错过了什么？

Answer 1

要从站点获取数据，可以使用下一个示例（有几个<iframe>需要解析）：

import bs4
import requests

url = "http://www.amis.pk/Arrivalreports/Arrival%20source%20target.aspx"

with requests.session() as s:
    soup = BeautifulSoup(s.get(url).content, "html.parser")

    data = {}
    for inp in soup.select("input"):
        name = inp["name"]
        if name.startswith("all"):
            continue
        data[name] = inp["value"]

    data["allmarket$ctl00$ctl03$ctl00"] = "Tomato, Potato, Onion"
    data["allmarket$ctl00$ctl05$ctl00"] = "Lahore"
    data["allmarket$ctl00$ctl07$ctl00"] = "07/01/2021"
    data["allmarket$ctl00$ctl03$ctl03$ctl01"] = "on"
    data["allmarket$ctl00$ctl03$ctl03$ctl02"] = "on"
    data["allmarket$ctl00$ctl03$ctl03$ctl03"] = "on"
    data["allmarket$ctl00$ctl05$ctl03$ctl01"] = "on"
    data["allmarket$ctl01$ctl01$ctl02"] = "1"
    data["allmarket$ctl01$ctl05$ctl00"] = "Select a format"
    data["allmarket$ctl04"] = ""
    data["allmarket$ctl05"] = ""
    data["allmarket$ctl06"] = "1"
    data["allmarket$ctl07"] = "0"
    data["allmarket$ctl00$ctl00"] = "View Report"

    r = s.post(
        url,
        headers={
            "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0",
            "X-MicrosoftAjax": "Delta=true",
        },
        data=data,
    )

    # 1. iframe

    soup = BeautifulSoup(r.content, "html.parser")
    soup = BeautifulSoup(
        s.get(
            "http://www.amis.pk"
            + soup.select_one("#ReportFrameallmarket")["src"]
        ).content,
        "html.parser",
    )

    # 2. iframe

    soup = BeautifulSoup(
        s.get("http://www.amis.pk" + soup.select_one("#report")["src"]).content,
        "html.parser",
    )

    for tr in soup.select("tr:not(:has(tr))"):
        tds = [td.text for td in tr.select("td")]
        print(*tds, sep=" ")

打印：

Arrival Source & Target Detail in Quintal
    
ACrop Name Arrived From Province Name Arrived At Quantity
Onion Jacobabad Sindh Lahore 700
Onion Kahrorpacca Punjab Lahore 1500
Onion Swat KPK Lahore 600
Potato Abbottabad KPK Lahore 700
Potato Gilgit Gilgit Baldistan Lahore 400
Potato Lahore Punjab Lahore 1800
Potato Mansehra KPK Lahore 800
Tomato Peshawar KPK Lahore 2500