使用 HTML 内容解析“加载更多”响应

Question

我正在尝试抓取位于以下链接的伊斯坦布尔省公告部分中的每个内容，该部分会在页面底部加载带有“加载更多”的内容。从开发工具/网络中，我检查了发送的 POST 请求的属性并相应地更新了标头。响应显然不是 json 而是一个 html 代码。

我想生成解析后的 html 响应，但是当我抓取它时，它不会返回任何内容，并且永远停留在第一个请求中。提前谢谢你。

您能解释一下我的代码有什么问题吗？我在这里检查了数十个问题，但无法解决问题。据我了解，它只是无法解析响应 html，但我不知道为什么。

ps：20 天来，我一直热衷于 Python 和爬虫。原谅我的无知。

import scrapy

class DuyurularSpider(scrapy.Spider):
    name = 'duyurular'
    allowed_domains = ['istanbul.gov.tr']
    start_urls = ['http://istanbul.gov.tr/duyurular']

    headerz = {
        "Accept": "*/*",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "en-US,en;q=0.9",
        "Connection" : "keep-alive",
        "Content-Length": "112",
        "Content-Type": "application/json",
        "Cookie" : "_ga=GA1.3.285027250.1638576047; _gid=GA1.3.363882495.1639180128; ASP.NET_SessionId=ijw1mmc5xrpiw2iz32hmqb3a; NSC_ESNS=3e8876df-bcc4-11b4-9678-e2abf1d948a7_2815152435_0584317866_00000000013933875891; _gat_gtag_UA_136413027_31=1",
        "Host": "istanbul.gov.tr",
        "Origin": "http://istanbul.gov.tr",
        "Referer": "http://istanbul.gov.tr/duyurular",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
        "X-Requested-With": "XMLHttpRequest",
                }

    def parse(self, response):

        url = 'http://istanbul.gov.tr/ISAYWebPart/Announcement/AnnouncementDahaFazlaYukle'
        load_more = scrapy.Request(url, callback = self.parse_api, method = "POST", headers = self.headerz)    

        yield load_more    

    def parse_api(self, response):
        raw_data = response.body

        
        data = raw_data.xpath('//div[@class="ministry-announcements"]')

        for bilgi in data:

            gun =  bilgi.xpath('//div[@class = "day"]/text()').extract_first()  #day
            ay = bilgi.xpath('//div[@class = "month"]/text()').extract_first() #month

            metin = bilgi.xpath('//a[@class ="announce-text"]/text()').extract_first() #text

            yield {'Ay:' : ay,
                   'Gün' : gun,
                   'Metin': metin,}

Result I encounter:

Answer 1

1. 删除Content-Length，也永远不要将它包含在标题中。另外你应该删除cookie并让scrapy处理它。

2. 查看请求正文并为每个页面重新创建它：

3. 您需要知道何时停止，在这种情况下它是一个空白页面。

4. 在bilgi.xpath 部分，您一遍又一遍地得到同一行，因为您忘记了开头的一个点。

完整的工作代码：

import scrapy
import json


class DuyurularSpider(scrapy.Spider):
    name = 'duyurular'
    allowed_domains = ['istanbul.gov.tr']
    start_urls = ['http://istanbul.gov.tr/duyurular']
    page = 1
    
    headers = {
        "Accept": "*/*",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "en-US,en;q=0.9",
        "Connection": "keep-alive",
        "Content-Type": "application/json",
        "Host": "istanbul.gov.tr",
        "Origin": "http://istanbul.gov.tr",
        "Referer": "http://istanbul.gov.tr/duyurular",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36",
        "X-Requested-With": "XMLHttpRequest",
    }

    def parse(self, response):
        url = 'http://istanbul.gov.tr/ISAYWebPart/Announcement/AnnouncementDahaFazlaYukle'
        body = {
            "ContentCount": "8",
            "ContentTypeId": "D6mHJdtwBYsvtS2xCvXiww==",
            "GosterimSekli": "1",
            "OrderByAsc": "true",
            "page": f"{str(self.page)}"
        }

        if response.body.strip():    # check if we get an empty page
            load_more = scrapy.Request(url, method="POST", headers=self.headers, body=json.dumps(body))
            yield load_more
            self.page += 1

            data = response.xpath('//div[@class="ministry-announcements"]')
            for bilgi in data:
                gun = bilgi.xpath('.//div[@class = "day"]/text()').extract_first()  #day
                ay = bilgi.xpath('.//div[@class = "month"]/text()').extract_first() #month

                metin = bilgi.xpath('.//a[@class ="announce-text"]/text()').extract_first() #text

                yield {
                    'Ay:': ay,
                    'Gün': gun,
                    'Metin': metin.strip(),
                }