抓取多个下一页

Question

我想抓取每一页。我找到了一种使用scrapy shell 的方法，但我不知道我的蜘蛛是否会遍历每一页或只遍历下一页；我不太确定如何实现。

alphabet = string.ascii_uppercase
each_link = '.' +  alphabet 
each_url =  ["https://myanimelist.net/anime.php?letter={0}".format(i) for i in each_link]
#sub_page_of_url = [[str(url)+"&show{0}".format(i) for i in range(50, 2000, 50)] for url in each_url] #start/stop/steps
#full_url =  each_url + sub_page_of_url

class AnimeScraper_Spider(scrapy.Spider):
    name = "Anime"

    def start_requests(self):
        for url in each_url:
            yield scrapy.Request(url=url, callback= self.parse)

    def parse(self, response):
     next_page_url = response.xpath(
        "//div[@class='bgColor1']//a[text()='Next']/@href").extract_first()

     for href in response.css('#content > div.normal_header.clearfix.pt16 > div > div > span > a:nth-child(1)') :
        url = response.urljoin(href.extract())
        yield Request(url, callback = self.parse_anime)
    yield Request(next_page_url, callback=self.parse)

    def parse_anime(self, response):
        for tr_sel in response.css('div.js-categories-seasonal tr ~ tr'):
            return {
            "title" :  tr_sel.css('a[id] strong::text').extract_first().strip(),
            "synopsis" : tr_sel.css("div.pt4::text").extract_first(),
            "type_" : tr_sel.css('td:nth-child(3)::text').extract_first().strip(),
            "episodes" : tr_sel.css('td:nth-child(4)::text').extract_first().strip(), 
            "rating" : tr_sel.css('td:nth-child(5)::text').extract_first().strip()
            }

Answer 1

我认为你尝试的东西太复杂了，应该很简单：

1. 从主页开始
2. 识别所有以特定字母开头的页面
3. 对于这些页面中的每一个，获取所有下一个链接并重复

看起来像这样：

import string

import scrapy
from scrapy import Request

class AnimeSpider(scrapy.Spider):

    name = "Anime"
    start_urls = ['https://myanimelist.net/anime.php']

    def parse(self, response):
        xp = "//div[@id='horiznav_nav']//li/a/@href"
        return (Request(url, callback=self.parse_anime_list_page) for url in response.xpath(xp).extract())

    def parse_anime_list_page(self, response):
        for tr_sel in response.css('div.js-categories-seasonal tr ~ tr'):
            yield {
                "title":  tr_sel.css('a[id] strong::text').extract_first().strip(),
                "synopsis": tr_sel.css("div.pt4::text").extract_first(),
                "type_": tr_sel.css('td:nth-child(3)::text').extract_first().strip(),
                "episodes": tr_sel.css('td:nth-child(4)::text').extract_first().strip(), 
                "rating": tr_sel.css('td:nth-child(5)::text').extract_first().strip(),
            }

        next_urls = response.xpath("//div[@class='spaceit']//a/@href").extract()
        for next_url in next_urls:
            yield Request(response.urljoin(next_url), callback=self.parse_anime_list_page)