有没有更有效的方法将这个 html 表刮到数据框中？

Question

我编写这段代码是为了从here 的游戏数据库中抓取数据。

它实际上工作得很好，我能够抓取数据并适当地操作它，但似乎我必须以一种低效的方式来做这件事，因为它会吐出我想要的数据 50 倍......让我解释一下：

下面是我的爬虫的代码：

class JsonWriterPipeline(object):

    def open_spider(self, spider):
        self.file = open('gamesresult.jl', 'w')

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + "\n"
        self.file.write(line)
        return item


class VGSpider(scrapy.Spider):
    name = "game_spider"
    start_urls = ['https://www.vgchartz.com/gamedb/']

    custom_settings = {
        'LOG_LEVEL': logging.WARNING,
        'ITEM_PIPELINES': {'__main__.JsonWriterPipeline': 1}, # Used for pipeline 1
        'FEED_FORMAT':'json',                                 # Used for pipeline 2
        'FEED_URI': 'gamesresult.json'                        # Used for pipeline 2
    }

    def parse(self, response):
        TABLE_SELECTOR = ('//*[@id="generalBody"]/table[1]/tr')

        table = response.xpath(TABLE_SELECTOR)


        IMAGE_SELECTOR = '//td[2]/div/a/div/img/@src'
        TITLE_SELECTOR = '//td[3]/a/text()'
        CONSOLE_SELECTOR = '//td[4]/img/@alt'
        PUBLISHER_SELECTOR = '//td[5]/text()'
        VGSCORE_SELECTOR = '//td[6]/text()'
        CRITIC_SELECTOR = '//td[7]/text()'
        USER_SELECTOR = '//td[8]/text()'
        TOTALSHIPPED_SELECTOR = '//td[9]/text()'
        RELEASE_SELECTOR = '//td[10]/text()'
        UPDATE_SELECTOR = '//td[11]/text()'

        yield {
            'img' : table.xpath(IMAGE_SELECTOR).extract(),
            'title' : table.xpath(TITLE_SELECTOR).extract(),
            'console' : table.xpath(CONSOLE_SELECTOR).extract(),
            'publisher' : table.xpath(PUBLISHER_SELECTOR).extract(),
            'vg_score' : table.xpath(VGSCORE_SELECTOR).extract(),
            'critic_score' : table.xpath(CRITIC_SELECTOR).extract(),
            'user_score' : table.xpath(USER_SELECTOR).extract(),
            'total_shipped' : table.xpath(TOTALSHIPPED_SELECTOR).extract(),
            'release_date' : table.xpath(RELEASE_SELECTOR).extract(),
            'last_update' : table.xpath(UPDATE_SELECTOR).extract()
            }

process = CrawlerProcess(get_project_settings())

process.crawl(VGSpider)
process.start(stop_after_crawl=True)

这是页面中的源 html：

<td style="font-size:12pt;">    <a href="https://www.vgchartz.com/game/226034/pokemon/?region=All" >Pokemon    </a>  </td>
            <td align="center">

<img src="/images/consoles/Series_b.png" alt="Series">
            </td>  
<td width="100">Nintendo  </td>  
<td align="center">N/A  </td>  
<td align="center">N/A  </td>  
<td align="center">N/A  </td>  
<td align="center">365.60m</td>  
<td width="75" align="center">28th Sep 98  </td>  
<td width="75" align="center">03rd Feb 20</td></tr><tr style="background-image:url(../imgs/chartBar_alt_large.gif); height:70px">
            <td>2</td>
            <td>
              <div id="photo3">
                <a href="/games/game.php?id=226187&region=All">
                  <div style="height:60px; width:60px; overflow:hidden;">          <img src="/games/boxart/full_4441628AmericaFrontccc.jpg" border="0" width="60" alt="Boxart Missing">
                  </div>
                </a>
              </div>
            </td>  
<td style="font-size:12pt;">    
<a href="https://www.vgchartz.com/game/226187/super-mario/?region=All" >Super Mario    </a>  </td>
            <td align="center">

<img src="/images/consoles/Series_b.png" alt="Series">
            </td>  
<td width="100">Nintendo  </td>  
<td align="center">N/A  </td>  
<td align="center">N/A  </td>  
<td align="center">N/A  </td>  
<td align="center">356.59m</td>  
<td width="75" align="center">20th Jul 83  </td>  
<td width="75" align="center">20th Feb 20</td></tr><tr style="background-image:url(../imgs/chartBar_large.gif); height:70px">

这会输出到适当的 json 文件中，其中包含基本正确的信息......在一定程度上。基本上，我得到如下内容：

{"img": ["/games/boxart/full_4261443AmericaFrontccc.jpg", "/games/boxart/full_4441628AmericaFrontccc.jpg", "/games/boxart/full_9594883AmericaFrontccc.png", "/games/boxart/full_7292379AmericaFrontccc.jpg", "/games/boxart/full_4446063AmericaFrontccc.jpg", "/games/boxart/full_1356677AmericaFrontccc.jpg", "/games/boxart/full_5967679AmericaFrontccc.jpg", "/games/boxart/full_2881264AmericaFrontccc.jpg", "/games/boxart/full_6200477AmericaFrontccc.jpg", "/games/boxart/full_3250466AmericaFrontccc.jpg", "/games/boxart/full_4399408AmericaFrontccc.jpg"...
}

现在，这不会那么糟糕，因为我会在列表中有一整列，但它实际上包括列中的所有图像，但每个图像都重复了 50 次。我的代码中是否缺少一些循环？为什么此信息重复 50 次？我绝对可以在另一端提取它，但我只需要选择每个列表中的前 50 个元素。看来我只是在这里做了一些低效的事情。

Answer 1

你需要使用relative XPath：

def parse(self, response):
    IMAGE_SELECTOR = './/td[2]/div/a/div/img/@src'
    TITLE_SELECTOR = './/td[3]/a/text()'
    CONSOLE_SELECTOR = './/td[4]/img/@alt'
    PUBLISHER_SELECTOR = './/td[5]/text()'
    VGSCORE_SELECTOR = './/td[6]/text()'
    CRITIC_SELECTOR = './/td[7]/text()'
    USER_SELECTOR = './/td[8]/text()'
    TOTALSHIPPED_SELECTOR = './/td[9]/text()'
    RELEASE_SELECTOR = './/td[10]/text()'
    UPDATE_SELECTOR = './/td[11]/text()'
    for row in response.xpath('//*[@id="generalBody"]/table[1]/tr'):
        yield {
            'img' : row.xpath(IMAGE_SELECTOR).extract(),
            'title' : row.xpath(TITLE_SELECTOR).extract(),
            'console' : row.xpath(CONSOLE_SELECTOR).extract(),
            'publisher' : row.xpath(PUBLISHER_SELECTOR).extract(),
            'vg_score' : row.xpath(VGSCORE_SELECTOR).extract(),
            'critic_score' : row.xpath(CRITIC_SELECTOR).extract(),
            'user_score' : row.xpath(USER_SELECTOR).extract(),
            'total_shipped' : row.xpath(TOTALSHIPPED_SELECTOR).extract(),
            'release_date' : row.xpath(RELEASE_SELECTOR).extract(),
            'last_update' : row.xpath(UPDATE_SELECTOR).extract()
        }