网页抓取规则创建

Question

我在这个页面上：http://www.metacritic.com/browse/games/title/ps4/a?view=condensed

我想进入每个项目并获得开发者和流派，但我的代码似乎不起作用。

比如我想进入这个页面：http://www.metacritic.com/game/playstation-4/angry-birds-star-wars

然后离开它并继续其余的做同样的事情并添加到数据库中。我可以在我的代码中进行哪些更改以使其正常工作？现在数据库用于开发，类型为空，但它会获取其余数据，所以它就像它永远不会进入 parse_Game

我还在 parseGame 中添加了打印语句，但它们都没有打印

from scrapy.spider import BaseSpider
from scrapy.selector import Selector
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector
from metacritic.items import MetacriticItem
import MySQLdb
import re
from string import lowercase

class MetacriticSpider(BaseSpider):
def start_requests(self):
    #iterate through ps4 pages
    for c in lowercase:
        for i in range(self.max_id):
            yield Request('http://www.metacritic.com/browse/games/title/ps4/{0}?page={1}'.format(c, i), callback = self.parseps4)

    #gets the developer and genre of a game
def parseGame(self, response):

    print("Here")

    item = response.meta['item']

    db1 = MySQLdb.connect("localhost", "root", "andy", "metacritic")
    cursor = db1.cursor()
    hxs = HtmlXPathSelector(response)   
    sites = hxs.select('//div[@class="product_wrap"]')
    items = []

    item['dev'] = site.xpath('.//span[contains(@class, "summary_detail developer")]/span[1]/text()').extract()
    item['genre'] = site.xpath('.//span[contains(@class, "summary_detail product_genre")]/span[1]/text()').extract()    

    cursor.execute("INSERT INTO ps4 (dev, genre) VALUES (%s,%s)",[item['dev'][0],item['genre'][0]])
    items.append(item)

    print item['dev']
    print item['genre']

def parseps4(self, response):
    #some local variables
    db1 = MySQLdb.connect("localhost", "root", "andy", "metacritic")
    cursor = db1.cursor()
    hxs = HtmlXPathSelector(response)   
    sites = hxs.select('//div[@class="product_wrap"]')
    items = []

    #iterates through each site
    for site in sites:
        with db1:
            item = MetacriticItem()

            #sets the item
            item['title'] = site.xpath('.//div[contains(@class, "basic_stat product_title")]/a/text()').extract()
            item['cscore'] = site.xpath('.//div[contains(@class, "basic_stat product_score brief_metascore")]/div[1]/text()').extract() 
            item['uscore'] = site.xpath('.//div/ul/li/span[contains(@class, "data textscore")]/text()').extract()
            item['release'] = site.xpath('.//li[contains(@class, "stat release_date full_release_date")]/span[2]/text()').extract()

            #some processing to check if there is a score attached, if there is, it adds it to the database
            if ("tbd" in item['cscore'][0] and "tbd" not in item['uscore'][0]) or ("tbd" not in item['cscore'][0] and "tbd" in item['uscore'][0]) or ("tbd" not in item['cscore'][0] and "tbd" not in item['uscore'][0]):
                cursor.execute("INSERT INTO ps4 (title, criticalscore, userscore, releasedate) VALUES (%s,%s,%s, %s)",[(' '.join(item['title'][0].split())).replace("(PS4)","",1),item['cscore'][0],item['uscore'][0],item['release'][0]])
                items.append(item)

            itemLink = site.xpath('.//div[contains(@class, "basic_stat product_title")]/a/@href' ).extract()

            req = Request('http://www.metacritic.com' +  itemLink[0], callback = self.parseGame)
            req.meta['item'] = item

Answer 1

代码中的几个问题：

• 元参数应该包含一个字典{'item': item}
• HtmlXPathSelector 已弃用 - 请改用 Selector
• 我认为您不应该在蜘蛛内部执行 mysql 插入 - 改用数据库管道：
  • Writing items to a MySQL database in Scrapy
• 您需要获取extract() 调用的第一项并对其执行strip()（这将有助于在字段中包含字符串，而不是列表，并且没有前导和尾随空格和换行符）

这是没有mysql相关调用的代码：

from string import lowercase

from scrapy.item import Field, Item
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector, Selector

from metacritic.items import MetacriticItem


class MetacriticSpider(BaseSpider):
    name = 'metacritic'
    allowed_domains = ['metacritic.com']

    max_id = 1 # your max_id value goes here!!!

    def start_requests(self):
        for c in lowercase:
            for i in range(self.max_id):
                yield Request('http://www.metacritic.com/browse/games/title/ps4/{0}?page={1}'.format(c, i), callback=self.parseps4)

    def parseGame(self, response):
        item = response.meta['item']
        hxs = HtmlXPathSelector(response)
        site = hxs.select('//div[@class="product_wrap"]')

        # get additional data!!!

        yield item

    def parseps4(self, response):
        hxs = Selector(response)
        sites = hxs.select('//div[@class="product_wrap"]')
        for site in sites:
            item = MetacriticItem()
            item['title'] = site.xpath('.//div[contains(@class, "basic_stat product_title")]/a/text()').extract()[0].strip()
            item['cscore'] = site.xpath('.//div[contains(@class, "basic_stat product_score brief_metascore")]/div[1]/text()').extract()[0].strip()
            item['uscore'] = site.xpath('.//div/ul/li/span[contains(@class, "data textscore")]/text()').extract()[0].strip()
            item['release'] = site.xpath('.//li[contains(@class, "stat release_date full_release_date")]/span[2]/text()').extract()[0].strip()

            link = site.xpath('.//div[contains(@class, "basic_stat product_title")]/a/@href').extract()[0]
            yield Request('http://www.metacritic.com/' + link, meta={'item': item}, callback=self.parseGame)

它对我有用 - 我在控制台上看到来自 parseGame() 的已生成项目。

确保它首先产生项目，然后查看!!! cmets - 相应地填写这些行。

之后，如果您在控制台上看到项目，请尝试创建数据库管道以将项目写入 mysql。