使用 scrapy StatsCollector 将抓取统计信息写入数据库

Question

我对 Stackoverflow 和编码世界完全陌生，所以如果这个问题的格式不正确，请提前道歉！

我已经构建了爬取网站以获取关键信息然后将输出写入数据库的蜘蛛。我现在想将每次抓取的一些关键统计信息写入数据库中的单独表。

我尝试使用 scrapy Stats Collector，但到目前为止还无法将任何内容写入数据库，并且我正在努力寻找任何这样的示例。

谁能帮我指出正确的方向？下面是一个尝试只写一列的示例（蜘蛛在抓取期间发现的总条目）。爬虫成功了，但我的 statscollector 没有写任何东西。具体来说，我很想知道：

1) 我是否以正确的方式处理这个问题，尤其是当我为项目添加更多蜘蛛和更复杂时

2) 我如何每次抓取只向数据库写入一个统计数据（例如，找到的产品总数）。目前我相信每次调用 parse_main_item 时它都会尝试写入数据库，对于几十个子类别来说，这将是几十次。

提前感谢您的帮助！

# One of my spiders
class Retailer1(scrapy.Spider):
    name = 'R1_monitor'
    allowed_domains = ['example.com']
    start_urls = ['https://www.example.com']

    def __init__(self):
        self.declare_xpath()
    def declare_xpath(self):
        #site specific xpaths

    def parse(self, response):
# returns all the category links in the home page menu
            yield scrapy.Request(url=url, callback=self.parse_category, )

    def parse_category(self, response):
# returns all the sub category links sitting beneath each category
            yield scrapy.Request(url, callback=self.parse_page_url, )

    def parse_page_url(self, response):
# uses a product count in the html to return a url that will display all products
            yield scrapy.Request(paginated_url, callback=self.parse_main_item, dont_filter=True,)

    def parse_main_item(self, response):
        items_by_id = {}

        for product in zip(
                #zips things like product name, price etc together ):

            item = PriceMonitorItem()
            item['product_id'] = product[0]
            item['product_name'] = product[1]
            item['product_url'] = product[2]
            item['product_image'] = product[3]
            item['retailer_site'] = host
            items_by_id[product[0]] = item

            #yield all of the items from the main parse function
            for items in items_by_id.values():
                yield items

#Here I'd like to count the total number of items that are successfully parsed and sent to the pipeline
        self.total_items = 0
        self.total_items += len(items_by_id)

        total_entries = self.crawler.stats.set_value("Total items scraped", self.total_items, spider=DdcMonitorSpider)
        yield total_entries

In pipelines.py

class PriceCrawlerStatsPipeline(object):
    def __init__(self):
        #db session is created

    def process_item(self, total_entries, spider):
        session = self.Session()
        **self.scrape_stats = ScrapeStats()
        self.scrape_stats.total_entries = total_entries**
        columns_to_dict = lambda obj: {c.name: getattr(obj, c.name) for c in obj.__table__.columns}

        try:
            insert_stmt = insert(ScrapeStats).values(
                **columns_to_dict(self.scrape_stats))

            on_duplicate_key_stmt = insert_stmt.on_duplicate_key_update(
                {'date_scraped': date.today(), 'time_scraped': pytz.timezone('GMT')} )

            session.execute(on_duplicate_key_stmt)
            session.commit()
        return total_entries

最后，这是我在日志中得到的回报

Traceback (most recent call last):
  File "C:\Users\sibla\Documents\CodingNomads\price_crawler\price_monitor\price_monitor\pipelines.py", line 112, in process_item
    session.execute(on_duplicate_key_stmt)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\orm\session.py", line 1269, in execute
    clause, params or {}
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\engine\base.py", line 982, in execute
    return meth(self, multiparams, params)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\elements.py", line 287, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1092, in _execute_clauseelement
    else None,
  File "<string>", line 1, in <lambda>
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\elements.py", line 462, in compile
    return self._compiler(dialect, bind=bind, **kw)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\elements.py", line 468, in _compiler
    return dialect.statement_compiler(dialect, self, **kw)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\compiler.py", line 571, in __init__
    Compiled.__init__(self, dialect, statement, **kwargs)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\compiler.py", line 319, in __init__
    self.string = self.process(self.statement, **compile_kwargs)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\compiler.py", line 350, in process
    return obj._compiler_dispatch(self, **kwargs)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\visitors.py", line 92, in _compiler_dispatch
    return meth(self, **kw)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\compiler.py", line 2405, in visit_insert
    self, insert_stmt, crud.ISINSERT, **kw
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\crud.py", line 62, in _setup_crud_params
    return _get_crud_params(compiler, stmt, **kw)
  File "c:\users\sibla\documents\codingnomads\price_crawler\venv\lib\site-packages\sqlalchemy\sql\crud.py", line 177, in _get_crud_params
    % (", ".join("%s" % c for c in check))
**sqlalchemy.exc.CompileError: Unconsumed column names: id, total_entries**

Answer 1

每次调用您的代码的原因是因为它是 pipelines.py 文件的一部分，并且您正在调用 process_item，该文件在每次处理项目时运行。

我的解决方案是在middleware.py 文件中创建一个新类并调用spider_closed 函数。该函数在蜘蛛关闭时运行，并包含大量蜘蛛定义的信息。

我的代码在下面，是middleware.py 文件的一部分。这应该可以指导您如何实现自己。

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware

from scrapy import signals
import datetime
from sqlalchemy.orm import sessionmaker
from <botname>.models import Jobs, db_connect

class SaveJobInDatabase(object):

    @classmethod
    def from_crawler(cls, crawler):
        s = cls(crawler)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def spider_closed(self, spider, reason):
        db_arg = getattr(spider,'addtodatabase','true')
        stats = spider.crawler.stats.get_stats()

        job = {}
        job['job_id'] = self.get_jobid()
        job['start_timestamp'] = stats.get('start_time').strftime('%Y-%m-%d %H:%M:%S')
        job['end_timestamp'] = stats.get('finish_time').strftime('%Y-%m-%d %H:%M:%S')
        job['spider_name'] = spider.name
        job['items_scraped'] = stats.get('item_scraped_count')
        job['items_dropped'] = stats.get('item_dropped_count')
        job['finish_reason'] = stats.get('finish_reason')

        """
        Save jobs in the database.
        This method is called whenever the spider is finished (closed)
        """
        session = self.Session()
        job_add = Jobs(**job)
        session.add(job_add)
        session.commit()

并确保您更新了您的 settings.py 并添加了课程：

SPIDER_MIDDLEWARES = {
    'botname.middlewares.SaveJobInDatabase': 300,
}