05 pipeline 管道

管道接收怕从爬取到的item, 并对item进行过滤, 去重, 清洗等工作, 然后对item进行持久化存储

 

去重管道:
　　如果图片名称相同, 我们就认为图片重复, 删除该项目

from scrapy.exceptions import DropItem
class DuplicatePipeline(object):
    def __init__(self):
        self.fingerprints = set()   # 我们用set作为一个数据指纹,
    
    def process_item(self, item, spider):
        if item[\'picname\'] in self.fingerprints:
            raise DropItem
        self.fingerprints.add(item[\'picname\'])
        return item

 

# 将数据存储到csv文件中

import csv

class CsvFeedPipeline(object):
    def __init__(self):
        self.fp = open(\'data.csv\',\'a\',encoding=\'utf8\')
        filednames = [\'classname\', \'picname\', \'picurl\']
        self.writer = csv.DictWriter(self.fp, fieldnames = fieldnames)
        self.writer.writeheader()
    
    def process_item(self, item, spider):
        self.writer.writerow( dict(item) )
        return item
       
    def close_spider(self, spider):
        self.fp.close()

# 过滤, 删除掉与游戏相关的新闻内容

class BlockGamePipeline(object):
    def process_item( self, item, spider ):
        filter_key = \'游戏\'
        if filter_key in (item[\'title\']).encode(\'utf8\'):
            raise DropItem
        return item

# 加工性管道:

class ProductPricePipeline():
    def process_item(self, item, spider):
        item[\'total\'] = float( item[\'price\'] ) * float( item[\'count\'] )
        return item

# 存储型管道

class json 
class JsonFeedPipeline:
    def __init__(self):
        self.json_file = open(\'feed.json\',\'wt\')
        self.json_file.write(\'[\n\')
    
    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + \',\n\'
        self.json_file.write(line)
    
    def close_spider(self, spider):
        self.json_file.write( \'\n]\' )
        self.json_file.close()