如何使用scrapy管道直接将抓取的图像保存到数据库？

Question

我打算将使用 python scrapy spider 抓取的图像直接保存到我的 mysql 数据库中，而不是下载到本地文件系统中。如何修改我在 pipelines.py 上的代码以使下载的图像直接保存到数据库中？以下是我使用的一些代码。

注意：我遇到的主要问题是我无法使用 item['images'] 访问图像项内容（二进制图像文件格式），如 pipelines 所示.py.

items.py

import scrapy

class MyItem(scrapy.Item):
    image_urls = scrapy.Field()
    images = scrapy.Field()

settings.py

ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1}

管道.py

import mysql.connector
from decouple import config

class SavetoMySQL:
    def __init__(self):
        self.user = config("DB_USER")
        self.password = config("DB_PASSWORD")
        self.host = config("DB_HOST")
        self.port = config("DB_PORT")
    
    def connect_database(self):
        self.conn = mysql.connector.connect(host=self.host,
                                            user=self.user,
                                            password=self.password,
                                            port=self.port)
    def create_table(self):
        self.curr = self.conn.cursor()
        self.curr.execute("""CREATE TABLE IF NOT EXISTS myitem_db(
                                image_urls VARCHAR(300),
                                images LONGBLOB);
                            """)

    def store_db(self, item):
        insert_stmt = ("""INSERT INTO Book(image_urls, images) 
                                VALUES (%s,%s)""")
        data = (item['image_urls'],item['images']) # item["images"] cannot access and save image binary file format into database
        self.curr.execute(insert_stmt, data)

    def open_spider(self, spider):
        self.connect_database()
        self.create_table()

    def process_item(self, item, spider):
        self.store_db(item)
        return item
    
    def close_spider(self, spider):
        self.curr.close()
        self.conn.close()

Answer 1

下面的代码是我如何将图像（以二进制数据形式）直接保存到数据库中而不将它们下载到数据库中。

my_spider1.py

import scrapy
import os
from MyProject.items import MyItem
class MySpider(scrapy.Spider):
    name = 'my_spider1'
    allowed_domains = ['books.toscrape.com']
    start_urls = ['http://books.toscrape.com/']
    pipelines=["image_db"]
    custom_settings={"ITEM_PIPELINES" :{'MyProject.pipelines.MyItemPipeline':5}}

    def parse(self, response):
        books = response.xpath("//*[@class='image_container']") 
        for book in books:
            image_link = book.xpath(".//img/@src").get() 
            # scrapy.Request use absolute url path to send request
            # while response.follow use relative path to send request

            yield response.follow(image_link, callback=self.scrape_image) 

        #next_page_relative_url = response.xpath("//*[@class='next']/a/@href").get()
        #yield response.follow(next_page_relative_url, callback = self.parse)
        
    def scrape_image(self, response):
        item = MyItem()
        item['image_binary'] = response.body
        return item

items.py

class MyItem(scrapy.Item):
    image_binary = scrapy.Field()

管道.py

import os
import mysql.connector
from decouple import config

class MyItemPipeline:
    def __init__(self):
        # retrieve database user login data from .env file
        self.user = config("DB_USER")
        self.password = config("DB_PASSWORD")
        self.host = config("DB_HOST")
        self.port = config("DB_PORT")
        self.database = config("DB_NAME")
    
    def connect_database(self):
        # connect database with user login data
        self.conn = mysql.connector.connect(user=self.user,
                                            password=self.password,
                                            host=self.host,
                                            port=self.port,
                                            database=self.database)
    
    def create_table(self):
        # create cursor connection
        self.curr = self.conn.cursor()
        # create table in database if not exist
        self.curr.execute("""CREATE TABLE IF NOT EXISTS 
                            Images (id INT NOT NULL AUTO_INCREMENT,
                            Photo LONGBLOB NOT NULL,
                            PRIMARY KEY (id));""")
    
    def store_db(self, item):
        # store item (or data) into table of database
        insert_stmt = """INSERT INTO Images (Photo) VALUES(%s)"""
        data = (item["image_binary"],)
        self.curr.execute(insert_stmt, data) # must pass into tuple into second input argument
        self.conn.commit()

    def close_database(self):
        # close database cursor connection
        self.curr.close()
        # close database connection
        self.conn.close()

    def open_spider(self, spider):
        if "image_db" in getattr(spider, "pipelines",[]):
            self.connect_database()
            self.create_table()

    def process_item(self, item, spider):
        if "image_db" in getattr(spider, "pipelines",[]):
            self.store_db(item)
        return item

    def close_spider(self, spider):
        if "image_db" in getattr(spider, "pipelines",[]):
            self.close_database()

我还创建了一个函数来从数据库中检索图像并将其下载到本地文件系统中，如下所示。在执行以下代码之前不要忘记先连接mysql数据库（ps：参考pipelines.py）。

    def retrieve_image(self, id, **kwargs):
        select_stmt = "SELECT * FROM Images WHERE id = '{0}'".format(str(id))
        self.curr.execute(select_stmt)
        result = self.curr.fetchone()
        if kwargs.get("storage_path"):
            storage_path = kwargs.get("storage_path")
            os.makedirs(storage_path, exist_ok=True)
            with open(storage_path,"wb") as File:
                File.write(result)
                File.close()
        return result