使用抓取的图像名称创建链接

Question

我从同一个网站抓取了一个网站和图片。 我为图像使用自定义管道。 我想从一个 URL（例如“www.urls.com/image”）在我的列表中创建一个链接，并添加相应图像的名称。

我的蜘蛛.py：

import json
import scrapy
import re
import pkgutil

from scrapy.loader import ItemLoader
from auctions_results.items import AuctionItem
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem

from datetime import datetime

class Spider(scrapy.Spider):

    name = 'results'

    def __init__(self, *args, **kwargs):
        data_file = pkgutil.get_data(
            "auctions_results", "json/input/scrape_demo_db.json")
        self.data = json.loads(data_file)

    def start_requests(self):
        for item in self.data:
            request = scrapy.Request(item['gm_url'], callback=self.parse)
            request.meta['item'] = item
            yield request

    def parse(self, response):
        item = response.meta['item']
        item['results'] = []

        for caritem in response.css("div.car-item-border"):
            data = AuctionItem()

            data["marque"] = caritem.css("div.make::text").extract_first().strip().split(" ", 2)[1]

            data["auction_house"] = caritem.css("div.auctionHouse::text").extract_first().split("-", 1)[0].strip()
            data["auction_country"] = caritem.css("div.auctionHouse::text").extract_first().rsplit(",", 1)[1].strip()
            data["auction_date"] = caritem.css("div.date::text").extract_first().replace(",", "").strip()

            data["image_urls"] = caritem.css("div.view-auction a img::attr(src)").extract_first()
            data["image_path"] = "urls.com/" + NAME OF THE SCRAPED IMAGE

            item['results'].append(data)

        yield item

我的图像管道：

import scrapy

from scrapy.pipelines.images import ImagesPipeline

class DownloadImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for result in item['results']:
            image_url = result['image_urls']
            if image_url is not None:
                request = scrapy.Request(url=image_url)
                yield request

输出样本：

[{
"gm_url": "url",
"results": [{
    "marque": "ferrari",
    "auction_house": "auction",
    "auction_country": "japan",
    "auction_date": "2019",
    "image_urls": "imgurl"
},
{
    "marque": "porsche",
    "auction_house": "auction2",
    "auction_country": "gb",
    "auction_date": "2018",
    "image_urls": "imgurl2"
}],
"images": [
{
    "url": "imgurl",
    "path": "full/0c2d1678d25dfed305d9a1f64366f005f2c3a138.jpg",
    "checksum": "e33f726186f72e2e9810555615964fb1"
},
{
    "url": "imgurl2",
    "path": "full/758825eeb421e6ef49e73bd6af2e0eeb9090cfe1.jpg",
    "checksum": "c172e40ab961108bdd5be1874d688f62"
}]

现在我想在我的 URL 的每个结果中添加一个键值“image_path”在我的示例中，并添加图像的名称以创建另一个 URL。 在 Python 中可以做到这一点吗？

Answer 1

我现在无法测试，但您可以在 parse() 中创建自己的名称，然后在 Pipeline 中使用它

在Spider:

• 在parse() 中创建filename 和url：

      data["filename"] = '... create your filename ...'
  
      data["image_path"] = "urls.com/" + data["filename"]
  
      item['results'].append(data)
  

在DownloadImagesPipeline:

• 在get_media_requests 中使用这个filename 在meta= 在Request() 中

      if image_url is not None:
  
          filename = result['filename']
  
          request = scrapy.Request(url=image_url, meta={'filename:', filename} )
  
          yield request
  

• 创建file_path，它为下载的文件生成名称并使用filename下载的文件

  def file_path(self, request, response=None, info=None):
      return request.meta['filename']
  

---

或多或少的所有代码

def parse(self, response):
        item = response.meta['item']
        item['results'] = []

        for caritem in response.css("div.car-item-border"):
            data = AuctionItem()

            data["marque"] = caritem.css("div.make::text").extract_first().strip().split(" ", 2)[1]

            data["auction_house"] = caritem.css("div.auctionHouse::text").extract_first().split("-", 1)[0].strip()
            data["auction_country"] = caritem.css("div.auctionHouse::text").extract_first().rsplit(",", 1)[1].strip()
            data["auction_date"] = caritem.css("div.date::text").extract_first().replace(",", "").strip()

            data["image_urls"] = caritem.css("div.view-auction a img::attr(src)").extract_first()

            data["filename"] = '... your filename ...'

            data["image_path"] = "urls.com/" + data["filename"]

            item['results'].append(data)


class DownloadImagesPipeline(ImagesPipeline):

    def get_media_requests(self, item, info):
        for result in item['results']:
            image_url = result['image_urls']
            if image_url is not None:

                filename = result['filename']

                request = scrapy.Request(url=image_url, meta={'filename:', filename} )
                yield request

    def file_path(self, request, response=None, info=None):
        return request.meta['filename']