Scrapy 检查抓取的 url 是否返回任何可下载的文件

Question

我是 Scrapy 的新手，到目前为止还没有找到任何帮助。

我想做一个小的scraper，它可以scraper页面上的所有url，然后一个一个地点击它们，如果Url返回任何可下载的文件然后下载任何扩展并将其保存到指定位置。 这是我编写的代码： items.py

import scrapy

class ZcrawlerItem(scrapy.Item):
    file = scrapy.Field()
    file_url = scrapy.Field()

spider.py

from scrapy import Selector
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request

DOMAIN = 'example.com'
URL = 'http://%s' % DOMAIN
from crawler.items import CrawlerItem


class MycrawlerSpider(CrawlSpider):
    name = "mycrawler"
    allowed_domains = [DOMAIN]
    start_urls = [
        URL
    ]
    def parse_dir_contents(self, response):
        print(response.headers)
        item = CrawlerItem()
        item['file_url'] = response.url
        return item       

    def parse(self, response):
        hxs = Selector(response)
        for url in hxs.xpath('//a/@href').extract():
            if (url.startswith('http://') or url.startswith('https://')):
                yield Request(url, callback=self.parse_dir_contents)
        for url in hxs.xpath('//iframe/@src ').extract():
            yield Request(url, callback=self.parse_dir_contents)

我面临的问题是 parse_dir_contents 没有显示标题，因此很难检查响应数据是任何可下载的文件还是只是内容。

顺便说一句，我正在使用 Scrapy 1.1.0 和 Python 3.4

任何帮助将不胜感激！

Answer 1

所以经过一些研发后，我找到了解决方案，这是更新后的 spider.py

from scrapy import Selector
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request

DOMAIN = 'example.com'
URL = 'http://%s' % DOMAIN
from crawler.items import CrawlerItem


class MycrawlerSpider(CrawlSpider):
    name = "mycrawler"
    allowed_domains = ''
    allowed_mime_type = [b'application/zip', b'application/x-msdownload', b'application/pdf', b'image/jpeg', b'image/jpg',
                     b'image/png', b'application/octet-stream']
    start_urls = [
        URL
    ]
    def parse(self, response):
        hxs = Selector(response)
        Urls = ''
        for url in hxs.xpath('//a/@href').extract():
            if (url.startswith('http://') or url.startswith('https://')):
                yield Request(url, callback=self.parse_item)
            elif 'javascript' not in url:
                new_url = urljoin(response.url, url.strip())
                print("New url : ", new_url)
                yield Request(new_url, callback=self.parse_item)
        for url in hxs.xpath('//iframe/@src ').extract():
            Urls += str(url) + ", "
            yield Request(url, callback=self.parse_item)

    def parse_item(self, response):
        if response.headers['Content-Type'] in self.allowed_mime_type:
            item = ZcrawlerItem()
            item['file_urls'] = response.url
            item['referer'] = str(response.request.headers['referer'].decode("utf-8"))
            yield item
        else:
            self.logger.info('Not found any allowed file type, lets try next page : %s', response.url)
            # self.process_next_url(response)
            yield Request(response.url, callback=self.parse, dont_filter=True)

然后输出将被传递到 pipeline.py 我在 PostgreSql 中保存信息并下载文件

import json
import scrapy
import psycopg2
import datetime
import hashlib
import os

try:
    import urllib.request as urllib2
except ImportError:
    import urllib2

FILES_STORE = '<location to save files>'


class Pipeline(object):
    def __init__(self):
        self.conn = psycopg2.connect(user="postgres", password="pass",
                                     dbname="db_name",
                                     host='localhost')

    def process_item(self, item, spider):
        item['path'] = self.write_to_file(item['file_urls'])
        cur = self.conn.cursor()
        cur.execute('''
                insert into scrape ( file_url, referer,path,created_date)
                values (%s, %s,%s, %s);
                ''', [
            item['file_urls'],
            item['referer'],
            item['path'],
            datetime.datetime.now()])
        self.conn.commit()
        return item

    def write_to_file(self, url):
        response = urllib2.urlopen(url)
        directory = FILES_STORE + str(hashlib.md5(url.encode('utf-8')).hexdigest()) + "/"
        if not os.path.exists(directory):
            os.makedirs(directory)
        file_name = url.split('/')[-1]
        with open(directory + str(file_name), "wb") as handle:
            handle.write(response.read())
        return directory + str(file_name)

希望这会对某人有所帮助，干杯（y）