scrapy下载大文件

1、spider.py

# -*- coding: utf-8 -*-
import scrapy
from ..items import BigfileItem


class ChoutiSpider(scrapy.Spider):
    name = "chouti"
    allowed_domains = ["chouti.com"]
    start_urls = (
        'http://www.chouti.com/',
    )

    def parse(self, response):
        yield BigfileItem(url='https://wx1.sinaimg.cn/large/8a04ef95ly1frbvy0d2opj20j00tm0vt.jpg',type='file',file_name='xx.jpg')

View Code

2022-03-07
2022-01-01
2021-09-21
2022-02-07
2021-12-31
2022-12-23
2022-12-23
2022-12-23