如何将数据从一个类传递到 Scrapy 类

Question

我需要将 URL、用户名和密码从一个类传递给 Scrapy 类以执行网络抓取。

import quotes as q
import scrapy
from scrapy.crawler import CrawlerProcess
class ValidateURL:
    def checkURL(self,urls):
        try:    
            if(urls):
                for key, value in urls.items():
                    if value['login_details']:
                        self.runScrap(value)                                                      
        except:
            return False

    def runScrap(self,data):       
            if data:
               ''' data= "url_4": {
                                                "url": ("https://quotes.toscrape.com/login",),
                                                "fields_in_response": ["Quotes to Scrape","Login"],
                                                "login_details": {"name":"foobar","pwd":"foobar" },
                                                "fields_in_main_page": ["Quotes to Scrape","Top Ten tags"]
                                }

               '''
               process = CrawlerProcess()
               process.crawl(q.QuotesSpider, start_urls=data['url'])
               process.start()

而scrapy类是

# -*- coding: utf-8 -*-
from scrapy import Spider
from scrapy.http import FormRequest
from scrapy.utils.response import open_in_browser
import sys
import logging
from bs4 import BeautifulSoup
# import scrapy
# from scrapy.crawler import CrawlerProcess

logging.basicConfig(filename='app.log',level=logging.INFO)

class QuotesSpider(Spider):
    name = 'quotes'
    start_urls = ('https://quotes.toscrape.com/login',)



    def parse(self, response):
        # print(self.req['url'])
        print('/'*100)
        self.start_urls=self.login_url
        # print(type(self.login_url))
        inputs =response.xpath('//form//input').extract()
        soup_dict={}
        for key,i in enumerate(inputs):
            soup = BeautifulSoup(i, 'html.parser')
            inp_type   =   soup.input['type'] if soup.input.has_attr('type') else None
            inp_value  =   soup.input['value'] if soup.input.has_attr('value')  else None
            inp_name   =   soup.input['name'] if soup.input.has_attr('name')  else None
            soup_dict[key]= {'name':inp_name,'value':inp_value,'type':inp_type}  
        token = response.xpath('//*[@name="csrf_token"]/@value').extract_first()
        return FormRequest.from_response(response,
                                         formdata={'csrf_token': token,
                                                   'password': 'foobar',
                                                   'username': 'foobar'},
                                         callback=self.scrape_pages)

    def fetch_form_data(self,response):

        if all(field in response for field in self.fields_in_response):
            inputs =response.xpath('//form//input').extract()
            soup_dict={}
            for key,i in enumerate(inputs):
                soup = BeautifulSoup(i, 'html.parser')
                inp_type   =   soup.input['type'] if soup.input.has_attr('type') else None
                inp_value  =   soup.input['value'] if soup.input.has_attr('value')  else None
                inp_name   =   soup.input['name'] if soup.input.has_attr('name')  else None
                soup_dict[key]= {'name':inp_name,'value':inp_value,'type':inp_type}  


    def scrape_pages(self, response):
        open_in_browser(response)

        # Complete your code here to scrape the pages that you are redirected to after logging in

        # ....
        # ....

但是，我无法更新类变量 start_urls。使用来自 ValidateURL 类的传递变量。我尝试在 QuotesSpider 类中使用 init ，但没有奏效。实际上 start_urls 是 BaseClass(Spider) 的类成员。有人可以帮我知道如何更新基类的类变量

有人可以建议缺少什么

Answer 1

你可以像这样通过抓取命令将参数传递给蜘蛛

process.crawl(q.QuotesSpider, first='James', last='Bond')