使用 selenium 和 scrapy 从 url 列表中抓取答案

【问题标题】：Scraping from list of urls using selenium and scrapy使用 selenium 和 scrapy 从 url 列表中抓取
【发布时间】：2017-11-12 10:35:41
【问题描述】：

我是 scrapy 和 Python 的新手。我正在尝试从使用 Selenium 和 scrapy 的 URL 列表中抓取。我试过这段代码：

class TechstarSpider(scrapy.Spider):

    name = "techstar"
    allowed_domains = ["techstar.com"]
    start_urls = [l.strip() for l in open('pages.txt').readlines()]

    def __init__(self, **kwargs):
        super(TechstarSpider, self).__init__(**kwargs)
        self.driver = webdriver.Firefox()

    def parse(self, response):
        self.driver.get(response.url)
        time.sleep(10)
        resp = TextResponse(url=self.driver.current_url, body=self.driver.page_source, encoding='utf-8')
        item = StartupweekendataItem()
        for data in resp.xpath('//*[@id="main"]/div[2]'):
            item['Title'] = data.xpath('//*[@id="main"]/div[2]/div[1]/div/div/ol/h3/text()')[0].extract(),
            item['Date'] = data.xpath('//*[@id="main"]/div[2]/div[1]/div/div/ol/h3/span/text()')[0].extract(),
            item['Judge1'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text()')[0].extract(),
            item['Judge1FB'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/a[@class="fa fa-facebook font-18"]/@href')[0].extract(),
            item['Judge1Linked'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/a[@class="fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Judge2'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][2]/div[2]/h2/text()')[0].extract(),
            item['Judge2FB'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][2]/div[2]/h2/a[@class="fa fa-facebook font-18"]/@href')[0].extract(),
            item['Judge2Linked'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/a[@class="fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Coach1'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][1]//h2//text()')[0].extract(),
            item['Coach1FB'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][1]//a[@class="fa fa-facebook font-18"]//@href')[0].extract(),
            item['Coach1Linked'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][1]//a[@class="fa fa-linkedin font-18"]//@href')[0].extract(),
            item['Coach2'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][2]//h2//text()')[0].extract(),
            item['Coach2FB'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][2]//a[@class="fa fa-facebook font-18"]//@href')[0].extract(),
            item['Coach2Linked'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][2]//a[@class="fa fa-linkedin font-18"]//@href')[0].extract(),
            item['Organizer1'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[1]/h5/text()')[0].extract(),
            item['Organizer1FB'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[1]/a["fa fa-facebook font-18"]/@href')[0].extract(),
            item['Organizer1Linked'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[1]/a["fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Organizer2'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[2]/h5/text()')[0].extract(),
            item['Organizer2FB'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[2]/a["fa fa-facebook font-18"]/@href')[0].extract(),
            item['Organizer2Linked'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[2]/a["fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Facilitator'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[1]//h2//text()")[0].extract(),
            item['FacilitatorFB'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[1]//a['fa fa-facebook font-18']//@href")[0].extract(),
            item['FacilitatorLinked'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[1]//a['fa fa-linkedin font-18']//@href")[0].extract(),
            item['Staff1'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[3]//h2//text()")[0].extract(),
            item['Staff1FB'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[3]//a[@class'fa fa-facebook font-18']//@href")[0].extract(),
            item['Staff1Linked'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[3]//a[@class='fa fa-linkedin font-18']//@href")[0].extract(),
            yield item

Pages.txt 包含这样的 url 列表：

http://communities.techstars.com/spain/madrid/startup-weekend/11425  
http://communities.techstars.com/usa/phlsw/startup-weekend/3334  
http://communities.techstars.com/bolivia/santacruz/startup-weekend/10815

但是它抛出了这个错误：

2017-11-12 12:38:26 [scrapy] ERROR: Spider error processing <GET http://communities.techstars.com/bolivia/santacruz/startup-weekend/10815> (referer: None)
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/scrapy/utils/defer.py", line 102, in iter_errback
    yield next(it)
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/offsite.py", line 29, in process_spider_output
    for x in result:
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/referer.py", line 22, in <genexpr>
    return (_set_referer(r) for r in result or ())
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/urllength.py", line 37, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/depth.py", line 58, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/home/habenn/Projects/Aleksandr/startupweekendata/startupweekendata/spiders/techstar.py", line 53, in parse
    item['Judge1'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text()')[0].extract(),
  File "/usr/lib/python2.7/site-packages/parsel/selector.py", line 61, in __getitem__
    o = super(SelectorList, self).__getitem__(pos)
IndexError: list index out of range

这是我想要实现的输出：

问题：它抛出一个错误，我该如何解决？和代码太长，你能帮我简化代码吗？任何帮助都会非常有帮助。

【问题讨论】：

您在 item['Judge1'] 处遇到错误，这意味着您正在尝试废弃甚至不存在的元素，您应该尝试用几个 url 进行废弃，然后逐渐查看您是什么错误得到

标签： python csv selenium xpath scrapy

【解决方案1】：

问题是使用[0] 访问选择器列表的第一个元素，更详细地说是访问选择器//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text() 的结果（参见底部第四行的错误消息）。

这是Judge1 的选择器。

查看您列表中第一页的源代码，我找不到带有id="event-judges" 的元素，所以它不可用。

如果您希望能够选择缺少一个或多个属性的项目（例如Judge1），我建议您使用方法extract_first() 而不是[0].extract()。

如果您的选择器不匹配任何元素，extract_first() 将返回 None，或者您可以使用 extract_first(default='no-judge') 定义另一个默认值。

对于Judge1 选择器，您可以这样使用它：

item['Judge1'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text()').extract_first(),

【讨论】：