起点作为主流的小说网站，在防止数据采集反面还是做了准备的，其对主要的数字采用了自定义的编码映射取值，想直接通过页面来实现数据的获取，是无法实现的。
单独获取数字还是可以实现的，通过requests发送请求，用正则去匹配字符元素，并再次匹配其映射关系的url,获取到的数据通过font包工具解析成字典格式，再做编码匹配，起点返回的编码匹配英文数字，英文数字匹配阿拉伯数字，最后拼接，得到实际的数字字符串，但这样多次发送请求，爬取效率会大大降低。本次集中爬取舍弃了爬取数字，选择了较容易获取的评分数字。评分值默认为0 ，是从后台推送的js数据里取值更新的。
实现的主要代码：
items部分：
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
from scrapy import Field
class QdItem(scrapy.Item):
# define the fields for your item here like:
book_name = scrapy.Field() #书名
author=scrapy.Field() #作者
state=scrapy.Field() #状态
type=scrapy.Field() #类型
about=scrapy.Field() #简介
# number=scrapy.Field() #字数
score=scrapy.Field() #评分
story=scrapy.Field() #故事
news=scrapy.Field() #最新章节
photo=scrapy.Field() #封面
spider部分:
# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from qd.items import QdItem
import re,requests
from fontTools.ttLib import TTFont
from io import BytesIO
import time
class ReadSpider(CrawlSpider):
name = \'read\'
# allowed_domains = [\'qidian.com\']
start_urls = [\'https://www.qidian.com/all?orderId=&style=1&pageSize=20&siteid=1&pubflag=0&hiddenField=0&page=1\']
rules = (
#匹配全部主页面的url规则 深度爬取子页面
Rule(LinkExtractor(allow=(r\'https://www.qidian.com/all\?orderId=\&style=1\&pageSize=20\&siteid=1\&pubflag=0\&hiddenField=0\&page=(\d+)\')),follow=True),
#匹配详情页面 不作深度爬取
Rule(LinkExtractor(allow=r\'https://book.qidian.com/info/(\d+)\'), callback=\'parse_item\', follow=False),
)
def parse_item(self, response):
item=QdItem()
item[\'book_name\']=self.get_book_name(response)
item[\'author\']=self.get_author(response)
item[\'state\']=self.get_state(response)
item[\'type\']=self.get_type(response)
item[\'about\']=self.get_about(response)
# item[\'number\']=self.get_number(response)
item[\'score\']=self.get_score(response)
item[\'story\']=self.get_story(response)
item[\'news\']=self.get_news(response)
item[\'photo\']=self.get_photo(response)
yield item
def get_book_name(self,response):
book_name=response.xpath(\'//h1/em/text()\').extract()[0]
if len(book_name)>0:
book_name=book_name.strip()
else:
book_name=\'NULL\'
return book_name
def get_author(self,response):
author=response.xpath(\'//h1/span/a/text()\').extract()[0]
if len(author)>0:
author=author.strip()
else:
author=\'NULL\'
return author
def get_state(self,response):
state=response.xpath(\'//p[@class="tag"]/span/text()\').extract()[0]
if len(state)>0:
state=state.strip()
else:
st=\'NULL\'
return state
def get_type(self,response):
type=response.xpath(\'//p[@class="tag"]/a/text()\').extract()
if len(type)>0:
t=""
for i in type:
t+=\' \'+i
type=t
else:
type=\'NULL\'
return type
def get_about(self,response):
about=response.xpath(\'//p[@class="intro"]/text()\').extract()[0]
if len(about)>0:
about=about.strip()
else:
about=\'NULL\'
return about
# def get_number(self,response):
#
# def get_font(url): #获取字体对应的字典编码
# time.sleep(2)
# resp=requests.get(url)
# font=TTFont(BytesIO(resp.content))
# cmap=font.getBestCmap()
# font.close()
# return cmap
#
# def get_encode(cmap,values):
# #values的值 \'𘛖𘛘𘛕𘛔𘛎𘛎\'
# #中英数字编码表
# WORD_MAP = {\'zero\': \'0\', \'one\': \'1\', \'two\': \'2\', \'three\': \'3\', \'four\': \'4\', \'five\': \'5\', \'six\': \'6\',
# \'seven\': \'7\',\'eight\': \'8\', \'nine\': \'9\', \'period\': \'.\'}
# list=values.split(\';\')
# list.pop(-1)
# new_num=\'\'
# #移除最后的分号；
# for num in list:
# value=num[2:]
# key=cmap[int(value)]
# new_num+=WORD_MAP[key]
# return new_num
#
# # pattern=re.compile(\'</style><span.*?>(.*?)</span>\',re.S) #数字字符匹配规则
# # # 𘛖𘛘𘛕𘛔𘛎𘛎
# # number_list=re.findall(pattern,response)
# # #匹配所有数字字符列表
# # reg=re.compile(\'<style.*?>(.*?)\s*</style>\',re.S) #包含字体链接的文本
# # font_url=re.findall(reg,response)[0]
# # url=re.search(\'woff.*?url.*?\\'(.+?)\\'.*?truetype\',font_url).group(1) #获取当前数字库的链接地址
# # # https://qidian.gtimg.com/qd_anti_spider/xxxxx.ttf
# #
# # cmap=get_font(url) #获取字典对应编码
# # # {100046: \'seven\', 100048: \'three\', 100049: \'five\', 100050: \'six\', 100051: \'one\', 100052: \'period\', 100053: \'nine\', 100054: \'four\', 100055: \'eight\', 100056: \'two\', 100057: \'zero\'}
# #
# #
# # d_num=[] #解码后的所有数字追加进去
# # for num in number_list: #遍历列表中的元素
# # d_num.append(get_encode(cmap,num))
# # if len(d_num)>0:
# # return d_num[0]+\'万字\'
# # else:
# return \'NULL\'
def get_score(self,response):
def get_sc(id):
urll = \'https://book.qidian.com/ajax/comment/index?_csrfToken=ziKrBzt4NggZbkfyUMDwZvGH0X0wtrO5RdEGbI9w&bookId=\' + id + \'&pageSize=15\'
rr = requests.get(urll)
# print(rr)
score = rr.text[16:19]
return score
bid=response.xpath(\'//a[@id="bookImg"]/@data-bid\').extract()[0] #获取书的id
if len(bid)>0:
score=get_sc(bid) #调用方法获取评分 若是整数 可能返回 9，"
if score[1]==\',\':
score=score.replace(\',"\',".0")
else:
score=score
else:
score=\'NULL\'
return score
def get_story(self,response):
story=response.xpath(\'//div[@class="book-intro"]/p/text()\').extract()[0]
if len(story)>0:
story=story.strip()
else:
story=\'NULL\'
return story
def get_news(self,response):
news=response.xpath(\'//div[@class="detail"]/p[@class="cf"]/a/text()\').extract()[0]
if len(news)>0:
news=news.strip()
else:
news=\'NULL\'
return news
def get_photo(self,response):
photo=response.xpath(\'//div[@class="book-img"]/a[@class="J-getJumpUrl"]/img/@src\').extract()[0]
if len(photo)>0:
photo=photo.strip()
else:
photo=\'NULL\'
return photo
middlewaver 中间件部分：
# # -*- coding: utf-8 -*-
#
# # Define here the models for your spider middleware
# #
# # See documentation in:
# # https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#
# from scrapy import signals
#
#
# class QdSpiderMiddleware(object):
# # Not all methods need to be defined. If a method is not defined,
# # scrapy acts as if the spider middleware does not modify the
# # passed objects.
#
# @classmethod
# def from_crawler(cls, crawler):
# # This method is used by Scrapy to create your spiders.
# s = cls()
# crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
# return s
#
# def process_spider_input(self, response, spider):
# # Called for each response that goes through the spider
# # middleware and into the spider.
#
# # Should return None or raise an exception.
# return None
#
# def process_spider_output(self, response, result, spider):
# # Called with the results returned from the Spider, after
# # it has processed the response.
#
# # Must return an iterable of Request, dict or Item objects.
# for i in result:
# yield i
#
# def process_spider_exception(self, response, exception, spider):
# # Called when a spider or process_spider_input() method
# # (from other spider middleware) raises an exception.
#
# # Should return either None or an iterable of Response, dict
# # or Item objects.
# pass
#
# def process_start_requests(self, start_requests, spider):
# # Called with the start requests of the spider, and works
# # similarly to the process_spider_output() method, except
# # that it doesn’t have a response associated.
#
# # Must return only requests (not items).
# for r in start_requests:
# yield r
#
# def spider_opened(self, spider):
# spider.logger.info(\'Spider opened: %s\' % spider.name)
#
#
# class QdDownloaderMiddleware(object):
# # Not all methods need to be defined. If a method is not defined,
# # scrapy acts as if the downloader middleware does not modify the
# # passed objects.
#
# @classmethod
# def from_crawler(cls, crawler):
# # This method is used by Scrapy to create your spiders.
# s = cls()
# crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
# return s
#
# def process_request(self, request, spider):
# # Called for each request that goes through the downloader
# # middleware.
#
# # Must either:
# # - return None: continue processing this request
# # - or return a Response object
# # - or return a Request object
# # - or raise IgnoreRequest: process_exception() methods of
# # installed downloader middleware will be called
# return None
#
# def process_response(self, request, response, spider):
# # Called with the response returned from the downloader.
#
# # Must either;
# # - return a Response object
# # - return a Request object
# # - or raise IgnoreRequest
# return response
#
# def process_exception(self, request, exception, spider):
# # Called when a download handler or a process_request()
# # (from other downloader middleware) raises an exception.
#
# # Must either:
# # - return None: continue processing this exception
# # - return a Response object: stops process_exception() chain
# # - return a Request object: stops process_exception() chain
# pass
#
# def spider_opened(self, spider):
# spider.logger.info(\'Spider opened: %s\' % spider.name)
import random,base64
from qd.settings import USER_AGENT,PROXIES
class RandomUserAgent(object):
def process_request(self,request,spider):
user_agent=random.choice(USER_AGENT)
if user_agent:
request.headers.setdefault("User-Agent",user_agent)
class RandomProxy(object):
def process_request(self,request,spider):
proxy=random.choice(PROXIES)
if proxy[\'user_psd\']is None: #没有用户名和密码则不需要认证
request.meta[\'proxy\']=\'http://\'+proxy[\'ip_port\']
else:
bs64_user_psd=base64.b64encode(proxy[\'user_psd\'])
request.meta[\'proxy\']=\'http://\'+proxy[\'ip_port\']
request.headers[\'Proxy-Authorization\']=\'Basic \'+bs64_user_psd
pipeline管道部分：
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don\'t forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql,re
from scrapy.exceptions import DropItem
class QdPipeline(object):
def __init__(self):
self.connect = pymysql.connect(
user=\'root\', # 用户名
password=\'1234\', # 密码
db=\'lgweb\', # 数据库名
host=\'127.0.0.1\', # 地址
port=3306,
charset=\'utf8\'
)
def table_exists(self, con, table_name):
# 判断数据表是否已经创建
sql = \'show tables;\'
con.execute(sql)
tables = [con.fetchall()]
table_list = re.findall(\'(\\'.*?\\')\', str(tables))
table_list = [re.sub("\'", \'\', each) for each in table_list] # 遍历并获得数据库表
if table_name in table_list:
return 1 # 创建了返回1
else:
return 0 # 不创建返回0
def process_item(self, item, spider):
conn = self.connect.cursor() # 创建该链接的游标
conn.execute(\'use lgweb\') # 指定数据库
table_name = \'db_read\' # 数据库表
valid = True
for data in item:
if not data:
valid = False
raise DropItem(\'Missing %s of blogpost from %s\' % (data, item[\'url\']))
if valid: # 如果item里面有数据则取出来
book_name = item[\'book_name\']
author = item[\'author\']
state = item[\'state\']
type = item[\'type\']
about = item[\'about\']
# number = item[\'number\']
score = item[\'score\']
story = item[\'story\']
news = item[\'news\']
photo = item[\'photo\']
# 没有对应数据库表则创建
if (self.table_exists(conn, table_name) != 1):
sql = \'create table db_read(书名 VARCHAR (30),作者 VARCHAR (30),评分 VARCHAR (10),类型 VARCHAR (30),状态 VARCHAR (30),简介 VARCHAR (50),详情 VARCHAR (1000),最新章节 VARCHAR (50),封面 VARCHAR (100))\'
conn.execute(sql) # 不存在则创建数据库表
try:
# 有数据则插入数据表
sql = "insert into db_read(书名,作者,评分,类型,状态,简介,详情,最新章节,封面)VALUES (\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\')" % (
book_name,author,score,type,state,about,story,news, photo)
conn.execute(sql) # 执行插入数据操作
self.connect.commit() # 提交保存
finally:
conn.close()
return item
settings进行简单配置，就可以运行程序了。
为了方便调试程序，可以在项目外编写一个main.py入口文件，和命令行执行 scrapy crawl read 效果是一样的。
main代码如下：
from scrapy import cmdline
cmdline.execute(\'scrapy crawl read\'.split())
爬取数据效果图：