微信公众号爬取
#Weichat
import scrapy
import urllib.parse
from news_project.middlewares import Deal_Content
from news_project.items import NewsProjectItem
from lxml import etree
import js2py
import time
import requests
import re
import bs4
class Weichat(scrapy.Spider):
name = \'Weichat\'
base = \'https://mp.weixin.qq.com\'
allowed_domains = [\'weixin.sogou.com\'] #允许的页面最好不要定义 http:// 这样的
start_urls = [\'http://weixin.sogou.com\']
#微信
def parse(self,response):
url_1 = \'https://weixin.sogou.com/weixin?type=1&query={}&ie=utf8&s_from=input&_sug_=y&_sug_type_=\'.format(\'宝鸡招商局\')
yield scrapy.Request(url=url_1, callback=self.detail_parse,dont_filter=True)
def detail_parse(self, response):
baoji_url = response.xpath("//a[@uigs=\'account_name_0\']/@href").extract_first()
print(\'baoji_url\',baoji_url)
print(\'baoji_url\', type(baoji_url))
yield scrapy.Request(url=baoji_url, callback=self.baoji_parse, dont_filter=True)
def baoji_parse(self,response):
selector = etree.HTML(response.text)
print("------------------宝鸡招商局---------------------",response.text)
script = selector.xpath(\'.//script[not(@nonce) and @type="text/javascript"]/text()\')
script = script[1]
script = script.replace(\'seajs.use("sougou/profile.js");\', \'\')
script = \'function getList(){\' + script + \'return msgList \n}\'
script = script.replace(\'amp;\', \'\')
getList = js2py.eval_js(script)
js = getList()
js = str(js)
js = eval(js)
lis = js.get(\'list\')
firstLinks = []
otherStyleTimes = []
for li in lis:
# 获取文章发布时间，转换为时间戳，格式化为数据库可以保存的格式
datimes = li[\'comm_msg_info\'][\'datetime\']
timeArray = time.localtime(datimes)
otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
# 找到文章url，后期构建真实url时用到
try:
content_url = li.get(\'app_msg_ext_info\').get(\'content_url\')
print(content_url)
firstLink = self.base + content_url
except IndexError:
firstLink = None
print(\'CAPTCHA!\')
firstLinks.append(firstLink)
otherStyleTimes.append(otherStyleTime)
print(\'firstLinks, otherStyleTimes***********************\',firstLinks, otherStyleTimes)
yield scrapy.Request(url=firstLinks[0], callback=self.baoji_detail_parse, meta={\'time\':otherStyleTimes[0]},dont_filter=True)
def baoji_detail_parse(self,response):
item = NewsProjectItem()
content = \'\'
meta = response.meta
print("response.url",response.url)
#res.raise_for_status()
detailPage = bs4.BeautifulSoup(response.text, "html.parser")
# 获取文章标题
title = detailPage.title.text.replace(\'\n\', \'\').replace(\'\r\', \'\').replace(\' \', \'\').replace(\'！\', \'\').replace(\'|\', \'\')
print(\'title\',title)
sections = detailPage.findAll(\'section\', class_=\'_editor\')
# 获取文章内容
for section in sections[:-3]:
content = content + section.text.replace(\'\n\', \'\').replace(\'\r\', \'\').replace(\' \', \'\')
content_1 = response.xpath("//div[@id=\'js_content\']")
print("meta[\'time\']**************",meta[\'time\'])
print("content---------------",content)
item[\'title_url\'] = response.url
# 详细页面的内容
etree = response.xpath(\'//div[@id="js_content"]\')
tagContet = etree.extract()
tagContet = \'\'.join(tagContet)
content = etree.xpath(\'.//text()\').extract()
content = \'\'.join(content)
img_urls = etree.xpath(\'.//img/@src\').extract()
img_urls_dict = {}
for url in img_urls:
if "http://网站" not in url:
url1 = urllib.parse.urljoin(response.url, url) # 拼接url的网址
img_urls_dict[url] = url1
print("*******img_urls_dict****", img_urls_dict)
item[\'content\'], item[\'tags\'] = Deal_Content.handleText(content, tagContet, img_urls_dict, title)
print("************item[\'tags\']********************", item[\'tags\'])
item[\'content\'] = item[\'content\'].replace(re.findall("font-size: 18px;(.*)",item[\'content\'])[0],\'\')
item[\'title\'] = title
item[\'time\'] = meta[\'time\']
#item[\'content\'] = content
id, pid = Deal_Content.sql_read(response.url)
item[\'id\'] =id
item[\'pid\'] = pid
item[\'type_cn\'] = "省市级"
# #news 新闻来源、是那一个网站， 主页
item[\'news\'] = \'宝鸡招商局\'
# type_no 就是 id
item[\'type_no\'] = 18
yield item