拖了好久的一个爬虫
先上代码 文字慢慢补
update(2018-5-7):加了出版社＝ ＝
updata(2018-6-29):啥都加了 https://github.com/general10/duangduang
首先我们是要爬取当当网书籍信息
在当前页面可以获取的是书名 评论总数 价格 折扣
这几个数据都很好处理
直接就在html里 抓出来就可以了
.text是获取内容 []是获取属性= =
1 bookname = data.find_all(\'div\', attrs={\'class\': \'name\'}) 2 bookstar = data.find_all(\'div\', attrs={\'class\': \'star\'}) 3 bookprice = data.find_all(\'div\', attrs={\'class\': \'price\'}) 4 bookoff = data.find_all(\'span\', attrs={\'class\': \'price_s\'}) 5 6 7 bookname[i].find(\'a\')[\'title\'] + " " # 书名 8 bookprice[i].find(\'span\').text[1:] + " " # 价格 9 bookoff[i].text[:-1] + " " # 折扣 10 bookstar[i].find(\'a\').text[:-3] + " " # 评论数
在书名的url里我们可以获取到好评 中评 差评 好评率
（其实上一步可以获取好评率 那个小星星是有填充的）
我们可以在检查里看到
但是我们在网页源代码里是看不到评价的
因为评论这部分是ajax异步获取的
然后我们打开F12
在post请求里找评论的post
看一下这个preview的内容大概可以确定这个请求是评论的请求
然后我们看header下的Request URL
这个url可以通过productId categoryPath mainProductId 这三个参数来确定
这三个参数可以在网页源代码里通过正则表达式匹配出来
1 def getId(html): 2 id = {} 3 ma = re.search(r\'"productId":"[\d]+"\', html) 4 id[\'productId\'] = eval(ma.group().split(\':\')[-1]) 5 ma = re.search(r\'"categoryPath":"[\d.]+"\', html) 6 id[\'categoryPath\'] = eval(ma.group().split(\':\')[-1]) 7 ma = re.search(r\'"mainProductId":"[\d.]+"\', html) 8 id[\'mainProductId\'] = eval(ma.group().split(\':\')[-1]) 9 return id
然后拼接url之后下载url内容
1 def getCommentUrl(id): 2 return \'http://product.dangdang.com/index.php?r=comment%2Flist&productId={productId}&categoryPath={categoryPath}&mainProductId={mainProductId}&mediumId=0&pageIndex=1&sortType=1&filterType=1&isSystem=1&tagId=0&tagFilterCount=0\'.format( 3 productId=id[\'productId\'], categoryPath=id[\'categoryPath\'], mainProductId=id[\'mainProductId\'])
url内容是个json
格式化一下之后就容易看得多了
直接用py自带的json解析工具解析一下就好了
1 def getCommentCount(url): 2 html = urllib2.urlopen(url).read() 3 4 # 用正则表达式获取对应id 5 id = getId(html) 6 7 # 拼接ajax对应的url 8 json_url = getCommentUrl(id) 9 10 # 获取url对应的json 11 json_html = json.loads(getJsonText(json_url)) 12 13 # 获取评论数 14 summary = json_html[\'data\'][\'list\'][\'summary\'] 15 comment= {} 16 comment[\'好评\'] = summary[\'total_crazy_count\'] # 好评数 17 comment[\'中评\'] = summary[\'total_indifferent_count\'] # 中评数 18 comment[\'差评\'] = summary[\'total_detest_count\'] # 差评数 19 comment[\'好评率\'] = summary[\'goodRate\'] # 好评率 20 return comment
最后是写入excel
.write(行, 列, 内容)
1 sheet1.write(page * 20 + i + 1, 0, page * 20 + i + 1) 2 sheet1.write(page * 20 + i + 1, 1, bookname[i].find(\'a\')[\'title\']) 3 sheet1.write(page * 20 + i + 1, 2, bookprice[i].find(\'span\').text[1:]) 4 sheet1.write(page * 20 + i + 1, 3, bookoff[i].text[:-1]) 5 sheet1.write(page * 20 + i + 1, 4, bookstar[i].find(\'a\').text[:-3]) 6 sheet1.write(page * 20 + i + 1, 5, data[\'好评\']) 7 sheet1.write(page * 20 + i + 1, 6, data[\'中评\']) 8 sheet1.write(page * 20 + i + 1, 7, data[\'差评\']) 9 sheet1.write(page * 20 + i + 1, 8, data[\'好评率\']) 10 wb.save(\'test.xls\')
result
all code
（这个是一开始的）
1 # -*- coding: utf-8 -* 2 3 import urllib2 4 import xlwt 5 from bs4 import BeautifulSoup 6 from datashape import json 7 import re 8 import json 9 import requests 10 11 12 def getJsonText(url): 13 try: 14 r = requests.get(url, timeout=1) 15 r.raise_for_status() 16 r.encoding = r.apparent_encoding 17 return r.text 18 except: 19 print \'获取失败\' 20 return \'\' 21 22 23 def getId(html): 24 id = {} 25 ma = re.search(r\'"productId":"[\d]+"\', html) 26 id[\'productId\'] = eval(ma.group().split(\':\')[-1]) 27 ma = re.search(r\'"categoryPath":"[\d.]+"\', html) 28 id[\'categoryPath\'] = eval(ma.group().split(\':\')[-1]) 29 ma = re.search(r\'"mainProductId":"[\d.]+"\', html) 30 id[\'mainProductId\'] = eval(ma.group().split(\':\')[-1]) 31 return id 32 33 def getCommentUrl(id): 34 return \'http://product.dangdang.com/index.php?r=comment%2Flist&productId={productId}&categoryPath={categoryPath}&mainProductId={mainProductId}&mediumId=0&pageIndex=1&sortType=1&filterType=1&isSystem=1&tagId=0&tagFilterCount=0\'.format( 35 productId=id[\'productId\'], categoryPath=id[\'categoryPath\'], mainProductId=id[\'mainProductId\']) 36 37 def getCommentCount(url): 38 html = urllib2.urlopen(url).read() 39 40 # 用正则表达式获取对应id 41 id = getId(html) 42 43 # 拼接ajax对应的url 44 json_url = getCommentUrl(id) 45 46 # 获取url对应的json 47 json_html = json.loads(getJsonText(json_url)) 48 49 # 获取评论数 50 summary = json_html[\'data\'][\'list\'][\'summary\'] 51 comment= {} 52 comment[\'好评\'] = summary[\'total_crazy_count\'] # 好评数 53 comment[\'中评\'] = summary[\'total_indifferent_count\'] # 中评数 54 comment[\'差评\'] = summary[\'total_detest_count\'] # 差评数 55 comment[\'好评率\'] = summary[\'goodRate\'] # 好评率 56 return comment 57 58 def main(): 59 wb = xlwt.Workbook() 60 sheet1 = wb.add_sheet("Sheet") 61 sheet1.write(0, 0, unicode(\'序号\', "utf-8")) 62 sheet1.write(0, 1, unicode(\'书名\', "utf-8")) 63 sheet1.write(0, 2, unicode(\'价格\', "utf-8")) 64 sheet1.write(0, 3, unicode(\'折扣\', "utf-8")) 65 sheet1.write(0, 4, unicode(\'评论数\', "utf-8")) 66 sheet1.write(0, 5, unicode(\'好评\', "utf-8")) 67 sheet1.write(0, 6, unicode(\'中评\', "utf-8")) 68 sheet1.write(0, 7, unicode(\'差评\', "utf-8")) 69 sheet1.write(0, 8, unicode(\'好评率\', "utf-8")) 70 71 for page in range(25): 72 73 url = \'http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-%d\' % (page+1) 74 get = urllib2.urlopen(url).read() 75 data = BeautifulSoup(get, \'lxml\') 76 77 bookname = data.find_all(\'div\', attrs={\'class\': \'name\'}) 78 bookstar = data.find_all(\'div\', attrs={\'class\': \'star\'}) 79 bookprice = data.find_all(\'div\', attrs={\'class\': \'price\'}) 80 bookoff = data.find_all(\'span\', attrs={\'class\': \'price_s\'}) 81 82 for i in range(20): 83 bookurl = bookname[i].find(\'a\')[\'href\'] 84 data = getCommentCount(bookurl) 85 print (str(page*20+i+1) + " " 86 + bookname[i].find(\'a\')[\'title\'] + " " # 书名 87 + bookprice[i].find(\'span\').text[1:] + " " # 价格 88 + bookoff[i].text[:-1] + " " # 折扣 89 + bookstar[i].find(\'a\').text[:-3] + " " # 评论数 90 + data[\'好评\'] + " " # 好评数 91 + data[\'中评\'] + " " # 中评数 92 + data[\'差评\'] + " " # 差评数 93 + data[\'好评率\'] + " " # 好评率 94 ) 95 96 sheet1.write(page * 20 + i + 1, 0, page * 20 + i + 1) 97 sheet1.write(page * 20 + i + 1, 1, bookname[i].find(\'a\')[\'title\']) 98 sheet1.write(page * 20 + i + 1, 2, bookprice[i].find(\'span\').text[1:]) 99 sheet1.write(page * 20 + i + 1, 3, bookoff[i].text[:-1]) 100 sheet1.write(page * 20 + i + 1, 4, bookstar[i].find(\'a\').text[:-3]) 101 sheet1.write(page * 20 + i + 1, 5, data[\'好评\']) 102 sheet1.write(page * 20 + i + 1, 6, data[\'中评\']) 103 sheet1.write(page * 20 + i + 1, 7, data[\'差评\']) 104 sheet1.write(page * 20 + i + 1, 8, data[\'好评率\']) 105 wb.save(\'test.xls\') 106 107 main()
在mac里换了点东西才可以用（谁知道为什么
1 # -*- coding: utf-8 -* 2 3 import xlwt 4 from bs4 import BeautifulSoup 5 from datashape import json 6 import re 7 import json 8 import requests 9 10 11 def getJsonText(url): 12 try: 13 r = requests.get(url, timeout=1) 14 r.raise_for_status() 15 r.encoding = r.apparent_encoding 16 return r.text 17 except: 18 print(\'获取失败\') 19 return \'\' 20 21 22 def getId(html): 23 id = {} 24 ma = re.search(r\'"productId":"[\d]+"\', html) 25 id[\'productId\'] = eval(ma.group().split(\':\')[-1]) 26 ma = re.search(r\'"categoryPath":"[\d.]+"\', html) 27 id[\'categoryPath\'] = eval(ma.group().split(\':\')[-1]) 28 ma = re.search(r\'"mainProductId":"[\d.]+"\', html) 29 id[\'mainProductId\'] = eval(ma.group().split(\':\')[-1]) 30 return id 31 32 def getCommentUrl(id): 33 return \'http://product.dangdang.com/index.php?r=comment%2Flist&productId={productId}&categoryPath={categoryPath}&mainProductId={mainProductId}&mediumId=0&pageIndex=1&sortType=1&filterType=1&isSystem=1&tagId=0&tagFilterCount=0\'.format( 34 productId=id[\'productId\'], categoryPath=id[\'categoryPath\'], mainProductId=id[\'mainProductId\']) 35 36 def getCommentCount(url): 37 html = requests.get(url).text 38 39 # 用正则表达式获取对应id 40 id = getId(html) 41 42 # 拼接ajax对应的url 43 json_url = getCommentUrl(id) 44 45 # 获取url对应的json 46 json_html = json.loads(getJsonText(json_url)) 47 48 # 获取评论数 49 summary = json_html[\'data\'][\'list\'][\'summary\'] 50 comment = {} 51 comment[\'好评\'] = summary[\'total_crazy_count\'] # 好评数 52 comment[\'中评\'] = summary[\'total_indifferent_count\'] # 中评数 53 comment[\'差评\'] = summary[\'total_detest_count\'] # 差评数 54 comment[\'好评率\'] = summary[\'goodRate\'] # 好评率 55 return comment 56 57 def main(): 58 wb = xlwt.Workbook() 59 sheet1 = wb.add_sheet("Sheet") 60 sheet1.write(0, 0, \'序号\') 61 sheet1.write(0, 1, \'书名\') 62 sheet1.write(0, 2, \'作者\') 63 sheet1.write(0, 3, \'出版社\') 64 sheet1.write(0, 4, \'价格\') 65 sheet1.write(0, 5, \'折扣\') 66 sheet1.write(0, 6, \'评论数\') 67 sheet1.write(0, 7, \'好评\') 68 sheet1.write(0, 8, \'中评\') 69 sheet1.write(0, 9, \'差评\') 70 sheet1.write(0, 10, \'好评率\') 71 72 for page in range(2): 73 74 url = \'http://bang.dangdang.com/books/bestsellers/01.00.00.00.00.00-24hours-0-0-1-%d\' % (page+1) 75 get = requests.get(url).text 76 data = BeautifulSoup(get, \'lxml\') 77 78 bookname = data.find_all(\'div\', attrs={\'class\': \'name\'}) 79 bookstar = data.find_all(\'div\', attrs={\'class\': \'star\'}) 80 bookpublish = data.find_all(\'div\', attrs={\'class\': \'publisher_info\'}) 81 bookprice = data.find_all(\'div\', attrs={\'class\': \'price\'}) 82 bookoff = data.find_all(\'span\', attrs={\'class\': \'price_s\'}) 83 84 85 for i in range(20): 86 bookurl = bookname[i].find(\'a\')[\'href\'] 87 comments = getCommentCount(bookurl) 88 print (str(page*20+i+1) + " " 89 + bookname[i].find(\'a\')[\'title\'] + " " # 书名 90 + bookpublish[i*2].find(\'a\').text + " " # 作者 91 + bookpublish[i*2+1].find(\'a\').text + " " # 出版社 92 + bookprice[i].find(\'span\').text[1:] + " " # 价格 93 + bookoff[i].text[:-1] + "折 " # 折扣 94 + bookstar[i].find(\'a\').text[:-3] + " " # 评论数 95 + comments[\'好评\'] + " " # 好评数 96 + comments[\'中评\'] + " " # 中评数 97 + comments[\'差评\'] + " " # 差评数 98 + comments[\'好评率\'] + "% " # 好评率 99 ) 100 101 sheet1.write(page * 20 + i + 1, 0, page * 20 + i + 1) 102 sheet1.write(page * 20 + i + 1, 1, bookname[i].find(\'a\')[\'title\']) 103 sheet1.write(page * 20 + i + 1, 2, bookpublish[i*2].find(\'a\').text) 104 sheet1.write(page * 20 + i + 1, 3, bookpublish[i*2+1].find(\'a\').text) 105 sheet1.write(page * 20 + i + 1, 4, bookprice[i].find(\'span\').text[1:]) 106 sheet1.write(page * 20 + i + 1, 5, bookoff[i].text[:-1] + \'折\') 107 sheet1.write(page * 20 + i + 1, 6, bookstar[i].find(\'a\').text[:-3]) 108 sheet1.write(page * 20 + i + 1, 7, comments[\'好评\']) 109 sheet1.write(page * 20 + i + 1, 8, comments[\'中评\']) 110 sheet1.write(page * 20 + i + 1, 9, comments[\'差评\']) 111 sheet1.write(page * 20 + i + 1, 10, comments[\'好评率\'] + \'%\') 112 wb.save(\'test.xls\') 113 114 main()
（存个图片）