1 import requests
2 from requests.exceptions import RequestException
3 import re,json
4 import xlwt,xlrd
5
6 # 数据
7 DATA = []
8 KEYWORD = \'python\'
9 HEADERS = {\'user-agent\':\'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome\'\
10 \'/63.0.3239.132 Safari/537.36\'}
11 MAX_PAGE = 10
12
13
14 def get_target(data_list):
15 for item in data_list:
16 temp = {
17 \'title\': item[\'title\'],
18 \'price\': item[\'view_price\'],
19 \'sales\': item[\'view_sales\'],
20 \'isTmall\': \'否\' if float(item[\'view_fee\']) else \'是\',
21 \'area\': item[\'item_loc\'],
22 \'name\': item[\'nick\'],
23 \'url\': item[\'detail_url\']
24 }
25 DATA.append(temp)
26 return True
27
28
29 # 发送http请求，获取网页源码
30 def get_html(url,*args):
31 try:
32 if not args:
33 response = requests.get(url,headers=HEADERS)
34 global COOKIES
35 COOKIES = response.cookies # 获取cookie
36 else:
37 response = requests.get(url,headers=HEADERS,cookies=COOKIES)
38
39 response.encoding = response.apparent_encoding
40 return response.text
41 except RequestException:
42 print(\'请求源码出错！\')
43
44 # 解析源码，得到目标信息
45 def parse_html(html,*args):
46 if not args:
47 pattern = re.compile(r\'g_page_config = (.*?)g_srp_loadCss\',re.S)
48 # 去掉末尾的\';\'
49 result = re.findall(pattern, html)[0].strip()[:-1]
50 # 格式化json，可以用json在线解析工具查看结构
51 content = json.loads(result)
52 data_list = content[\'mods\'][\'itemlist\'][\'data\'][\'auctions\']
53 else:
54 pattern = re.compile(r\'{.*}\',re.S)
55 result = re.findall(pattern,html)[0]
56 content = json.loads(result)
57 data_list = content[\'API.CustomizedApi\'][\'itemlist\'][\'auctions\']
58
59 get_target(data_list)
60
61
62 def save_to_excel():
63 f_name = \'淘宝%s数据\'%KEYWORD
64 book = xlwt.Workbook(encoding=\'utf-8\',style_compression=0)
65 sheet = book.add_sheet(f_name)
66 sheet.write(0, 0, \'title\')
67 sheet.write(0, 1, \'price\')
68 sheet.write(0, 2, \'sales\')
69 sheet.write(0, 3, \'isTmall\')
70 sheet.write(0, 4, \'area\')
71 sheet.write(0, 5, \'name\')
72 sheet.write(0, 6, \'url\')
73 for i in range(len(DATA)):
74 sheet.write(i+1, 0, DATA[i][\'title\'])
75 sheet.write(i+1, 1, DATA[i][\'price\'])
76 sheet.write(i+1, 2, DATA[i][\'sales\'])
77 sheet.write(i+1, 3, DATA[i][\'isTmall\'])
78 sheet.write(i+1, 4, DATA[i][\'area\'])
79 sheet.write(i+1, 5, DATA[i][\'name\'])
80 sheet.write(i+1, 6, DATA[i][\'url\'])
81 book.save(\'淘宝%s数据.xls\'%KEYWORD)
82
83
84
85 def main():
86 for offset in range(MAX_PAGE):
87 # 首页有12条异步加载的数据 api?
88 if offset == 0:
89 url1 = \'https://s.taobao.com/search?q={}&s={}\'.format(KEYWORD,offset*44)
90 html = get_html(url1)
91 contents = parse_html(html)
92
93 url2 = \'https://s.taobao.com/api?_ksTS=1532524504679_226&callback=jsonp227&ajax=true&m=customized&\' \
94 \'stats_click=search_radio_all:1&q={}\'.format(KEYWORD)
95 html = get_html(url2,2)
96 contents = parse_html(html,2)
97 else:
98 url = \'https://s.taobao.com/search?q={}&s={}\'.format(KEYWORD,offset*44)
99 html = get_html(url)
100 contents = parse_html(html)
101
102 save_to_excel()
103 print(len(DATA))
104
105 if __name__ == \'__main__\':
106 main()