前言：
爬取网站www.tianqihoubao.com上各城市的历史天气数据，并利用matplotlib将数据可视化。
代码实现：
1 import asyncio 2 import aiohttp 3 from lxml import etree 4 import re 5 from collections import namedtuple 6 import matplotlib.pyplot as plt 7 8 #用来正常显示中文标签 9 plt.rcParams[\'font.sans-serif\']=[\'SimHei\'] 10 11 Args = namedtuple(\'Args\',[\'city\',\'year\',\'month\']) 12 13 #获取给定城市列表在给定年列表的以月为单位的每天平均温度字典， 14 #返回格式 {Args(city=\'wuhan\', year=2018, month=4): [21.5, 22.5, ...], Args(city=\'shanghai\', year=2018, month=2): [3.5, 2.5, ...], ...} 15 def get_weather(citys, years): 16 tdata = {} 17 wdata = {} 18 async def work(args): 19 url = "http://www.tianqihoubao.com/lishi/%s/month/%d%02d.html" % (args.city, args.year, args.month) 20 headers = {\'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\'} 21 async with aiohttp.ClientSession() as session: 22 async with session.get(url, headers=headers, timeout=1000, verify_ssl=False) as response: 23 html = await response.text() 24 ethtml = etree.HTML(html) 25 26 #从html页面提取数据，数据格式如： \'10℃/-2℃\' 27 result = [re.sub("\r\n *","",item.text) for item in ethtml.xpath("//table[@class=\'b\']/tr/td") if not re.fullmatch("\r\n *",item.text)] 28 29 #获取温度数据 \'10℃/-2℃\'，将\'10℃/-2℃\'格式数据正则提取，求平均值 30 tstrs = result[1::3] 31 ts = [sum(map(lambda x: int(x), re.findall(\'(\d+|-\d+)℃\/(\d+|-\d+)℃\', tstr)[0]))/2 for tstr in tstrs 32 if re.fullmatch(\'(\d+|-\d+)℃\/(\d+|-\d+)℃\', tstr)] 33 34 #获取天气数据 \'多云/晴\', 35 wstrs = [re.findall(\'([\u4E00-\u9FA5]+)\/([\u4E00-\u9FA5]+)\',wstr)[0] for wstr in result[0::3] 36 if re.fullmatch(\'([\u4E00-\u9FA5]+)\/([\u4E00-\u9FA5]+)\',wstr)] 37 ws = [re.sub(\' \',\'\',b) for a in wstrs for b in a] 38 39 tdata[args] = ts 40 wdata[args] = ws 41 print(args, \'done\') 42 return None 43 44 loop = asyncio.get_event_loop() 45 46 #用列表推导是为所有城市所有年的每月添加协程 47 tasks = [asyncio.ensure_future(work(Args(city, year, month))) for city in citys for year in years for month in range(1,13)] 48 49 #开始执行协程，事件循环处理异步IO事件，程序阻塞在这里，直到所有的协程执行完毕 50 loop.run_until_complete(asyncio.wait(tasks)) 51 52 #return [task.result() for task in tasks] 53 return (tdata,wdata) 54 55 56 #绘制多个城市在某一年的温度曲线 57 def temperature_curve(citys, year, data): 58 for city in citys: 59 x = range(365) 60 y = [b for a in [data[Args(city, year, month)] for month in range(1,13)] for b in a][:365] 61 plt.plot(x, y, label=city) 62 plt.xticks(list(x)[::31], [\'Jan\',\'Feb\',\'Mar\',\'Apr\',\'May\',\'Jun\',\'Jul\',\'Aug\',\'Sept\',\'Oct\',\'Nov\',\'Dec\']) 63 plt.yticks([-10,0,10,20,30,40],[\'-10℃\',\'0℃\',\'10℃\',\'20℃\',\'30℃\',\'40℃\']) 64 plt.legend() 65 66 plt.title(\'各城市%d年温度曲线图\' % year) 67 plt.ylabel(\'temperature\') 68 plt.xlabel(\'month\') 69 plt.show() 70 71 #绘制多个城市在某一年各个天气比例 72 def weather_ratio(citys, year, data): 73 74 #创建 天气-颜色 字典 75 color_dict = {\'晴\':\'tomato\', \'多云\':\'gold\', \'阴\':\'turquoise\', \'阴天\':\'turquoise\', \'雷阵雨\':\'darkseagreen\', 76 \'暴雨\':\'black\',\'晴间多云\':\'cyan\', \'阵雨\':\'cadetblue\', \'浮尘\':\'pink\', \'扬沙\':\'plum\',\'霾\':\'dimgray\', 77 \'小雨\':\'cornflowerblue\', \'中雨\':\'blue\', \'大雨\':\'navy\', \'小雪\':\'lime\', \'中雪\':\'violet\', \'大雪\':\'teal\', 78 \'雨夹雪\':\'sienna\', \'冻雨\':\'skyblue\', \'少云\':\'steelblue\', \'雾\':\'white\'} 79 for city in citys: 80 labels = [] 81 sizes = [] 82 colors = [] 83 city_weathers = [b for a in [data[Args(city, year, month)] for month in range(1,13)] for b in a] 84 for weather in set(city_weathers): 85 labels.append(weather) 86 sizes.append(city_weathers.count(weather) * 100 / len(city_weathers)) 87 88 #给不同天气添加颜色 89 colors.append(color_dict[weather]) 90 91 #关联排序，sizes按逆序排序，labels，colors按sizes关联排序 92 z = zip(sizes,labels,colors) 93 z = sorted(z, reverse=True) 94 sizes,labels,colors = zip(*z) 95 96 #只显示最频繁的7中天气文字标签（以免文字间相互遮挡） 97 labels = list(labels)[:7] + [\'\'] * (len(labels) - 7) 98 99 #使比率最大的天气突出显示 100 explode = [0] * len(sizes) 101 explode[0] = 0.1 102 103 ax = plt.subplot(1, len(citys), citys.index(city) + 1) 104 ax.pie(sizes, labels=labels,autopct=\'%1.1f%%\',shadow=False,startangle=150,colors=colors, explode=explode) 105 ax.set_title(\'%s %d年天气饼图\' % (city, year)) 106 plt.show() 107 108 tdata,wdata = get_weather(citys=[\'wuhan\',\'shanghai\',\'beijing\'], years=[2018]) 109 temperature_curve(citys=[\'wuhan\', \'shanghai\', \'beijing\'], year=2018, data=tdata) 110 weather_ratio(citys=[\'wuhan\',\'shanghai\',\'beijing\'], year=2018, data=wdata)
效果展示：
总结：
代码还有很多可以有优化的地方，比如：
1. 数据存储数据库，以防爬取大量数据时程序意外崩溃导致数据丢失
2. 记录已爬url，好让程序重新开始后跳过爬取过的url，节省时间
3. 数据可视化还可以新增内容，同一个城市不同年份天气与气温的纵向比较