用python抓取摩拜单车API数据 - Kernel_wu

用python抓取摩拜单车API数据

最近摩拜单车超级火，下班了各种骑，然后膝盖伤了。。。。。。

-----------------------------------------------------------------------

看到这篇文章http://mp.weixin.qq.com/s?__biz=MzA5NDExMTAzNA==&mid=2649982414&idx=1&sn=68b638c4f019baa3a783c045b294d6de&chksm=8854b19bbf23388dd91509a2a692736a5c0505ab144c5605d4015330044699c20e5c42f92496&mpshare=1&scene=1&srcid=0519Fkgd6ew4TuOHUerHMlZs#rd

说好的给源码，但貌似没给。。。。

然后就自己写了一份

--------------------------------------------------------------------------------------

import requests
import numpy as np
import json
import multiprocessing
from requests.packages.urllib3.exceptions import InsecureRequestWarning
#忽略警告
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

#进程数
PROCESS_NUM = 4
#模拟的数据包头
HEADERS = {
    \'charset\':\'utf-8\',
    \'platform\':\'4\',
    \'referer\':\'https://servicewechat.com/wx40f112341ae33edb/1/\',
    \'content-type\':\'application/x-www-form-urlencoded\',
    \'user-agent\':\'MicroMessenger/6.5.4.1000 NetType/WIFI Language/zh_CN\',
    \'host\':\'mwx.mobike.com\',
    \'connection\':\'Keep-Alive\',
    \'accept-encoding\':\'gzip\',
    \'cache-control\':\'no-cache\'
}
#初始化的部分鹿城区经纬度
LEFT = 28.026735
TOP = 120.644784
RIGHT = 27.99344
BOTTOM = 120.715068
INIT_POS = (TOP, LEFT, BOTTOM, RIGHT)

#将一个区域分成num份，每份后续对应一个进程进行处理
def split_pos(init_pos, num):
    result = []
    x1 = init_pos[0]
    y1 = init_pos[1]
    x2 = init_pos[2]
    y2 = init_pos[3]
    xrange = np.linspace(x1, x2, num + 1)
    for i in range(num):
        #可以做一些重叠的优化，暂时没做
        result.append((xrange[i], y1, xrange[i+1], y2))
    return result


#解析并保存数据
#可以直接通过MONGDB操作，这台机器没安装。。
def save(text, output):
    try:
        json_data = json.loads(text)
    except:
        print(\'***ERR1***   \', text)
        return
    if \'object\' not in json_data:
        print(\'***ERR2***   \', text)
        return
    for object in json_data[\'object\']:
        obj_values = []
        for key in sorted(object.keys()):
            try:
                if(object[key] == None):
                    obj_values.append(\'\')
                else:
                    obj_values.append(str(object[key]))
            except:
                print(\'***ERR3***   \', object)
                return
        output.write(\'$\'.join(obj_values) + \'\n\')


#爬取摩拜单车的数据,没有对反爬虫的限制进行处理，可以考虑IP池、拨号等手段
#此处只是闹着玩。。
def worker(num, location):
    raw_fd = open(\'worker\' + str(num) + \'_rawdata.txt\', \'w\')
    tabular_fd = open(\'worker\' + str(num) + \'_tabulardata.txt\', \'w\')
    session = requests.session()
    url = \'https://mwx.mobike.com/mobike-api/rent/nearbyBikesInfo.do\'
    # 模拟登录
    postdata = {
        \'latitude\': 0,
        \'longitude\': 0,
        \'errMsg\': \'getMapCenterLocation\'
    }
    (top, left, bottom, right) = location
    #按照经纬度，一个区域一个区域的获取信息
    lat_range = np.arange(left, right, -0.002)
    for lat in lat_range:
        lon_range = np.arange(top, bottom, 0.002)
        for lon in lon_range:
            postdata[\'latitude\'] = lat
            postdata[\'longitude\'] = lon
            mobike_data = session.post(url, headers=HEADERS, data=postdata, verify=False)
            #print(mobike_data.text)
            #保存原始数据，原始数据还是很关键的，最好都保留下来
            raw_fd.write(mobike_data.text + \'\n\')
            #解析并保存数据，还需要去除，后续单独进行，尽量不要再多进程或者多线程环境下处理
            save(mobike_data.text, tabular_fd)
    raw_fd.close()
    tabular_fd.close()

#主函数入口
if __name__ == \'__main__\':
    jobs = []
    location = split_pos(INIT_POS, PROCESS_NUM)
    #执行多进程，也可以用多线程
    for process_id in range(PROCESS_NUM):
        p = multiprocessing.Process(target=worker, args=(process_id,location[process_id]))
        jobs.append(p)
        p.start()