转自 https://blog.csdn.net/u011660391/article/details/83991867?utm_medium=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-2.control&depth_1-utm_source=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-2.control
一、说明：
在网上百度下载省市区等数据，不是要积分下载就是要钱，我等穷逼既无积分又无钱，那就只有另想办法，学过几天python，用python将就抓点数据，借鉴别人一些写法，再修修补补，调试bug，基本上可以运行，并将抓取的数据保存至MySQL数据库中（抓取之前换成自己的数据库配置）。开发环境python3.6，下面附上源码，对python略懂基本，代码写得不好，如有更好的抓取方式和写法，欢迎指导。
二、代码
#!/usr/bin/python3 # -*- coding: utf-8 -*- # author=yjrowe import sys import os import re from urllib import request from bs4 import BeautifulSoup import pymysql import requests import time # 请求页面 def http_request(url, charset=\'utf8\'): try: print(url) data = requests.get(url, headers=header, timeout=5) kk = data.content kk = kk.decode(charset) except Exception as e: print(e) kk = http_request(url, charset) return kk # 获取全国省份和直辖市 def province(): print(\'爬取省数据中。。。\') t = http_request(url, \'gbk\') province_list = {} # if t: # soup = BeautifulSoup(t, \'html.parser\') # for i in soup.find_all(attrs={\'class\': \'provincetr\'}): # for a in i.find_all(\'a\'): # id = re.sub("\D", "", a.get(\'href\')) # province_list[id] = {\'id\': id, \'name\': a.text, \'code\': id, \'href\': url + a.get(\'href\')} # #print(province_list[id]) # # exit(province_list) province_list[0] = {\'id\': \'43\', \'name\': \'湖南省\', \'code\': \'43\', \'href\': \'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/43.html\'} insert_data(\'province\', province_list) return province_list # 获取省下级市 def city(province_list): print(\'爬取市数据中。。。\') city_list = {} # for i in province_list: # t = http_request(province_list[i][\'href\'], \'gbk\') # parent_href = get_parent_url(province_list[i][\'href\']) # if not t: # continue # soup = BeautifulSoup(t, \'html.parser\') # for v in soup.find_all(attrs={\'class\': \'citytr\'}): # id = str(v.find_all(\'td\')[0].text) # city_href = str(v.find_all(\'td\')[1].find_all(\'a\')[0].get(\'href\')) # city_list[id[0:4]] = {\'id\': id[0:4], \'name\': str(v.find_all(\'td\')[1].text), \'province_id\': i, \'code\': id, # \'href\': parent_href + city_href} city_list[\'4301\'] = {\'id\': \'4301\', \'name\': \'长沙市\', \'province_id\': 0, \'code\': \'430100000000\', \'href\': \'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/43/4301.html\'} #insert_data(\'city\', city_list) return city_list # 获取市下级区县 def country(city_list): print(\'爬取区县数据中。。。\') county_list = {} for i in city_list: t = http_request(city_list[i][\'href\'], \'gbk\') parent_href = get_parent_url(city_list[i][\'href\']) if not t: continue soup = BeautifulSoup(t, \'html.parser\') for v in soup.find_all(attrs={\'class\': \'countytr\'}): id = str(v.find_all(\'td\')[0].text) if len(v.find_all(\'td\')[1].find_all(\'a\')): country_href = str(v.find_all(\'td\')[1].find_all(\'a\')[0].get(\'href\')) else: continue county_list[id[0:6]] = {\'id\': id[0:6], \'name\': str(v.find_all(\'td\')[1].text), \'city_id\': i, \'code\': id, \'href\': parent_href + country_href} insert_data(\'country\', county_list) return county_list # 县下级街道、镇 def street(county_list): print(\'爬取街道数据中。。。\') street_list = {} for i in county_list: t = http_request(county_list[i][\'href\'], \'gbk\') parent_href = get_parent_url(county_list[i][\'href\']) if not t: continue soup = BeautifulSoup(t, \'html.parser\') time.sleep(1) for v in soup.find_all(attrs={\'class\': \'towntr\'}): id = str(v.find_all(\'td\')[0].text) street_href = str(v.find_all(\'td\')[1].find_all(\'a\')[0].get(\'href\')) street_list[id[0:9]] = {\'id\': id[0:9], \'name\': str(v.find_all(\'td\')[1].text), \'country_id\': i, \'code\': id, \'href\': parent_href + street_href} insert_data(\'street\', street_list) return street_list # 社区、街道办 def community(street_list): print(\'爬取社区数据中。。。\') community_list = {} for i in street_list: t = http_request(street_list[i][\'href\'], \'gbk\') if not t: continue soup = BeautifulSoup(t, \'html.parser\') time.sleep(1) for v in soup.find_all(attrs={\'class\': \'villagetr\'}): id = str(v.find_all(\'td\')[0].text) community_list[id[0:12]] = {\'id\': id[0:12], \'name\': str(v.find_all(\'td\')[2].text), \'street_id\': i, \'code\': id, \'category\': str(v.find_all(\'td\')[1].text)} insert_data(\'community\', community_list) return community_list # 获取上级url def get_parent_url(href): # print(\'上级链接：\' + href) arr = href.split(\'/\') last_value = arr[len(arr) - 1] href = href.replace(last_value, \'\') return href # 插入数据 def insert_data(table, data): if len(data) == 0: return False keys = list(data.keys()) table_keys = \'`\'+\'`,`\'.join(list(data[keys[0]].keys()))+\'`\' for k in range(0, len(keys)): table_values = \'\\'\'+\'\\',\\'\'.join(list(data[keys[int(k)]].values()))+\'\\'\' sql = \'INSERT INTO %s (%s) VALUES (%s)\' % (table, table_keys, table_values) # exit(sql) try: cursor.execute(sql) except Exception as e: print(e) conn.commit() if __name__ == \'__main__\': conn = pymysql.connect(host=\'139.199.16.31\', port=3379, user=\'root\', passwd=\'Yunzin@yz123\', db=\'yz_cbe\', charset=\'utf8\') cursor = conn.cursor() sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) url = \'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/\' header = { \'Cookie\': \'AD_RS_COOKIE=20181108\', \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ AppleWeb\Kit/537.36 (KHTML, like Gecko)\ \' \'Chrome/58.0.3029.110 Safari/537.36\'} provinceList = province() print(provinceList) #provinceList = [\'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/43.html\'] cityList = city(provinceList) # print(cityList) countryList = country(cityList) print(countryList) streetList = street(countryList) print(streetList) communityList = community(streetList) print(\'数据抓取完成\')
CREATE TABLE `province` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) DEFAULT NULL, `code` varchar(255) DEFAULT NULL, `href` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `city` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) DEFAULT NULL, `province_id` int(11) unsigned DEFAULT NULL, `code` varchar(255) DEFAULT NULL, `href` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`), KEY `city_province_id_foreign` (`province_id`), CONSTRAINT `city_province_id_foreign` FOREIGN KEY (`province_id`) REFERENCES `province` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `country` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) DEFAULT NULL, `city_id` int(11) unsigned DEFAULT NULL, `code` varchar(255) DEFAULT NULL, `href` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`), KEY `country_city_id_foreign` (`city_id`), CONSTRAINT `country_city_id_foreign` FOREIGN KEY (`city_id`) REFERENCES `city` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `street` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) DEFAULT NULL, `country_id` int(11) unsigned DEFAULT NULL, `code` varchar(255) DEFAULT NULL, `href` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`), KEY `street_country_id_foreign` (`country_id`), CONSTRAINT `street_country_id_foreign` FOREIGN KEY (`country_id`) REFERENCES `country` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `community` ( `id` varchar(100) NOT NULL, `name` varchar(255) DEFAULT NULL, `street_id` int(11) unsigned DEFAULT NULL, `code` varchar(255) DEFAULT NULL, `category` varchar(255) DEFAULT NULL COMMENT \'城乡分类代码\', PRIMARY KEY (`id`), KEY `community_street_id_foreign` (`street_id`), CONSTRAINT `community_street_id_foreign` FOREIGN KEY (`street_id`) REFERENCES `street` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;