python中邮政编码列表的Web抓取地址

Question

是否可以根据postcoede从地址定位器网站上抓取地址。 如果可以使用scrapy或任何其他python库，请帮助我。

我有一个 .csv 中的邮政编码列表，我正在尝试提取这些邮政编码的地址信息 并保存在 .csv 或 .txt 中

假设一个列表包含邮政编码

POSTCODE
HU9 5PX  
OX5 1HX  

我正在尝试以 .csv 格式获取输出

HU9 5PX  Aarhus Karlshamn,King George Dock,HULL
OX5 1HX  Abric Europe Ltd,Langford Locks,KIDLINGTON

我试图做这样的事情。

import logging
import os
import pandas as pd
import re
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
from googlesearch import search
logging.getLogger('scrapy').propagate = False

def get_urls(tag, n, language):
    urls = [url for url in search(tag, stop=n, lang=language)][:n]
    return urls

address_list = re.findall(a-z)


class MailSpider(scrapy.Spider):
    
    name = 'address'
    
    def parse(self, response):
        
        links = LxmlLinkExtractor(allow=()).extract_links(response)
        links = [str(link.url) for link in links]
        links.append(str(response.url))
        
        for link in links:
            yield scrapy.Request(url=link, callback=self.parse_link) 
            
    def parse_link(self, response):
        
        for word in self.reject:
            if word in str(response.url):
                return
            
        html_text = str(response.text)
        mail_list = re.findall('a-z', html_text)

但是上面的代码没有给出正确的结果。 错误：NameError：未定义名称“a” 请帮忙

Answer 1

这是可能的。不过，您还有其他选择。您可以使用 geopy、mapquest has 和 API，或理想的邮政编码。以下是它可以返回的一些结果（注意：您需要自己的 api 密钥来进行 mapquest）

POSTCODE_list = ['HU9 5PX', 'OX5 1HX']

代码：geopy

import geopy
from geopy.geocoders import Nominatim

geopy.geocoders.options.default_user_agent = "locations-application"
geolocator=Nominatim(timeout=10) 

for postal_code in POSTCODE_list:
    location = geolocator.geocode('%s' %(postal_code))
    print (location)

输出：

Kingston upon Hull, HU9 5PX, UK
Kidlington, OX5 1HX, UK

代码：mapquest api

import requests

for postal_code in POSTCODE_list:    
    url = 'http://www.mapquestapi.com/geocoding/v1/address?key=<ENTER YOU API KEY HERE>&location=%s' %postal_code
    r = requests.get(url)
    results = r.json()['results']

    print (results[0]['locations'])

输出：

[{'street': '', 'adminArea6': '', 'adminArea6Type': 'Neighborhood', 'adminArea5': 'Kingston upon Hull', 'adminArea5Type': 'City', 'adminArea4': 'Kingston upon Hull, City of', 'adminArea4Type': 'County', 'adminArea3': '', 'adminArea3Type': 'State', 'adminArea1': 'GB', 'adminArea1Type': 'Country', 'postalCode': 'HU9 5PX', 'geocodeQualityCode': 'Z1XAA', 'geocodeQuality': 'ZIP', 'dragPoint': False, 'sideOfStreet': 'N', 'linkId': '287067758', 'unknownInput': '', 'type': 's', 'latLng': {'lat': 53.746092, 'lng': -0.270408}, 'displayLatLng': {'lat': 53.746092, 'lng': -0.270408}, 'mapUrl': 'http://www.mapquestapi.com/staticmap/v5/map?key=ZGPYZNMeDbhK2NVXjTQ944ejAm2F1kqG&type=map&size=225,160&locations=53.746092,-0.270408|marker-sm-50318A-1&scalebar=true&zoom=15&rand=-982249466'}]
[{'street': '', 'adminArea6': '', 'adminArea6Type': 'Neighborhood', 'adminArea5': 'Kidlington', 'adminArea5Type': 'City', 'adminArea4': 'Oxfordshire', 'adminArea4Type': 'County', 'adminArea3': '', 'adminArea3Type': 'State', 'adminArea1': 'GB', 'adminArea1Type': 'Country', 'postalCode': 'OX5 1HX', 'geocodeQualityCode': 'Z1XAA', 'geocodeQuality': 'ZIP', 'dragPoint': False, 'sideOfStreet': 'N', 'linkId': '287512940', 'unknownInput': '', 'type': 's', 'latLng': {'lat': 51.830272, 'lng': -1.305073}, 'displayLatLng': {'lat': 51.830272, 'lng': -1.305073}, 'mapUrl': 'http://www.mapquestapi.com/staticmap/v5/map?key=ZGPYZNMeDbhK2NVXjTQ944ejAm2F1kqG&type=map&size=225,160&locations=51.830272,-1.305073|marker-sm-50318A-1&scalebar=true&zoom=15&rand=-1049337783'}]

代码：理想邮政编码

import requests
import pandas as pd

postal_code_df = pd.read_csv('file.csv')    


results = pd.DataFrame()    
for postal_code in postal_code_df['postal_code_column_name']:  # <---- USE THE HEADER NAME FOR THAT COLUMN IN YOUR FILE
    postal_code = ''.join(postal_code.split())
    url = 'https://api.ideal-postcodes.co.uk/v1/postcodes/%s?api_key=iddqd' %(postal_code)
    data = requests.get(url).json()

    data = data['result'][0]

    print ('%s %s, %s, %s' %(postal_code, data['organisation_name'], data['line_2'], data['post_town']))
    temp_df = pd.DataFrame([[postal_code, data['organisation_name'], data['line_2'], data['post_town']]],
                            columns=['postal_code','organisation','street','town'])

    results = results.append(temp_df, sort=True).reset_index(drop=True)

results.to_csv('file.csv', index=False)

输出：

HU95PX Aarhus Karlshamn, King George Dock, HULL
OX51HX Essentra Components Ltd, Langford Locks, KIDLINGTON