如何使用 Python 快速查找 XML 元素？

Question

我对 XML 以及使代码有效的原因还很陌生，而且我使用的代码需要很长时间才能运行。

所以我想尽快从给定的经纬度值中提取海拔高度（我有很多经度、经度点）。我就是这样尝试的：

import xml.etree.ElementTree as ET
from urllib.request import urlopen
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

def elevation(lat, long):
    query = ('http://openwps.statkart.no/skwms1/wps.elevation2?request=Execute&service=WPS&version=1.0.0'
                 f'&identifier=elevation&datainputs=lat={lat};lon={long};epsg=4326')
    parsing = "{http://www.opengis.net/wps/1.0.0}"
    with urlopen(query) as f:
        tree = ET.parse(f)
        root = tree.getroot()
        return float(root.findall(f".//{parsing}Data/*")[0].text)

在我从 csv 文件中提取的数据集上使用此函数，同一文件中的多个数据集由“new_sheep”-line 分隔：

df = pd.read_csv("/Users/ninsalv/Documents/Sheepdata/Data.csv", delimiter=';',
                 dtype={"Initial start": "str", "Start": "str", "Stop": "str"})
print(df.head())

dataset = 1
Lat = []
Long = []
temp = 0
for i in range(len(df)):
    if "new_sheep" in df.iloc[i][0]:
        temp += 1
        continue
    if temp == dataset:
        Lat.append(df.iloc[i][3])
        Long.append(df.iloc[i][4])
    if temp > dataset:
        break

step = np.linspace(0,len(Lat),len(Lat))
altitude = []
for i in range(len(Lat)):
    altitude.append(elevation(Lat[i], Long[i]))
    if (i % 100) == 0:
        print("round number ", i)
plt.plot(step, altitude)

这行得通，但是每 100 个海拔高度需要将近一分钟的时间，而且我有大约 7000-15000 个点要检查我的数据集。有人知道 XML、pandas 或其他可以让我的代码更快的东西吗？

Answer 1

您需要做的是并行获取您正在寻找的数据（HTTP 请求）。为此，您可以使用多线程。

请参见下面的示例。

import requests
from requests.sessions import Session
import time
from threading import Thread,local
from queue import Queue

url_list = [] # TODO long list of urls to be populated by your code
q = Queue(maxsize=0)            #Use a queue to store all URLs
for url in url_list:
    q.put(url)
thread_local = local()          #The thread_local will hold a Session object

def get_session() -> Session:
    if not hasattr(thread_local,'session'):
        thread_local.session = requests.Session() # Create a new Session if not exists
    return thread_local.session

def download_link() -> None:
    '''download link worker, get URL from queue until no url left in the queue'''
    session = get_session()
    while not q.empty():
        url = q.get()
        with session.get(url) as response:
            print(f'Read {len(response.content)} from {url}')
        q.task_done()          # tell the queue, this url downloading work is done

def download_all(urls) -> None:
    '''Start 10 threads, each thread as a wrapper of downloader'''
    thread_num = 10
    for i in range(thread_num):
        t_worker = Thread(target=download_link)
        t_worker.start()
    q.join()                   # main thread wait until all url finished downloading

print("start work")
start = time.time()
download_all(url_list)
end = time.time()
print(f'download {len(url_list)} links in {end - start} seconds')