Python - 使用 Lambda 的 Pandas 脚本太慢

Question

我的代码按我的意愿运行，但是当我运行这一行时非常慢。

--- newdf['Login'] = newdf['Site'].apply(lambda x : "yes" if get(x).status_code == 200 else "no") ---

注释后代码运行速度很快。 如何更改此行以在登录列中添加“是”或“否”并保持快速？ 如果我能改进这一切，我将不胜感激。 我希望我能理解自己。 谢谢！

import pandas as pd
import requests
from requests import get
from requests.exceptions import HTTPError
lista = pd.read_csv('sites4.csv', sep=',')
df = pd.DataFrame(lista, columns=['Site', 'Login'])
newdf = df.assign(Site=df['Site'].map(str) + 'Login')

headers = {'Content-Type': 'application/json'}

for i in newdf['Site']:
    try:
        result = get(i, headers=headers, timeout=5)
    except HTTPError as http_err:
        print(f'HTTP error occurred: {http_err}')
    except Exception as err:
        print(f'Other error occurred: {err}')
    else:
        if 'application/json' in result.headers.get('Content-Type') or result.status_code == 406 or result.status_code == 403:
            newdf['Login'] = newdf['Site'].apply(lambda x : "yes" if get(x).status_code == 200 else "no")
            print(i + ' é Login')
            print(result)

Answer 1

这应该是您的函数的更快实现

from typing import Optional, Coroutine, List
import aiohttp
from pandas import DataFrame
from pandas.errors import EmptyDataError
import pandas as pd
import asyncio


def create_df_form_file() -> Optional[DataFrame]:
    try:
        site_list = pd.read_csv('sites4.csv', sep=',')
        df = pd.DataFrame(site_list, columns=['Site', 'Login'])
        return df.assign(Site=df['Site'].map(str) + 'Login')
    except EmptyDataError as e:
        print(f'File Error: {e}')
    return None


new_df: Optional[DataFrame] = create_df_form_file()

if not isinstance(new_df, DataFrame):
    print("empty data goodbye")
    exit(1)


# NOTE: Async Get Request
async def get_request(x_url: str) -> bool:
    async with aiohttp.ClientSession() as session:
        async with session.get(url=x_url) as result:
            return result.ok


# functions to test if site needs a login and if header contains json and to include Yes or No
def needs_login(result): return result.status == 406 or result.status == 403


def is_json(result): return result.headers.get('Content-Type') == 'application/json'


async def yes_no(x): return 'yes' if await get_request(x) else 'no'


async def _do_work(site_column, _headers: dict) -> Optional[DataFrame]:

    async with aiohttp.ClientSession() as session:
        async with session.get(site_column['Site'], headers=_headers) as result:
            if is_json(result) or needs_login(result):
                # Appending Yes No
                site_column['login'] = site_column['Site'].apply(yes_no)
                print(site_column['Site'] + ' é Login')
                print(result)
                return site_column
            return None


def get_results():
    global new_df
    headers = {'Content-Type': 'application/json'}
    try:
        _coro: List[Coroutine] = [_do_work(site_column, _headers=headers) for site_column in new_df['Site']]
    except KeyError:
        print("please insure your input file is accurate")
        exit(1)

    try:
        event_loop = asyncio.get_event_loop()
    except RuntimeError:
        asyncio.set_event_loop(asyncio.new_event_loop())
        event_loop = asyncio.get_event_loop()

    # noinspection PyUnboundLocalVariable
    results = event_loop.run_until_complete(*_coro)
    print(results)


if __name__ == '__main__':
    get_results()

for more information on python programming please visit my tutorial site here

Answer 2

因为 get 在请求中是 func 。

由于requests不是异步包，所以所有代码都会中断，直到请求完成。

如果你使用 asyncio 和 aiohttp，你可以改进它。