- 对于具有 100k 行的数据帧，此答案比其他 solution 快 8 倍
- 另一种实现方式有效，但使用了两次
.apply 和列表解析，与向量化方法相比，这很慢。
说明
-
.apply(literal_eval) 将'activity' 列从strings 转换为python 文字（例如dicts 的lists；'[{"name":"STILL","conf":100}]' → [{"name":"STILL","conf":100}]）
-
.explode 将每个 list 中的 dicts 分隔为单独的行
- 将
'activity' 列中的keys 和values 提取到单独的列中，然后将.join 列提取回df
- 此answer 的时序分析显示，将单级
dicts 的列提取到数据帧的最快方法是使用pd.DataFrame(df.pop('activity').values.tolist())
-
.pivotdf改成宽格式
- 将
dfp.columns.name 从 'name' 更改为 None - 这是装饰性的，可以删除
import pandas as pd
from ast import literal_eval
# test data
data = {'id': [4, 4, 4, 6, 6, 8, 9], 'time': [1596213715048, 1596213739171, 1596213755797, 1596214842817, 1596214931090, 1596214957246, 1596215304418], 'activity': ['[{"name":"STILL","conf":100}]', '[{"name":"STILL","conf":54},{"name":"ON_FOOT","conf":19},{"name":"WALKING","conf":19},{"name":"ON_BICYCLE","conf":9},{"name":"IN_VEHICLE","conf":8},{"name":"UNKNOWN","conf":3}]', '[{"name":"STILL","conf":97},{"name":"UNKNOWN","conf":2},{"name":"IN_VEHICLE","conf":1}]', '[{"name":"STILL","conf":100}]', '[{"name":"STILL","conf":34},{"name":"IN_VEHICLE","conf":28},{"name":"ON_FOOT","conf":15},{"name":"WALKING","conf":15},{"name":"ON_BICYCLE","conf":8},{"name":"UNKNOWN","conf":3}]', '[{"name":"STILL","conf":100}]', '[{"name":"STILL","conf":100}]']}
df = pd.DataFrame(data)
# function to transform column of strings
def test(df):
df.activity = df.activity.apply(literal_eval)
df = df.explode('activity').reset_index(drop=True)
df = df.join(pd.DataFrame(df.pop('activity').values.tolist()))
dfp = df.pivot(index=['id', 'time'], columns='name', values='conf').fillna(0).astype(int).reset_index()
dfp.columns.rename(None, inplace=True)
return dfp
# call the function
test(df)
# result
id time IN_VEHICLE ON_BICYCLE ON_FOOT STILL UNKNOWN WALKING
0 4 1596213715048 0 0 0 100 0 0
1 4 1596213739171 8 9 19 54 3 19
2 4 1596213755797 1 0 0 97 2 0
3 6 1596214842817 0 0 0 100 0 0
4 6 1596214931090 28 8 15 34 3 15
5 8 1596214957246 0 0 0 100 0 0
6 9 1596215304418 0 0 0 100 0 0
%%timeit 测试
import numpy as np
import random
import pandas
import json
from ast import literal_eval
# test data with 100000 rows
np.random.seed(365)
random.seed(365)
rows = 1000000
activity = ['[{"name":"STILL","conf":100}]', '[{"name":"STILL","conf":54},{"name":"ON_FOOT","conf":19},{"name":"WALKING","conf":19},{"name":"ON_BICYCLE","conf":9},{"name":"IN_VEHICLE","conf":8},{"name":"UNKNOWN","conf":3}]', '[{"name":"STILL","conf":97},{"name":"UNKNOWN","conf":2},{"name":"IN_VEHICLE","conf":1}]', '[{"name":"STILL","conf":100}]', '[{"name":"STILL","conf":34},{"name":"IN_VEHICLE","conf":28},{"name":"ON_FOOT","conf":15},{"name":"WALKING","conf":15},{"name":"ON_BICYCLE","conf":8},{"name":"UNKNOWN","conf":3}]', '[{"name":"STILL","conf":100}]', '[{"name":"STILL","conf":100}]']
data = {'time': pd.bdate_range('2021-01-15', freq='s', periods=rows),
'id': np.random.randint(10, size=(rows)),
'activity': [random.choice(activity) for _ in range(rows)]}
df = pd.DataFrame(data)
# test the function in this answer
%%timeit -r1 -n1 -q -o
test(df)
[out]:
<TimeitResult : 31.8 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>
# test the implementation from the other answer
def flatten_json_to_dict(s):
return {obj['name']: obj['conf'] for obj in json.loads(s)}
def nick(df):
expanded = df['activity'].apply(flatten_json_to_dict).apply(pd.Series)
df = df.join(expanded)
df = df.drop('activity', axis=1)
df = df.fillna(0)
return df
%%timeit -r1 -n1 -q -o
nick(df)
[out]:
<TimeitResult : 4min 28s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)>