有条件地合并时间变量上的数据帧

Question

为了计算持续时间，我必须在日期、id 和时间变量上合并我想要合并的数据框。

from numpy import *
from pandas import *

df1 = DataFrame({
'id': ['a']*4,
'date': ['02-02-2015']*4,
'time_1': ['08:00:00', '09:00:00', '10:30:00', '12:45']})

df1
   id   date         time
0   a   02-02-2015  08:00:00
1   a   02-02-2015  09:00:00
2   a   02-02-2015  10:30:00
3   a   02-02-2015  12:45:00

-------------------------------------------------------------------------------------------------

df2 = DataFrame({
'id': ['a']*7,
'date': ['02-02-2015']*7,
'time_2': ['08:00:00', '08:09:00', '08:04:01','08:52:36', '09:34:25', '10:30:00', '11:23:38']})

df2
    id  date         time
0   a   02-02-2015  08:00:00
1   a   02-02-2015  08:09:00
2   a   02-02-2015  08:04:01
3   a   02-02-2015  08:52:36
4   a   02-02-2015  09:00:00
5   a   02-02-2015  10:30:00
6   a   02-02-2015  11:23:38

我希望我的合并遵循的规则是 df2 中需要的每一行都需要与 df1 中最接近的先前时间一致。

中间结果是

intermediateResult = DataFrame({
'id': ['a']*8,
'date': ['02-02-2015']*8,
'time_1': ['08:00:00', '08:00:00', '08:00:00','08:00:00', '09:00:00', '10:30:00', '10:30:00', '12:45'],
'time_2': ['08:00:00', '08:09:00', '08:04:01','08:52:36', '09:34:25', '10:30:00', '11:23:38', nan] })

intermediateResult
    id  date        time_1      time_2
0   a   02-02-2015  08:00:00    08:00:00
1   a   02-02-2015  08:00:00    08:09:00
2   a   02-02-2015  08:00:00    08:04:01
3   a   02-02-2015  08:00:00    08:52:36   # end
4   a   02-02-2015  09:00:00    09:34:25   # end
5   a   02-02-2015  10:30:00    10:30:00
6   a   02-02-2015  10:30:00    11:23:38   # end
7   a   02-02-2015  12:45           NaN

最后，我想得到每个时段的最新time_2（以注释#end表示）与其对应的time_1之间的时间差。

最终的结果应该是这样的

finalResult = DataFrame({
'id': ['a']*4,
'date': ['02-02-2015']*4,
'Duration': ['00:52:36', '00:34:25', '00:53:38', nan]})

finalResult 
    id  date        Duration
0   a   02-02-2015  00:52:36
1   a   02-02-2015  00:34:25
2   a   02-02-2015  00:53:38
3   a   02-02-2015  NaN

Answer 1

使用不同的合并方法，得出了相同的答案。最终根据您的要求使用了merge_as0fdirection =backward。不幸的是，在我没有 NaN 的意义上，与你的不相似。如果您提供有关如何在一行中最终得到 NaN 的信息，我们很乐意提供进一步的帮助。

    #Join dateto time and coerce to datetime
    df1['datetime']=pd.to_datetime(df1.date.str.cat(df1.time_1,sep=' '))
    df2['datetime']=pd.to_datetime(df2.date.str.cat(df2.time_2,sep=' '))
    df2['time_2'] = df2['time_2'].apply(lambda x: (x[-5:]))#StripHours from time_2. I anticipate to use it as duration
    #sort to allow merge_asof
    df1=df1.sort_values('datetime')
    df2=df2.sort_values('datetime')
    #Merge to the dataframes joining using datetime to the nearest hour
    df3=pd.merge_asof(df2, df1,on='datetime', by='id', tolerance=pd.Timedelta('2H'),allow_exact_matches=True,direction='backward').dropna()
    #df3=df2.merge(df1, left_on=df2.datetime.dt.hour, right_on=df1.datetime.dt.hour, how='left').drop(columns=['key_0', 'id_y', 'date_y']).fillna(0)#Alternative merge
    df3.set_index('datetime', inplace=True)#set datetime as index
    df3['minutes']=df3.index.minute#Extract minute in each row. Looks to me you want the highest minute in each hour
    #Groupby hour idxmax Helps boolean select the index with the highest minutes in an hour. aND DROP UNWANTED ROWS
    finalResult=df3.loc[df3.groupby([df3.index.hour, df3.date_x])['minutes'].idxmax()].reset_index().drop(columns=['datetime','time_1','date_y','minutes'])
   finalResult.columns=['id','date','Duration(min)']

最终结果

Answer 2

使用@wwnde 建议的解决方案，我找到了一个更适合我的真实数据集的解决方案：

import numpy as np
import pandas as pd

df1 = DataFrame({
    'id': ['a']*4,
    'date': ['02-02-2015']*4,
    'time_1': ['08:00:00', '09:00:00', '10:30:00', '12:45:00']
})

df2 = DataFrame({
    'id': ['a']*7,
    'date': ['02-02-2015',
        '02-02-2015',
        '03-02-2015', # small change here relatively to the df un my first post
        '02-02-2015',
        '02-02-2015',
        '02-02-2015',
        '02-02-2015'],
    'time_2': ['08:00:00', '08:09:00', '08:04:01','08:52:36', '09:34:25', '10:30:00', '11:23:38']
})


----------------------------------------------------

def preproDf(df1, df2, time_1, time_2, _id, date):
    '''
    Preprocess the dataframes for the following operations

    df1: pd.DataFrame, left dataframe
    df2: pd.DataFrame, right dataframe
    time_1:str, name of the left dataframe
    time_2:str, name of the right dataframe
    _id:str, name of the id variable. Should be the same for both dataframes
    date:str, name of the date variable. Should be the same for both dataframes

    return: None
    '''
    df2[time_2] = df2[time_2].apply(pd.to_datetime)
    df1[time_1] = df1[time_1].apply(pd.to_datetime)

    #sort to allow merge_asof
    df1=df1.sort_values([_id, date, time_1])
    df2=df2.sort_values([_id, date, time_2])



def processDF(df1, df2, time_1, time_2, _id, date):
    # initialisation
    groupKeys = list(df2.groupby([_id, date]).groups.keys())
    dfGroup=groupKeys[0]
    group = df2.groupby([_id, date]).get_group(dfGroup)
    rslt = pd.merge_asof(group, df1, left_on=time_2,  right_on=time_1, by=[_id, date], tolerance=pd.Timedelta('2H'),allow_exact_matches=True,direction='backward')#.dropna()

    # For loop to get the values in an array
    for group in groupKeys[1:]: # Iteration start at the second elmt
        group = df2.groupby([_id, date]).get_group(group)
        item = pd.merge_asof(group, df1, left_on=time_2,  right_on=time_1, by=[_id, date], tolerance=pd.Timedelta('2H'),allow_exact_matches=True,direction='backward')#.dropna()
        rslt = np.vstack((rslt, item))
    rslt = DataFrame(rslt, columns=item.columns)

    # Creating timeDifference variable
    rslt['timeDifference'] = rslt[time_2] - rslt[time_1]

    # Getting the actual result
    rslt = rslt.groupby([_id, date, time_1]).timeDifference.max()
    rslt = pd.DataFrame(rslt).reset_index()
    rslt.rename({time_1: 'openTime'}, axis='columns')

    return rslt

结果：

preproDf(df1, df2, 'time_1', 'time_2', 'id', 'date')

processDF(df1, df2, 'time_1', 'time_2', 'id', 'date')
    id  date    time_1  screenOnDuration
0   a   02-02-2015  2020-05-29 08:00:00 00:52:36
1   a   02-02-2015  2020-05-29 09:00:00 00:34:25
2   a   02-02-2015  2020-05-29 10:30:00 00:53:38