基于列中列表的匹配项合并表行的有效算法

Question

我正在尝试在我的工作场所完成一项更大项目的任务，并且我有一个可行的解决方案，但由于解决方案的时间复杂性，它需要很长时间才能完成任务（数据帧的长度是几百万）。这不是一次性任务，必须每天运行。

目标：给定一个包含两列的表：“a”和“b”，其中“a”有单个字符串作为值，“b”有一个字符串列表作为值，合并“b”中的项目的行一行与其他行的“b”中的一个项目匹配，这样合并表中的“a”和“b”都将是一个项目列表。

示例 1：

输入表：

   a          b
0  1  [a, b, e]
1  2     [a, g]
2  3     [c, f]
3  4        [d]
4  5        [b]

所需输出：

           a             b
0  [1, 2, 5]  [a, b, e, g]
1        [3]        [c, f]
2        [4]           [d]

示例 2：

输入表：

   a          b
0  1  [a, b, e]
1  3  [a, g, f]
2  4     [c, f]
3  6     [d, h]
4  9  [b, g, h]

所需输出：

                 a                         b
0  [1, 3, 4, 6, 9]  [a, b, c, d, e, f, g, h]

我的工作解决方案：

import pandas as pd

def merge_rows(df):
    df_merged = pd.DataFrame(columns=df.columns)
    matched = False
    while len(df) > 0:
        if not matched:
            x = len(df_merged)
            df_merged.loc[x, 'a'] = list(df.iloc[0, 0])
            df_merged.loc[x, 'b'] = df.iloc[0, 1]
            df = df.iloc[1:, :]
        for rm in range(len(df_merged)):
            matched = False
            right_b_lists_of_lists = df.b.tolist()
            df.reset_index(drop=True, inplace=True)
            match_index_list = [i for b_part in df_merged.loc[rm, 'b'] for (i, b_list) in enumerate(right_b_lists_of_lists) if b_part in b_list]
            df_matches = df.loc[match_index_list]
            if len(df_matches) > 0:
                df_merged.loc[rm, 'a'] = list(set(df_merged.loc[rm, 'a'] + df_matches.a.tolist()))
                df_merged.loc[rm, 'b'] = list(set(df_merged.loc[rm, 'b'] + [item for sublist in df_matches.b.tolist() for item in sublist]))
                df = df.drop(df_matches.index)
                matched = True
                break
    return df_merged

df1 = pd.DataFrame({'a': ['1', '2', '3', '4', '5'], 'b': [['a', 'b', 'e'], ['a', 'g'], ['c', 'f'], ['d'], ['b']]})
df1_merged = merge_rows(df1)
print('Original DF:')
print(df1.to_string())
print('Merged DF:')
print(df1_merged.to_string())

df2 = pd.DataFrame({'a': ['1', '3', '4', '6', '9'], 'b': [['a', 'b', 'e'], ['a', 'g', 'f'], ['c', 'f'], ['d', 'h'], ['b', 'g', 'h']]})
df2_merged = merge_rows(df2)
print('Original DF:')
print(df2.to_string())
print('Merged DF:')
print(df2_merged.to_string())

上面的代码打印如下：

Original DF:
   a          b
0  1  [a, b, e]
1  2     [a, g]
2  3     [c, f]
3  4        [d]
4  5        [b]

Merged DF:
           a             b
0  [1, 2, 5]  [e, b, a, g]
1        [3]        [c, f]
2        [4]           [d]

Original DF:
   a          b
0  1  [a, b, e]
1  3  [a, g, f]
2  4     [c, f]
3  6     [d, h]
4  9  [b, g, h]

Merged DF:
                 a                         b
0  [4, 3, 6, 9, 1]  [e, h, c, g, f, d, b, a]

请注意，上述代码输出中的“a”和“b”中的列表未排序，但这是可以接受的。

考虑到 O(n^2) 的渐近时间复杂度作为解决方案的平均情况，这个解决方案实际上是不可行的，并且无法想出并行化这个多项式解决方案的方法，我需要的 n 的大尺寸每天运行它，我必须在机器上运行它。

对于线性解或可并行多项式解决方案（或更好！）的任何帮助将不胜感激！

首选 Python 解决方案，但我欢迎使用 R / C / C++ / Java / P 的解决方案。

Answer 1

这是一个使用不相交集结构思想的实现。 请注意，有很多方法可以提高效率（也可能存在错误）。 至少它适用于这两种情况，并且运行速度比我笔记本电脑上问题帖子中的原始功能快 10 倍。

import pandas as pd

def merge_rows2(df):
    parents = {}   # maps elements to the parent member
    
    for row in df.values:
        elems = row[1]
        if len(elems) < 1:
            continue  # edge case, empty letter list
        for elem in elems:
            if not elem in parents:       # new letter
                parents[elem] = elems[0]  # register the first element as the parent
            else:   # this letter has already be seen
                # find the root parent
                p = parents[elem]
                path = [elem]
                while True:
                    path.append(p)
                    if p == parents[p]:
                        break
                    p = parents[p]
                # map to the new parent, two sets merged
                parents[p] = elems[0]
                # path compression, for fast access next time
                for e in path:
                    parents[e] = elems[0]
    #print(parents)  # debug
    
    # make sure all elements directly maps to the root
    for e, p in parents.items():
        if e == p:  # root node
            continue
        # find the root node
        path = [e]
        while True:
            path.append(p)
            if p == parents[p]:
                break
            p = parents[p]
        # path compression
        for e in path:
            parents[e] = p
    #print(parents)  # debug
    groups = {}
    for e, p in parents.items():
        if p in groups:
            groups[p].append(e)
        else:
            groups[p] = [e]
    #print(groups)  # debug
    # collect values
    values = {g:[] for g in groups}
    for row in df.values:
        elems = row[1]
        if len(elems) < 1:
            continue
        p = parents[elems[0]]  # group identity
        values[p].append(row[0])
    # make data frame
    rows = [{"a":values[g], "b":groups[g]} for g in groups]
    return pd.DataFrame(rows) 

# test
df1 = pd.DataFrame({'a': ['1', '2', '3', '4', '5'], 'b': [['a', 'b', 'e'], ['a', 'g'], ['c', 'f'], ['d'], ['b']]})
print(merge_rows2(df1))

df2 = pd.DataFrame({'a': ['1', '3', '4', '6', '9'], 'b': [['a', 'b', 'e'], ['a', 'g', 'f'], ['c', 'f'], ['d', 'h'], ['b', 'g', 'h']]})
print(merge_rows2(df2))

# test
df1 = pd.DataFrame({'a': ['1', '2', '3', '4', '5'], 'b': [['a', 'b', 'e'], ['a', 'g'], ['c', 'f'], ['d'], ['b']]})
print(merge_rows2(df1))
#           a             b
#0  [1, 2, 5]  [a, b, e, g]
#1        [3]        [c, f]
#2        [4]           [d]

df2 = pd.DataFrame({'a': ['1', '3', '4', '6', '9'], 'b': [['a', 'b', 'e'], ['a', 'g', 'f'], ['c', 'f'], ['d', 'h'], ['b', 'g', 'h']]})
print(merge_rows2(df2))
#                 a                         b
#0  [1, 3, 4, 6, 9]  [a, b, e, g, f, c, d, h]

%timeit merge_rows(df1)
%timeit merge_rows2(df1)
#7.47 ms ± 277 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
#365 µs ± 3.66 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

%timeit merge_rows(df2)
%timeit merge_rows2(df2)
#4.1 ms ± 90.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
#351 µs ± 14 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

Answer 2

这使用纯 Python 而不是 Pandas，但可能需要更具代表性的示例数据集才能真正了解哪个更快，因为它大量使用具有不同时间和内存使用特性的 dicts 和 set。

我从 Rosetta Code 上的 Set consolidation 任务中复制了 consolidation 函数。

代码

# -*- coding: utf-8 -*-
"""
Answering:
        "Efficient algorithm to merge rows of a table based on matching items from a list in a column"
        https://stackoverflow.com/questions/62817492/efficient-algorithm-to-merge-rows-of-a-table-based-on-matching-items-from-a-list
        
Created on Fri Jul 10 04:49:26 2020

@author: Paddy3118
"""
#%%
from collections import defaultdict
from pprint import pprint as pp

def consolidate(sets):
    setlist = [s for s in sets if s]
    for i, s1 in enumerate(setlist):
        if s1:
            for s2 in setlist[i+1:]:
                intersection = s1.intersection(s2)
                if intersection:
                    s2.update(s1)
                    s1.clear()
                    s1 = s2
    return [s for s in setlist if s]

#%%
dat1 = {'a': ['1', '2', '3', '4', '5'], 
        'b': [['a', 'b', 'e'], ['a', 'g'], 
              ['c', 'f'], ['d'], ['b']]}

dat2 = {'a': ['1', '3', '4', '6', '9'], 
        'b': [['a', 'b', 'e'], ['a', 'g', 'f'], 
              ['c', 'f'], ['d', 'h'], ['b', 'g', 'h']]}
#data = dat2

def row_merge(data):
    data['a'] = [set(x) for x in data['a']]
    data['b'] = [set(x) for x in data['b']]
    
    b_map = defaultdict(list)
    for i, b_list in enumerate(data['b']):
        for item in b_list:
            b_map[item].append(i)
    
    index_merge = consolidate([set(v) for v in b_map.values()])
    
    a, b = defaultdict(set), defaultdict(set)
    a, b = [], []
    adata, bdata = data['a'], data['b']
    
    for merge in index_merge:
        arow, brow = set(), set()
        for row_index in merge:
            arow |= adata[row_index]
            brow |= bdata[row_index]
        a.append(sorted(arow))
        b.append(sorted(brow))
    
    return {'a': a, 'b': b}


answer = row_merge(dat1)
pp(answer)
answer = row_merge(dat2)
pp(answer)

输出

{'a': [['1', '2', '5'], ['3'], ['4']],
 'b': [['a', 'b', 'e', 'g'], ['c', 'f'], ['d']]}
{'a': [['1', '3', '4', '6', '9']],
 'b': [['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']]}