使用嵌套的 defaultdict 重建数组

Question

这个问题是上一个问题的延伸：rebuild python array based on common elements - 但不同之处足以保证一个新问题：

我已经为此苦苦挣扎了一段时间。我的数据是来自 sql 查询的字典数组。数组中的每个元素都代表一个货件，根据keys有共同的值。

data = [
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0001", "qty":"10", "memo":"blah1"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0002", "qty":"10", "memo":"blah2"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0003", "qty":"10", "memo":"blah3"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0004", "qty":"20", "memo":"blah4"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0005", "qty":"20", "memo":"blah5"},
    {"CustName":"customer3", "PartNum":"partXYZ", "delKey":"0006", "qty":"50", "memo":"blah6"},
    {"CustName":"customer3", "PartNum":"partABC", "delKey":"0007", "qty":"100", "memo":"blah7"}]

我想要的输出是根据特定的键分组

dataOut = [
   {"CustName":"customer1", "Parts":[
        {"PartNum":"part1", "deliveries":[
            {"delKey":"0001", "qty":"10", "memo":"blah1"},
            {"delKey":"0002", "qty":"10", "memo":"blah2"},
            {"delKey":"0003", "qty":"10", "memo":"blah3"}]}]},
   {"CustName":"customer2", "Parts":[
        {"PartNum":"part3", "deliveries":[
            {"delKey":"0004", "qty":"20", "memo":"blah4"},
            {"delKey":"0005", "qty":"20", "memo":"blah5"}]}]},
   {"CustName":"customer3", "Parts":[
        {"PartNum":"partXYZ", "deliveries":[
            {"delKey":"0006", "qty":"50", "memo":"blah6"}]},
        {"PartNum":"partABC", "deliveries":[
            {"delKey":"0007", "qty":"100", "memo":"blah7"}]}]}]

我可以使用上一个问题提供的 defaultdict 和列表理解来获得单个级别的分组，并稍作修改

d = defaultdict(list)
for item in data:
    d[item['CustName']].append(item)
print([{'CustName': key, 'parts': value} for key, value in d.items()])

但我似乎无法在输出数组中获得第二级 - 分组 b PartNum 键。通过一些研究，我认为我需要做的是使用defaultdict 作为外部`defaultdict'的类型，如下所示：

d = defaultdict(defaultdict(list))

因为 defaultdict 返回一个函数而引发错误，所以我需要使用lambda（是吗？）

d = defaultdict(lambda:defaultdict(list))
for item in data:
    d[item['CustName']].append(item) <----this?

我的问题是如何“访问”循环中的第二级数组并告诉“内部”默认字典要分组的内容（PartNum）？数据来自数据库程序员，项目不断发展以添加越来越多的数据（键），所以我希望这个解决方案尽可能通用，以防更多数据被我扔掉。我希望能够根据我需要去的级别来“链接”defaultdicts。我边走边学，所以我在努力理解lambda 和defaultdict 类型的基础知识以及从这里到哪里去。

Answer 1

这是我能做到的最漂亮的方式。它使用相同的defaultdict 思想来实现正确的分组，因为python 的内置groupby 函数仅适用于有序数据。

请注意，此版本将改变输入数据集中的项，因此结果中的叶项与输入是相同的 dict 实例，但删除了 "CustName" 和 "PartNum" 条目。

from collections import defaultdict

def groupby_mutate(seq, key):
  d = defaultdict(list)
  for item in seq:
    d[item[key]].append(item)
    del item[key]
  return d

def your_operation(data):
  return [ {
    'CustName': CustName,
    'Parts': [ { 
      'PartNum': PartNum,
      'deliveries': deliveries
    } for PartNum,deliveries in groupby_mutate(custItems, 'PartNum').items() ]
  } for CustName,custItems in groupby_mutate(data, 'CustName').items() ]


# try it
from pprint import *
data = [
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0001", "qty":"10", "memo":"blah1"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0002", "qty":"10", "memo":"blah2"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0003", "qty":"10", "memo":"blah3"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0004", "qty":"20", "memo":"blah4"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0005", "qty":"20", "memo":"blah5"},
    {"CustName":"customer3", "PartNum":"partXYZ", "delKey":"0006", "qty":"50", "memo":"blah6"},
    {"CustName":"customer3", "PartNum":"partABC", "delKey":"0007", "qty":"100", "memo":"blah7"}
]

pprint(your_operation(data))

编辑：

以防将来有人需要它，这里有一个不会改变原始数据的版本：

from collections import defaultdict

def groupby_getitem(seq, key):
  d = defaultdict(list)
  for item in seq:
    d[item[key]].append(item)
  return d

def your_operation(data):
  return [ {
    'CustName': CustName,
    'Parts': [ { 
      'PartNum': PartNum,
      'deliveries': [ dict(
        (k,v) for k,v in delivery.items() if not k in ['CustName', 'PartNum']
      ) for delivery in deliveries ]
    } for PartNum,deliveries in groupby_getitem(custItems, 'PartNum').items() ]
  } for CustName,custItems in groupby_getitem(data, 'CustName').items() ]

Answer 2

您可以使用基于OrderedDefaultdict 而不是defaultdict(list) 的树状数据结构。 （定义来自我一个不相关的answer。）

from collections import OrderedDict

class OrderedDefaultdict(OrderedDict):
    def __init__(self, *args, **kwargs):
        if not args:
            self.default_factory = None
        else:
            if not (args[0] is None or callable(args[0])):
                raise TypeError('first argument must be callable or None')
            self.default_factory = args[0]
            args = args[1:]
        super(OrderedDefaultdict, self).__init__(*args, **kwargs)

    def __missing__ (self, key):
        if self.default_factory is None:
            raise KeyError(key)
        self[key] = default = self.default_factory()
        return default

Tree = lambda: OrderedDefaultdict(Tree)

d = Tree()
for rec in data:
    custName, partNum, delKey = rec['CustName'], rec['PartNum'], rec['delKey']
    details = {"qty": rec["qty"], "memo": rec["memo"]}
    d[custName]['Parts'][partNum]['deliveries'][delKey] = details

因此，对于您的问题中显示的data，d 最终将包含：

d = {
    "customer1": {
        "Parts": {
            "part1": {
                "deliveries": {"0001": {"memo": "blah1", "qty": "10"},
                               "0002": {"memo": "blah2", "qty": "10"},
                               "0003": {"memo": "blah3", "qty": "10"}}}}},
    "customer2": {
        "Parts": {
            "part3": {
                "deliveries": {"0004": {"memo": "blah4", "qty": "20"},
                               "0005": {"memo": "blah5", "qty": "20"}}}}},
    "customer3": {
        "Parts": {
            "partXYZ": {
                "deliveries": {"0006": {"memo": "blah6", "qty": "50"}}},
            "partABC": {
                "deliveries": {"0007": {"memo": "blah7", "qty": "100"}}}}}
}

这可以简单地打印出来，因为它现在按您想要的方式分组。

Answer 3

按"CustName", "PartNum", "delKey" 排序。迭代每个零件、每个客户的交付项目并累积以匹配您的输出规格。

我喜欢使用operator.itemgetter - 对我来说，它让事情变得更清晰。

import collections, itertools, operator

cust_name = operator.itemgetter('CustName')
part_num = operator.itemgetter('PartNum')
group_sort = operator.itemgetter('CustName', 'PartNum', 'delKey')
del_key = operator.itemgetter('delKey')
qty = operator.itemgetter('qty')
memo = operator.itemgetter('memo')


# sort on the relavent keys
data.sort(key = group_sort)
result = []

# iterate over customers
for custname, group1 in itertools.groupby(data, cust_name):
    cust_dict = {'CustName' : custname, 'Parts': []}
    # iterate over parts for this customer
    for partnum, group2 in itertools.groupby(group1, part_num):
        part_dict = {"PartNum" : partnum, 'deliveries' : []}
        # iterate over delivery items for this part
        for thing in group2:
            part_dict['deliveries'].append({'delKey':del_key(thing),
                                            'qty':qty(thing),
                                            'memo':memo(thing)})
        cust_dict['Parts'].append(part_dict)
    result.append(cust_dict)

这显然会多次迭代原始数据中的项目，这可能会影响性能——但我没有看到多次迭代的方法来解决您需要做的事情。

Answer 4

按照@Pynchia 的建议使用groupby，并按照@hege_hegedus 的建议对无序数据使用sorted：

from itertools import groupby
dataOut = []
dataSorted = sorted(data, key=lambda x: (x["CustName"], x["PartNum"]))
for cust_name, cust_group in groupby(dataSorted, lambda x: x["CustName"]):
    dataOut.append({
        "CustName": cust_name,
        "Parts": [],
    })
    for part_num, part_group in groupby(cust_group, lambda x: x["PartNum"]):
        dataOut[-1]["Parts"].append({
            "PartNum": part_num,
            "deliveries": [{
                "delKey": delivery["delKey"],
                "memo": delivery["memo"],
                "qty": delivery["qty"],
            } for delivery in part_group]
        })

如果您查看第二个 for 循环，这有望回答您有关访问循环中的第二级数组的问题。