Pandas：如何在 python3 中对混合类型的多索引使用切片？

Question

正如我在this partially related question 中指出的，不再可以对混合类型的序列进行排序：

# Python3.6
sorted(['foo', 'bar', 10, 200, 3])
# => TypeError: '<' not supported between instances of 'str' and 'int'

这会影响 pandas 中的切片查询。下面的例子说明了我的问题。

import pandas as pd
import numpy as np
index = [(10,3),(10,1),(2,2),('foo',4),('bar',5)]
index = pd.MultiIndex.from_tuples(index)
data = np.random.randn(len(index),2)
table = pd.DataFrame(data=data, index=index)

idx=pd.IndexSlice
table.loc[idx[:10,:],:]
# The last line will raise an UnsortedIndexError because 
# 'foo' and 'bar' appear in the wrong order.

异常信息如下：

UnsortedIndexError: 'MultiIndex slicing requires the index to be lexsorted: slicing on levels [0], lexsort depth 0'

在 python2.x 中，我通过对索引进行 lex-sorting 来从这个异常中恢复：

# Python2.x:
table = table.sort_index()

#               0         1
# 2   2  0.020841  0.717178
# 10  1  1.608883  0.807834
#     3  0.566967  1.978718
# bar 5 -0.683814 -0.382024
# foo 4  0.150284 -0.750709

table.loc[idx[:10,:],:]
#              0         1
# 2  2  0.020841  0.717178
# 10 1  1.608883  0.807834
#    3  0.566967  1.978718

但是，在 python3 中，我最终遇到了我在开头提到的异常：

TypeError: '<' not supported between instances of 'str' and 'int'

如何从中恢复？在排序之前将索引转换为字符串不是一种选择，因为这会破坏索引的正确排序：

# Python2/3
index = [(10,3),(10,1),(2,2),('foo',4),('bar',5)]
index = list(map(lambda x: tuple(map(str,x)), index))
index = pd.MultiIndex.from_tuples(index)
data = np.random.randn(len(index),2)
table = pd.DataFrame(data=data, index=index)
table = table.sort_index()
#               0         1
# 10  1  0.020841  0.717178
#     3  1.608883  0.807834
# 2   2  0.566967  1.978718
# bar 5 -0.683814 -0.382024
# foo 4  0.150284 -0.750709

使用此排序，基于值的切片将被破坏。

table.loc[idx[:10,:],:]     # Raises a TypeError
table.loc[idx[:'10',:],:]   # Misses to return the indices [2,:]

我该如何从中恢复？

Answer 1

这是我能想到的最好的。分三步解决：

• 以 lex 排序保留来自 python2 的旧混合类型排序的方式对多索引进行字符串化。例如，ints 可以在前面加上足够的 0。
• 对表格进行排序。
• 访问带有切片的表时使用相同的字符串化。

代码如下（完整示例）：

import numpy as np
import pandas as pd 

# Stringify whatever needs to be converted.
# In this example: only ints are stringified.
def toString(x):
    if isinstance(x,int):
        x = '%03d' % x
    return x
# Stringify an index tuple.
def idxToString(idx):
    if isinstance(idx, tuple):
        idx = list(idx)
        for i,x in enumerate(idx):
            idx[i] = toString(x)
        return tuple(idx)
    else:
        return toString(idx)
# Replacement for pd.IndexSlice
class IndexSlice(object):
    @staticmethod
    def _toString(arg):
        if isinstance(arg, slice):
            arg = slice(toString(arg.start),
                        toString(arg.stop),
                        toString(arg.step))
        else:
            arg = toString(arg)
        return arg

    def __getitem__(self, arg):
        if isinstance(arg, tuple):
            return tuple(map(self._toString, arg))
        else:
            return self._toString(arg)

# Build the table.
index = [(10,3),(10,1),(2,2),('foo',4),('bar',5)]
index = pd.MultiIndex.from_tuples(index)
data = np.random.randn(len(index),2)
table = pd.DataFrame(data=data, index=index)
# 1) Stringify the index.
table.index = table.index.map(idxToString)
# 2) Sort the index.
table = table.sort_index()
# 3) Create an IndexSlice that applies the same
#    stringification rules. (Replaces pd.IndexSlice)
idx = IndexSlice()
# Now, the table rows can be accessed as usual.
table.loc[idx[10],:]
table.loc[idx[:10],:]
table.loc[idx[:'bar',:],:]
table.loc[idx[:,:2],:]

这不是很漂亮，但它修复了升级到python3后损坏的表数据的基于切片的访问。如果你们有更好的建议，我很高兴阅读。

Answer 2

这是我想出的第二个解决方案。它比我之前的建议更好，因为它不会改变 lex 排序表的索引值。在这里，我在对表格进行排序之前临时转换了非字符串索引，但在排序后我对这些索引进行了去字符串化。

该解决方案有效，因为 pandas 自然可以处理混合类型的索引。似乎只有基于字符串的索引子集需要进行 lex 排序。 （Pandas 内部使用了一个所谓的Categorical 对象，该对象似乎可以单独区分字符串和其他类型。）

import numpy as np
import pandas as pd

def stringifiedSortIndex(table):
    # 1) Stringify the index.
    _stringifyIdx = _StringifyIdx()
    table.index = table.index.map(_stringifyIdx)
    # 2) Sort the index.
    table = table.sort_index()
    # 3) Destringify the sorted table.
    _stringifyIdx.revert = True
    table.index = table.index.map(_stringifyIdx)
    # Return table and IndexSlice together.
    return table

class _StringifyIdx(object):
    def __init__(self):
        self._destringifyMap = dict()
        self.revert = False
    def __call__(self, idx):
        if not self.revert:
            return self._stringifyIdx(idx)
        else:
            return self._destringifyIdx(idx)

    # Stringify whatever needs to be converted.
    # In this example: only ints are stringified.
    @staticmethod
    def _stringify(x):
        if isinstance(x,int):
            x = '%03d' % x
            destringify = int
        else:
            destringify = lambda x: x
        return x, destringify

    def _stringifyIdx(self, idx):
        if isinstance(idx, tuple):
            idx = list(idx)
            destr = [None]*len(idx)
            for i,x in enumerate(idx):
                idx[i], destr[i] = self._stringify(x)
            idx = tuple(idx)
            destr = tuple(destr)
        else:
            idx, destr = self._stringify(idx)
        if self._destringifyMap is not None:
            self._destringifyMap[idx] = destr
        return idx

    def _destringifyIdx(self, idx):
        if idx not in self._destringifyMap:
            raise ValueError(("Index to destringify has not been stringified "
                              "this class instance. Index must not change "
                              "between stringification and destringification."))
        destr = self._destringifyMap[idx]
        if isinstance(idx, tuple):
            assert(len(destr)==len(idx))
            idx = tuple(d(i) for d,i in zip(destr, idx))
        else:
            idx = destr(idx)
        return idx


# Build the table.
index = [(10,3),(10,1),(2,2),('foo',4),('bar',5)]
index = pd.MultiIndex.from_tuples(index)
data = np.random.randn(len(index),2)
table = pd.DataFrame(data=data, index=index)
idx = pd.IndexSlice

table = stringifiedSortIndex(table)
print(table)

# Now, the table rows can be accessed as usual.
table.loc[idx[10],:]
table.loc[idx[:10],:]
table.loc[idx[:'bar',:],:]
table.loc[idx[:,:2],:]

# This works also for simply indexed table.
table = pd.DataFrame(data=data, index=[4,1,'foo',3,'bar'])
table = stringifiedSortIndex(table)
table[:'bar']