pandasdmx 无法为某些 OECD 数据创建 DataFrame

Question

我使用的是 pandasdmx 0.7.0 版，虽然我在使用频率维度的 OECD 数据集方面取得了成功，但还有其他数据集，例如 Fossil Fuel Supports 数据集，其中没有频率维度m 无法创建 pandas DataFrame，如下所示：

from pandasdmx import Request

# http://stats.oecd.org/sdmx-json/data/FFS_BRA/all/all

oecd = Request('OECD')

try:
    data_response = oecd.data(resource_id='FFS_BRA', key='all/all')
except UnicodeDecodeError:
    pass
except KeyError:
    pass
else:
    oecd_data = data_response.data

    print(oecd_data.dim_at_obs)
    series_list = list(oecd_data.series)
    print(len(series_list))
    print(series_list[0].key)
    print(set(s.key.FUEL for s in oecd_data.series))

    fuel = (s for s in oecd_data.series if s.key.FUEL == 'HARDCOAL')
    df = data_response.write(fuel)

print("completed ...")

Output:

TIME_PERIOD
25
SeriesKeyTuple(MEA='BRA_DT_03', FUEL='HARDCOAL',  IND='CSE',INC='CONSUMPTION', STG='GENER', MEC='DT', LVL='FED')
{'NONBIOJETK', 'LIGNITE', 'HARDCOAL', 'NAPHTHA', 'NONBIODIES', 'CRUDEOIL', 'RESFUEL', 'NATGAS', 'NGL', 'LPG', 'NONBIOGASO'}

File "stackoverflow_dataframe.py", line 23, in <module>
df = data_response.write(fuel)
File "pandasdmx\api.py", line 635, in write
return self._writer.write(source=source, **kwargs)
File "pandasdmx\writer\data2pandas.py", line 109, in write
reverse_obs, fromfreq, parse_time))
File "pandasdmx\writer\data2pandas.py", line 107, in <genexpr>
series_list = list(s for s in self.iter_pd_series(
File "pandasdmx\writer\data2pandas.py", line 228, in iter_pd_series
obs_values, index=series_index, name=series.key)
UnboundLocalError: local variable 'series_index' referenced before assignment

这是非频率维度数据集的正确方法，还是我只是编码错误？

Answer 1

pandasdmx (0.7.0) 的作者的官方回答是使用设置为 False 的 parse_time 参数用于写入函数。然后 write 函数不会尝试生成 DateTime 索引，而是会得到一个字符串索引。

from pandasdmx import Request

# http://stats.oecd.org/sdmx-json/data/FFS_BRA/all/all

oecd = Request('OECD')

try:
    data_response = oecd.data(resource_id='FFS_BRA', key='all/all')
except UnicodeDecodeError:
     pass
except KeyError:
    pass
else:
    oecd_data = data_response.data

    df = data_response.write(oecd_data.series, parse_time=False)
    df.to_csv('FFS_BRA_solution1.csv')

由于 pandasdmx 使用 SDMX ID 为列名和数据返回一个宽格式表，并且由于时间压力，我自己编写了一个长格式表来返回一个长格式表（然后我可以在需要时对其进行透视），可以选择使用SDMX ID 或完整描述。该解决方案已在 835 个 OECD 数据集上进行了测试！

from pandasdmx import Request
import pandas as pd
from collections import OrderedDict

# http://stats.oecd.org/sdmx-json/data/FFS_BRA/all/all

# generator of empty lists
def create(n, constructor=list):
    for _ in range(n):
        yield constructor()

# note this function extracts annual data
# DataFrame in long format
def createDF(sdmx_data, useIDs=False):

    series_list = list(sdmx_data.series)

    # does it have a frequency dimension?
    # if only use the annual data
    keycheck_tuple = series_list[0].key._fields
    for keycheck in keycheck_tuple:
        if keycheck == 'FREQUENCY':
            series_list = (anl for anl in sdmx_data.series if anl.key.FREQUENCY == 'A')
            break
        elif keycheck == 'FREQ':
            series_list = (anl for anl in sdmx_data.series if anl.key.FREQ == 'A')
            break

    # variable series key columns for the data set
    # list of empty lists
    key_columns = list(create(sdmx_data._reader._key_len))

    # fixed columns for time period and values
    time_period_col = []
    value_col = []

    for s in series_list:
        s_key_tuple = s.key
        s_elem_dict = s._elem
        s_reader = s._reader

        obs_dim_dict = s_reader._obs_dim[0]['values']

        total_keys = s_reader._key_len
        key_col_codes = []

        for key in range(total_keys):
            keys_list = s_reader._series_dim[key]['values']
            key_field_abbrev = s_key_tuple._fields[key]
            key_code = s_key_tuple[key_field_abbrev]
            if useIDs:
                key_col_codes.append(key_code)
            else:
                for entry_dict in keys_list:
                    if entry_dict['id'] == key_code:
                        key_col_codes.append(entry_dict['name'])

        obs_dict = s_elem_dict['observations']
        for key, val_list in sorted(obs_dict.items()):
            value_col.append(val_list[0])
            yr = obs_dim_dict[int(key)]['name']
            time_period_col.append(yr)
            for ky in range(sdmx_data._reader._key_len):
                key_columns[ky].append(key_col_codes[ky])

    pandasdict = OrderedDict()

    tp = 'Time Period'
    ob = 'Observation'
    if useIDs:
        tp = 'TIME_PERIOD'
        ob = 'OBS'

    pandasdict[tp] = time_period_col
    pandasdict[ob] = value_col

    for t in range(sdmx_data._reader._key_len):
        if useIDs:
            # use this for abbreviated column names
            pandasdict[s_key_tuple._fields[t]] = key_columns[t]
        else:
            # use this for un-abbreviated column names
            pandasdict[s_reader._series_dim[t]['name']] = key_columns[t]

    oecdDF = pd.DataFrame(pandasdict)

    return oecdDF


# OECD data
oecd = Request('OECD')

try:
    data_response = oecd.data(resource_id='FFS_BRA', key='all/all')
except UnicodeDecodeError:
    pass
except KeyError:
    pass
else:
    oecd_data = data_response.data

    # long format
    df = createDF(oecd_data, useIDs=False)
    df.to_csv('FFS_BRA_solution2_long.csv')

    # wide format
    all_cols_list = df.columns.values.tolist()
    pivot = df.pivot_table(values=all_cols_list[1], index=all_cols_list[0], columns=all_cols_list[2:])
    pivot.to_csv('FFS_BRA_solution2_wide.csv')