Pandas：如何在阅读 CSV 时将其他列添加到最后一列

Question

我在 CSV 中有固定的列。我正在使用 Pandas 的 read_csv。 但有些行有额外的列值。我需要将所有额外的列合并到最后一列。

基本上，我正在尝试读取 CSV，它在某些列中有特殊字符 ( , ) 和 ( ' )，因此它正在拆分起来并创建额外的列。因此，我收到 'ParserError: Error tokenizing data。 C 错误：第 7 行中应有 4 个字段，看到 5'。 所以我需要一种方法将最后一列/额外列动态转换为最后一列

例如，在以下示例中，问题出在最后一列，它混合了 , 和 '。

from StringIO import StringIO
import pandas as pd

csv = r"""dummy,obj,loc,query
bar,6usrg82hwsa3,a,'select * from abc'
bar,b6usrg82hwsa3,a,'select * from abc'
bar,4g9cgbm813czs,a,'select * from abc'
bar,fhf8upax5cxsz,b,'select * from abc'
bar,cnphq355f5rah,b,'select * from abc'
bar,b6usrg82hwsa3,b,'SELECT LIST(HIGHLIGHT, ',') WITHIN GR...'"""

df = pd.read_csv(StringIO(csv), quotechar="'")

这将抛出 'Error tokenizing data'

预期的输出是

>>> print(df)
  dummy            obj loc              query
0   bar   6usrg82hwsa3   a  select * from abc
1   bar  b6usrg82hwsa3   a  select * from abc
2   bar  4g9cgbm813czs   a  select * from abc
3   bar  fhf8upax5cxsz   b  select * from abc
4   bar  cnphq355f5rah   b  select * from abc
5   bar  b6usrg82hwsa3   b  SELECT LIST(HIGHLIGHT, ',') WITHIN GR...

Answer 1

如果您的数据包含文本列，则切勿使用 .csv 来存储您的数据，即使它们当时不包含逗号。在这种情况下，仅当且仅当您严格知道数据中的逗号是不可能的时才使用它。改用制表符分隔或其他文件类型。您可以使用以下解决方案，它适用于您的情况，

def refactor_text(csv):
    my_dict = dict(
        dummy=[],
        obj=[],
        loc=[],
        query=[]
        )
    for i,line in enumerate(csv.split('\n')):
        if i == 0:
            continue
        line_args = line.split(',')
        for i,key in enumerate(my_dict.keys()):
            if not key == 'query':
                my_dict[key].append(line_args[i])
            else:
                my_dict[key].append(','.join(line_args[i:]))
    return my_dict


df = pd.DataFrame(refactor_text(csv))

函数 refactor_text 接受一个参数作为字符串 (csv)，如果您直接从文件访问或其他情况，您可能希望重构它。

Answer 2

一种可能的解决方案是创建一列DataFrame，其分隔符不在| 等数据中，然后使用Series.str.split 和n 参数：

from io import StringIO
import pandas as pd

csv = r"""dummy,obj,loc,query
bar,6usrg82hwsa3,a,'select * from abc'
bar,b6usrg82hwsa3,a,'select * from abc'
bar,4g9cgbm813czs,a,'select * from abc'
bar,fhf8upax5cxsz,b,'select * from abc'
bar,cnphq355f5rah,b,'select * from abc'
bar,b6usrg82hwsa3,b,'SELECT LIST(HIGHLIGHT, ',') WITHIN GR...'"""

df = pd.read_csv(StringIO(csv), quotechar="'", sep='|')
print (df)
                                 dummy,obj,loc,query
0             bar,6usrg82hwsa3,a,'select * from abc'
1            bar,b6usrg82hwsa3,a,'select * from abc'
2            bar,4g9cgbm813czs,a,'select * from abc'
3            bar,fhf8upax5cxsz,b,'select * from abc'
4            bar,cnphq355f5rah,b,'select * from abc'
5  bar,b6usrg82hwsa3,b,'SELECT LIST(HIGHLIGHT, ',...

---

df1 = df.iloc[:, 0].str.split(',', expand=True, n=3).apply(lambda x: x.str.strip("'"))
df1.columns = df.columns[0].split(',')
print (df1)
  dummy            obj loc                                     query
0   bar   6usrg82hwsa3   a                         select * from abc
1   bar  b6usrg82hwsa3   a                         select * from abc
2   bar  4g9cgbm813czs   a                         select * from abc
3   bar  fhf8upax5cxsz   b                         select * from abc
4   bar  cnphq355f5rah   b                         select * from abc
5   bar  b6usrg82hwsa3   b  SELECT LIST(HIGHLIGHT, ',') WITHIN GR...

Answer 3

这个作品： 对于大型数据集，我认为我不确定这一点。

csv = r"""dummy,obj,loc,query
bar,6usrg82hwsa3,a,'select * from abc'
bar,b6usrg82hwsa3,a,'select * from abc'
bar,4g9cgbm813czs,a,'select * from abc'
bar,fhf8upax5cxsz,b,'select * from abc'
bar,cnphq355f5rah,b,'select * from abc'
bar,b6usrg82hwsa3,b,'SELECT LIST(HIGHLIGHT, ',') WITHIN GR...'"""

lengths = []
for line in csv.split('\n'):
    lengths.append(line.count(',') + 1)

n_columns = min(lengths)
rows = []
for line in csv.split('\n'):
    temp = line.split(',')
    row = temp[:(n_columns-1)]
    temp = temp[(n_columns-1):]
    try:
        temp = [string + ',' for string in temp[:-1]] + temp[-1]
        row += [''.join(temp)]
    except IndexError:
        pass
    rows.append(row)
data = pd.DataFrame(data=rows[1:] , columns=rows[0])

print(data)

# dummy            obj loc                                     query
# 0   bar   6usrg82hwsa3   a                       'select * from abc'
# 1   bar  b6usrg82hwsa3   a                       'select * from abc'
# 2   bar  4g9cgbm813czs   a                       'select * from abc'
# 3   bar  fhf8upax5cxsz   b                       'select * from abc'
# 4   bar  cnphq355f5rah   b                       'select * from abc'
# 5   bar  b6usrg82hwsa3   b  'SELECT LIST(HIGHLIGHT '') WITHIN GR...'