在 python 脚本中使用 utf-8 文件名时出错

Question

我有一个看似不可能的难题，希望你们能帮助我指出正确的方向。几个星期以来，我一直在这个项目中来来往往，我认为是时候解决这个问题了，希望能得到你的帮助。

我正在制作一个脚本，它应该从目录结构中读取一堆 .xls excel 文件，解析它们的内容并将其加载到 mysql 数据库中。现在，在 main 函数中，（克罗地亚语）文件名列表被传递给 xlrd，这就是问题所在。

环境是最新的 FreeBSD 9.1。

执行脚本时出现以下错误：

mars:~/20130829> python megascript.py

Python version: 2.7.5

Filesstem encoding is: UTF-8
Removing error.log if it exists...
It doesn't.
Done!
Connecting to database...
Done!
MySQL database version: 5.6.13
Loading pilots...
Done!
Loading tehnicians...
Done!
Loading aircraft registrations...
Done!
Loading file list...
Done!
Processing files...
/2006/1_siječanj.xls
Traceback (most recent call last):
  File "megascript.py", line 540, in <module>
    main()
  File "megascript.py", line 491, in main
    data = readxlsfile(files, 'UPIS', piloti, tehnicari, helikopteri)
  File "megascript.py", line 129, in readxlsfile
    workbook = open_workbook(f)
  File "/usr/local/lib/python2.7/site-packages/xlrd-0.9.2-py2.7.egg/xlrd/__init__.py",     line 394, in open_workbook
    f = open(filename, "rb")
IOError: [Errno 2] No such file or directory: u'/2006/1_sije\u010danj.xls'

我已包含完整的输出以使代码流更易于理解。 我想问题出在 xlrd 不接受 utf-8 文件列表。我不确定如何在不乱用 xlrd 代码的情况下解决这个问题。有什么想法吗？

代码如下：

#! /usr/bin/env/python
# -#*- coding: utf-8 -*-
import os, sys, getopt, codecs, csv, MySQLdb, platform
from mmap import mmap,ACCESS_READ
from xlrd import open_workbook, xldate_as_tuple

# Define constants

NALET_OUT = ''
PUTNICI_OUT = ''
DB_HOST = 'localhost'
DB_USER = 'user'
DB_PASS = 'pass'
DB_DATABASE = 'eth'
START_DIR = u'./'
ERROR_FILE = START_DIR + 'mega_error.log'

# Functions    
def isNumber(s):
# Check if a string could be a number
    try:
        float(s)
        return True
    except ValueError:
        return False

def getMonth(f):
# Izvuci mjesec iz imena datoteke u formatu "1_sijecanj.xls"

    temp = os.path.basename(f)
    temp = temp.split('_')
    mjesec = int(temp[0])
    return mjesec

def getYear(f):
# Izvuci godinu iz path
    f = f.split('/')
    godina = f[-2]
    return godina

def databaseVersion(cur):
# Print Mysql database version
    try:
        cur.execute("SELECT VERSION()")
        result = cur.fetchone()
    except MySQLdb.Error, e:
        try:
            print "MySQL Error [%d]: %s]" % (e.args[0], e.args[1])
        except IndexError:
            print "MySQL  Error: %s" % (e.args[0], e.args[1])

    print "MySQL database version: %s" % result

def getQuery(cur, sql_query):
# Perform passed query on passed database
    try:
        cur.execute(sql_query)
        result = cur.fetchall()
    except MySQLdb.Error, e:
        try:
            print "MySQL Error [%d]: %s]" % (e.args[0], e.args[1])
        except IndexError:
            print "MySQL  Error: %s" % (e.args[0], e.args[1])
    return result

def getFiles():

    files = []

    # Find subdirectories
    for i in [x[0] for x in os.walk(START_DIR)]:

        if (i != '.' and isNumber(os.path.basename(i))):

            # Find files in subdirectories
            for j in [y[2] for y in os.walk(i)]:

                # For every file in file list
                for y in j:
                    fn, fe = os.path.splitext(y)
                    is_mj = fn.split("_")
                    if(fe == '.xls' and y.find('_') and isNumber(is_mj[0])):
                        mj = fn.split('_')
                        files.append(i.lstrip('.') + "/" + y)

    # Sort list cronologically
    files.sort(key=lambda x: getMonth(x))
    files.sort(key=lambda x: getYear(x))

    return files

def errhandle(f, datum, var, vrijednost, ispravka = "NULL"):
# Get error information, print it on screen and write to error.log

    f = unicode(str(f), 'utf-8')
    datum = unicode(str(datum), 'utf-8')
    var = unicode(str(var), 'utf-8')
    try:
        vrijednost = unicode(str(vrijednost.decode('utf-8')), 'utf-8')
    except UnicodeEncodeError:
        vrijednost = vrijednost
    ispravka = unicode(str(ispravka), 'utf-8')

    err_f = codecs.open(ERROR_FILE, 'a+', 'utf-8')
    line = f + ": " + datum + " " + var + "='" + vrijednost\
                    + "' Ispravka='" + ispravka + "'"

    #print "%s" % line

    err_f.write(line)
    err_f.close()

def readxlsfile(files, sheet, piloti, tehnicari, helikopteri):
# Read xls file and return a list of rows

    data = []
    nalet = []
    putn = []
    id_index = 0

    # For every file in list
    for f in files:
        print "%s" % f
        temp = f.split('/')
        godina = str(temp[-2])
        temp = os.path.basename(f).split('_')
        mjesec = str(temp[0])

        workbook = open_workbook(f)
        sheet = workbook.sheet_by_name('UPIS')

        # For every row that doesn't contain '' or 'POSADA' or 'dan' etc...
        for ri in range(sheet.nrows):
            if sheet.cell(ri,1).value!=''\
               and sheet.cell(ri,2).value!='POSADA'\
               and sheet.cell(ri,1).value!='dan'\
               and (sheet.cell(ri,2).value!=''):

                temp = sheet.cell(ri, 1).value
                temp = temp.split('.')
                dan = temp[0]

                # Datum
                datum = "'" + godina + "-" + mjesec + "-" + dan + "'"

                # Kapetan
                kapetan = ''
                kapi=''
                if sheet.cell(ri, 2).value == "":
                    kapetan = "NULL"
                else:
                    kapetan = sheet.cell(ri, 2).value
                    if kapetan[-1:] == " ":
                        errhandle(f, datum, 'kapetan', kapetan, kapetan[-1:])
                        kapetan = kapetan[:-1]
                    if(kapetan):
                        try:
                            kapi = [x[0] for x in piloti if x[2].lower() == kapetan]
                            kapi = kapi[0]
                        except ValueError:
                            errhandle(f, datum, 'kapetan', kapetan, '')
                            kapetan = ''
                        except IndexError:
                            errhandle(f, datum, 'kapetan', kapetan, '')
                            kapi = 'NULL'
                    else:
                        kapi="NULL"

                # Kopilot
                kopilot = ''
                kopi = ''
                if sheet.cell(ri, 3).value == "":
                    kopi = "NULL"
                else:
                    kopilot = sheet.cell(ri, 3).value
                    if kopilot[-1:] == " ":
                        errhandle(f, datum,'kopilot', kopilot,\
                                  kopilot[:-1])
                    if(kopilot):
                        try:
                            kopi = [x[0] for x in piloti if x[2].lower() == kopilot]
                            kopi = kopi[0]
                        except ValueError:
                            errhandle(f, datum,'kopilot', kopilot, '')
                        except IndexError:
                            errhandle(f, datum, 'kopilot', kopilot, '')
                            kopi = 'NULL'
                    else:
                        kopi="NULL"

                # Teh 1
                teh1 = ''
                t1i = ''
                if sheet.cell(ri, 4).value=='':
                    t1i = 'NULL'
                else:
                    teh1 = sheet.cell(ri, 4).value
                    if teh1[-1:] == " ":
                        errhandle(f, datum,'teh1', teh1, teh1[:-1])
                        teh1 = 'NULL'
                    if(teh1):
                        try:
                            t1i = [x[0] for x in tehnicari if x[2].lower() == teh1]
                            t1i = t1i[0]
                        except ValueError:
                            errhandle(f, datum,'teh1', teh1, '')
                        except IndexError:
                            errhandle(f, datum, 'teh1', teh1, '')
                            t1i = 'NULL'
                        else:
                            t1i="NULL"

                # Teh 2
                teh2=''
                t2i=''
                if sheet.cell(ri, 5).value=='':
                    t2i = "NULL"
                else:
                    teh2 = sheet.cell(ri, 5).value
                    if teh2[-1:] == " ":
                        errhandle(f, datum,'teh2', teh2, teh2[-1:])
                        teh2 = ''
                    if(teh2):
                        try:
                            t2i = [x[0] for x in tehnicari if x[2].lower() == teh2]
                            t2i = t2i[0]
                        except ValueError:
                            errhandle(f, datum,'teh2', teh2, 'NULL')
                            t2i = 'NULL'
                        except IndexError:
                            errhandle(f, datum,'teh2', teh2, 'NULL')
                            t2i = 'NULL'
                    else:
                        t2i="NULL"

                # Oznaka
                oznaka = ''
                heli = ''
                if sheet.cell(ri, 6).value=="":
                    oznaka = errhandle(f, datum, "helikopter", oznaka, "")
                else:
                    oznaka = str(int(sheet.cell(ri, 6).value))
                    try:
                        heli = [x[0] for x in helikopteri if x[0] == oznaka]
                    except ValueError:
                        errhandle(f, datum, 'helikopter', oznaka, '')
                    except IndexError:
                        errhandle(f, datum, 'helikopter', oznaka, '')
                        heli = ''

                # Uvjeti
                uvjeti = sheet.cell(ri, 9).value

                # Letova
                letova_dan = 0
                letova_noc = 0
                letova_ifr = 0
                letova_sim = 0
                if sheet.cell(ri, 7).value == "":
                    errhandle(f, datum, 'letova', letova, '')
                else:
                    letova = str(int(sheet.cell(ri, 7).value))

                if uvjeti=="vfr":
                    letova_dan = letova
                elif uvjeti=="ifr":
                    letova_ifr = letova
                elif uvjeti=="sim":
                    letova_sim = letova
                else:
                    letova_noc = letova

                #Block time
                bt_dan = "'00:00:00'"
                bt_noc = "'00:00:00'"
                bt_ifr = "'00:00:00'"
                bt_sim = "'00:00:00'"
                try:
                    bt_tpl = xldate_as_tuple(sheet.cell(ri, 8).value, workbook.datemode)
                    bt_m = bt_tpl[4]
                    bt_h = bt_tpl[3]
                    bt = "'" + str(bt_h).zfill(2)+":"+str(bt_m)+":00'"
                except ValueError or IndexError:
                    errhandle(f, datum, 'bt', sheet.cell(ri,8).value, '')
                if uvjeti[:3]=="vfr":
                    bt_dan = bt
                elif uvjeti[:3]=="ifr":
                    bt_ifr = bt
                elif uvjeti[:3]=="sim":
                    bt_sim = bt
                elif uvjeti[:2] == "no":
                    bt_noc = bt
                else:
                    errhandle(f, datum, 'uvjeti', uvjeti, '')

                # Vrsta leta
                vrsta = "'" + sheet.cell(ri, 10).value + "'"

                # Vjezba
                vjezba = 'NULL';
                try:
                    vjezba = sheet.cell(ri, 11).value
                    if vjezba == '':
                        # Too many results
                        #errhandle(f, datum, 'vjezba', vjezba, '')
                        vjezba = 'NULL'
                    if vjezba == "?":
                        errhandle(f, datum, 'vjezba', str(vjezba), '')
                        vjezba = 'NULL'
                    if str(vjezba) == 'i':
                        errhandle(f, datum, 'vjezba', str(vjezba), '')
                        vjezba = 'NULL'
                    if str(vjezba)[-1:] == 'i':
                        errhandle(f, datum, 'vjezba', str(vjezba),\
                        str(vjezba).rstrip('i'))
                        vjezba = str(vjezba).rstrip('i')
                    if str(vjezba).find(' i ') != -1:
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split(' i ')[0])
                        vjezba = str(vjezba).split(' i ')
                        vjezba = vjezba[0]
                    if str(vjezba)[-1:] == 'm':
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).rstrip('m'))
                        vjezba = str(vjezba).rstrip('m')
                    if str(vjezba).find(';') != -1:
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split(';')[0])
                        temp = str(vjezba).split(';')
                        vjezba = temp[0]
                    if str(vjezba).find('/') != -1:
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split('/')[0])
                        temp = str(vjezba).split('/')
                        vjezba = temp[0]
                    if str(vjezba).find('-') != -1:
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split('-')[0])
                        temp = str(vjezba).split('-')
                        vjezba = temp[0]
                    if str(vjezba).find(',') != -1:
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split(',')[0])
                        temp = str(vjezba).split(',')
                        vjezba = temp[0]
                    if str(vjezba).find('_') != -1:
                        errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split('_')[0])
                        temp = str(vjezba).split('_')
                        vjezba = temp[0]
                    if str(vjezba) == 'bo':
                        errhandle(f, datum, 'vjezba', str(vjezba), '')
                        vjezba = 'NULL'
                    if str(vjezba).find(' ') != -1:
                        if str(vjezba) == 'pp 300':
                            errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split(' ')[1])
                            temp = str(vjezba).split(' ')
                            vjezba = temp[1]
                        else:
                            errhandle(f, datum, 'vjezba', str(vjezba), str(vjezba).split(' ')[0])
                            temp = str(vjezba).split(' ')
                            vjezba = temp[0]
                    if str(vjezba) == 'pp':
                        errhandle(f, datum, 'vjezba', str(vjezba), '')
                        vjezba = ''
                except UnicodeEncodeError:
                    errhandle(f, datum, 'Unicode error! vjezba', vjezba, '')

                if vjezba != 'NULL':
                    vjezba = int(float(vjezba))

                # Visinska slijetanja

                # Putnici
                vp1 = str(sheet.cell(ri, 12).value)
                bp1 = str(sheet.cell(ri, 13).value)
                vp2 = str(sheet.cell(ri, 14).value)
                bp2 = str(sheet.cell(ri, 15).value)

                # Teret
                teret = ''
                teret = str(sheet.cell(ri, 16).value)
                if teret == '':
                    teret = 0

                # Baja
                baja = ''
                if sheet.cell(ri, 17).value == '':
                    baja = 0
                else:
                    baja = int(sheet.cell(ri, 17).value) / 2 # dodano /2 da se dobiju tone  

# Redosljed csv
                id_index = id_index + 1

                row = [id_index, datum, kapi, kopi, t1i, t2i, oznaka,\
                       letova, letova_dan, letova_noc, letova_ifr,\
                       letova_sim, bt, bt_dan, bt_noc, bt_ifr,\
                       bt_sim, vrsta, vjezba, teret, baja]

                row = [str(i) for i in row]
                nalet.append(row)

                putn = []
                if bp1 != '':
                    put = [id_index, vp1, bp1]
                    putn.append(put)
                if bp2 != '':
                    put = [id_index, vp2, bp2]
                    putn.append(put)

    data.append(nalet)
    data.append(putn)
    return data

def main():

    # Python version
    print "\nPython version: %s \n" % platform.python_version()

    # Print filesystem encoding
    print "Filesstem encoding is: %s" % sys.getfilesystemencoding()

    # Remove error file if exists
    print "Removing error.log if it exists..."
    try:
        os.remove(ERROR_FILE)
        print "It did."
    except OSError:
        print "It doesn't."
        pass
    print "Done!"

    # Connect to database
    print "Connecting to database..."
    db = MySQLdb.connect(DB_HOST, DB_USER, DB_PASS, DB_DATABASE,\
                         use_unicode=True, charset='utf8')
    cur=db.cursor()
    print "Done!"

    # Database version

    databaseVersion(cur)

    # Load pilots, tehnicians and helicopters from db

    print "Loading pilots..."
    sql_query = "SELECT eth_osobnici.id, eth_osobnici.ime,\
    eth_osobnici.prezime FROM eth_osobnici RIGHT JOIN \
    eth_letacka_osposobljenja ON eth_osobnici.id=\
    eth_letacka_osposobljenja.id_osobnik WHERE \
    eth_letacka_osposobljenja.vrsta_osposobljenja='kapetan' \
    OR eth_letacka_osposobljenja.vrsta_osposobljenja='kopilot'"

    #piloti = []
    #piloti = getQuery(cur, sql_query)


    piloti=[]
    temp = []
    temp = getQuery(cur, sql_query)
    for row in temp:
        piloti.append(row)
    print "Done!"

    print "Loading tehnicians..."
    sql_query = "SELECT eth_osobnici.id, eth_osobnici.ime,\
    eth_osobnici.prezime FROM eth_osobnici RIGHT JOIN \
    eth_letacka_osposobljenja ON eth_osobnici.id=\
    eth_letacka_osposobljenja.id_osobnik WHERE \
    eth_letacka_osposobljenja.vrsta_osposobljenja='tehničar 1' \
    OR eth_letacka_osposobljenja.vrsta_osposobljenja='tehničar 2'"

    tehnicari=[]
    temp = []
    temp = getQuery(cur, sql_query)
    for row in temp:
        tehnicari.append(row)
    print "Done!"

    print "Loading aircraft registrations..."
    sql_query = "SELECT id FROM eth_helikopteri"

    helikopteri=[]
    temp = []
    temp = getQuery(cur, sql_query)
    for row in temp:
        helikopteri.append(row)
    print "Done!"

    # Get file names to process
    print "Loading file list..."
    files = getFiles()
    print "Done!"

    # Process all files from array
    print "Processing files..."
    data = readxlsfile(files, 'UPIS', piloti, tehnicari, helikopteri)
    print "Done!"

    # Enter new information in database
    result = 0

    print "Reseting database..."
    sql_query = "DELETE FROM eth_nalet"
    cur.execute(sql_query)
    db.commit()
    sql_query = "ALTER TABLE eth_nalet AUTO_INCREMENT=0"
    cur.execute(sql_query)
    db.commit()
    print "Done!"

    print "Loading data in 'eth_nalet'..."

    for row in data[0]:
        sql_query = """INSERT INTO eth_nalet (id, datum, kapetan, 
        kopilot, teh1, teh2, registracija, letova_uk, letova_dan, 
        letova_noc, letova_ifr, letova_sim, block_time, block_time_dan, 
        block_time_noc, block_time_ifr, block_time_sim, vrsta_leta,
        vjezba, teret, baja) VALUES (%s)""" % (", ".join(row))

        cur.execute(sql_query)
        db.commit()

    print "Done!"

    print "Loading data in 'eth_putnici'..."

    for row in data[1]:
        sql_query = """INSERT INTO eth_putnici (id_leta,
        vrsta_putnika, broj_putnika) VALUES (%s)""" % (", ".join(row))

        cur.execute(sql_query)
        db.commit()

    print "Done!"

    # Close the database connection
    print "Closing database connection..."
    if cur:
        cur.close()
    if db:
        db.close()
    print "Database closed!"

if __name__ == '__main__':
        main()

我很抱歉没有在代码中翻译 cmets，这是我的一个旧项目，我现在倾向于用英文制作 cmets。如果有什么需要解释的，请开火。

有趣的是，如果我将文件列表打印到屏幕上，它们会显示得很好。但是当它们被传递到 xlrd 时，它们的格式似乎不正确。

尊敬的， 我

Answer 1

我终于找到了一个错误！毕竟这不是由于编码错误。这是一个逻辑错误。

在函数 getFiles() 中，我去掉了开头的“。”从文件列表中，并没有像我应该的那样删除“./”。因此，文件名自然是“/2006/1_siječanj.xls”而不是“2006/1_siječanj.xls”。这是一个 IOError 而不是 UnicodeEncodeError。我疏忽的结果是脚本试图找到绝对路径而不是相对路径。

这很尴尬。谢谢大家，希望这篇文章能帮助其他人更多地关注python向我们抛出的错误类型。

Answer 2

看起来 xlrd 在尝试打开文件之前没有将 Unicode 类型转换为本地编码类型。 Python 已经猜到文件系统名称编码是 UTF-8 并且已经正确地将 č 转换为正确的 Unicode 点。

有两种方法可以解决这个问题：

1. 在要求 xlrd 打开它之前尝试对 Unicode 文件名进行编码：

   workbook = open_workbook(f.encode(sys.getfilesystemencoding() ) )
   

2. 使用原始 8 位文件名，不要将文件名转换为 Unicode

   START_DIR = './'
   

恕我直言，如果文件名没有使用 UTF-8 文件名写入，选项 2 可能更安全。

UPD

注意，当给定路径是 Unicode 字符串时，os.walk 返回 Unicode 字符串。正常的字符串路径将返回二进制字符串。这与 os.listdir (http://docs.python.org/2/library/os.html#os.listdir) 的行为相同。

例子：

$ ls
€.txt
$ python
>>> import os
>>> os.listdir(".")
['\xe2\x82\xac.txt']
>>> os.listdir(u".")
[u'\u20ac.txt']

(e282 = UTF-8 €)

记住：在 Unix 中，与 Windows 不同，文件名不包含编码提示。文件名只是 8 位字符串。如果要将它们转换为不同的编码，则需要知道它们是使用什么编码创建的。