Python 日志轮换和压缩

Question

我的脚本允许我将照片导入特定文件夹并通过 JSON 文件导入更新我的数据库 MongoDB。

我建立了一个带有日志轮换和压缩的日志系统。

我有不同的问题：

• 有些操作不是记录在文件中，而是在控制台中。 (subprocess.call)
• 记录器为我创建了三个文件而不是一个文件

python 脚本：

def moveFTPFiles(serverName,userName,passWord,remotePath,localPath,deleteRemoteFiles=False,onlyDiff=False):
    """Connect to an FTP server and bring down files to a local directory"""
    import os
    import sys
    import glob
    from sets import Set
    import ftplib 

    logger.info(' Deleting Files ')

    os.chdir(localDirectoryPath)
    files=glob.glob('*.*')
    for filename in files:
        os.unlink(filename)

    logger.info(' Retreiving Files ')

    try:
        ftp = ftplib.FTP(serverName)
        ftp.login(userName,passWord)
        ftp.cwd(remotePath)
        logger.info(' Connecting ')
        if onlyDiff:
            lFileSet = Set(os.listdir(localPath))
            rFileSet = Set(ftp.nlst())
            transferList = list(rFileSet - lFileSet)
            logger.info(' Missing ' + str(len(transferList)))
        else:
            transferList = ftp.nlst()
        delMsg = "" 
        filesMoved = 0
        for fl in transferList:
            # create a full local filepath
            localFile = localPath + fl
            # print "Create a full local filepath: " + localFile
            grabFile = True
            if grabFile:                
                #open a the local file
                fileObj = open(localFile, 'wb')
                # Download the file a chunk at a time using RETR
                ftp.retrbinary('RETR ' + fl, fileObj.write)
                # Close the file
                # print "Close the file "
                fileObj.close()
                filesMoved += 1
                #print "Uploaded: " + str(filesMoved)
                #sys.stdout.write(str(filesMoved)+' ')
                #sys.stdout.flush()

            # Delete the remote file if requested
            if deleteRemoteFiles:
                ftp.delete(fl)
                delMsg = " and Deleted"         

        logger.info('Files Moved' + delMsg + ': ' + str(filesMoved) + ' on ' + timeStamp())
    except ftplib.all_errors as e:
        logger.error('We have a problem on moveFTPFiles' + '%s' % e)
    ftp.close() # Close FTP connection
    ftp = None

def timeStamp():
    """returns a formatted current time/date"""
    import time
    return str(time.strftime("%a %d %b %Y %I:%M:%S %p"))

def importData(serverName,userName,passWord,directory,filematch,source,destination):
    import socket
    import ftplib
    import os
    import subprocess
    import json

    try:
        ftp = ftplib.FTP(serverName)
        ftp.login(userName,passWord)
        ftp.cwd(directory)
        logger.info(' Connecting ')
        # Loop through matching files and download each one individually
        for filename in ftp.nlst(filematch):
            fhandle = open(filename, 'wb')
            logger.info(' Getting ' + filename)
            ftp.retrbinary('RETR ' + filename, fhandle.write)
            fhandle.close()
        #convert xml to json
        logger.info(' Convert ' + filename + ' to .json ')
        subprocess.call('xml2json -t xml2json -o stockvo.json stockvo.xml --strip_text', shell=True)
        #remove xml file
        logger.info(' Delete ' + filename)
        os.unlink(source+'stockvo.xml')
        #modify json file
        logger.info(' Modify .json file')
        data = json.loads(open("stockvo.json").read())
        with open("stockvo.json", "w") as outfile:
                json.dump(data["Stock"]["Vehicule"], outfile)

        #import json file to MongoDB
        logger.info(' Import json file to MongoDB')
        subprocess.call('mongoimport --db AutoPrivilege -c cars stockvo.json --jsonArray --upsert --drop',shell=True)

        #remove old json file
        logger.info('Delete old .json file')
        ## if file exists, delete it ##
        myfile=destination+"stockvo.json"
        if os.path.isfile(myfile):
            os.remove(myfile)
        #os.unlink(destination+'stockvo.json')

        #move json file
        logger.info('Move .json')
        os.system('mv %s %s' % (source+'stockvo.json', destination+'stockvo.json'))

    except ftplib.all_errors as e:
        logger.error('We have a problem on importData' + '%s' % e)
    ftp.close() # Close FTP connection
    ftp = None

import time
import re
import os
import stat
import logging
import logging.handlers as handlers

class SizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler):
    """
    Handler for logging to a set of files, which switches from one file
    to the next when the current file reaches a certain size, or at certain
    timed intervals
    """
    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None,
                 delay=0, when='h', interval=1, utc=False):
        # If rotation/rollover is wanted, it doesn't make sense to use another
        # mode. If for example 'w' were specified, then if there were multiple
        # runs of the calling application, the logs from previous runs would be
        # lost if the 'w' is respected, because the log file would be truncated
        # on each run.
        if maxBytes > 0:
            mode = 'a'
        handlers.TimedRotatingFileHandler.__init__(
            self, filename, when, interval, backupCount, encoding, delay, utc)
        self.maxBytes = maxBytes

    def shouldRollover(self, record):
        """
        Determine if rollover should occur.

        Basically, see if the supplied record would cause the file to exceed
        the size limit we have.
        """
        if self.stream is None:                 # delay was set...
            self.stream = self._open()
        if self.maxBytes > 0:                   # are we rolling over?
            msg = "%s\n" % self.format(record)
            self.stream.seek(0, 2)  #due to non-posix-compliant Windows feature
            if self.stream.tell() + len(msg) >= self.maxBytes:
                return 1
        t = int(time.time())
        if t >= self.rolloverAt:
            return 1
        return 0

if __name__ == '__main__':      

    #log to a file
    log_filename='/opt/log/importData.log'
    logger=logging.getLogger('importData')
    logger.setLevel(logging.DEBUG)
    handler=SizedTimedRotatingFileHandler(
        log_filename, maxBytes=100, backupCount=5,
        when='s',interval=10,
        # encoding='bz2',  # uncomment for bz2 compression
    )
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    #--- constant connection values
    ftpServerName = "xxxxx.xxxxxx"
    ftpU = "xxxxxxxx"
    ftpP = "xxxxxx"
    remoteDirectoryPath = "/xxxxxx/xxxxxx/xxxxxx/xxxxxxx/"
    localDirectoryPath = "/xxxxx/xxxxxxxxx/xxxxxxxx/xxxxxxx/"

    directory = '/xxxxxxx/'
    filematch = '*.xml'
    source='/xxxxxx/xxxxxxxx/'
    destination='/xxxxxxxx/xxxxxx/'

    deleteAfterCopy = False     #set to true if you want to clean out the remote directory
    onlyNewFiles = True         #set to true to grab & overwrite all files locally
    importData(ftpServerName,ftpU,ftpP,directory,filematch,source,destination)
    moveFTPFiles(ftpServerName,ftpU,ftpP,remoteDirectoryPath,localDirectoryPath,deleteAfterCopy,onlyNewFiles)

importData.log：

2015-04-23 11:33:57,408 INFO 文件已移动：2015 年 4 月 23 日星期四上午 11:33:57 1145

importData.log.2015-04-23_11-33-40：

2015-04-23 11:33:40,896 INFO 删除文件 2015-04-23 11:33:40,956 INFO 检索文件

importData.log2015-04-23_11-33-41：

2015-04-23 11:33:41,386 INFO 连接 2015-04-23 11:33:41,825 INFO 缺少 1145

任何人都可以在 python 中提出一种方法来解决我的问题吗？

Answer 1

SizedTimedRotatingFileHandler(log_filename, maxBytes=100, ...

完全按照它的配置进行 - 最多 100 个字节将记录到一个文件中。将最大大小增加到几（百？）兆字节，您将获得不频繁的轮换。

仅关于文件的部分日志记录，您仅为“importData”模块定义处理程序。其他模块将写入默认处理程序（可能是控制台）。

关于subprocess.call() 本身，它实际上并没有记录任何内容。除非您使用stdout 参数捕获输出，否则输出将打印到普通的stdout。您需要设置一个管道并将其读取到记录器中。