使用多处理时无法在 python 中创建日志文件分钟

Question

使用多处理时无法在 python 中按分钟创建日志文件

import multiprocessing
import logging.handlers
import time
import logging.handlers
import multiprocessing

log_file_name = "test1.log"
logging_level = logging.DEBUG
formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
handler = logging.handlers.TimedRotatingFileHandler(log_file_name, when='M', interval=1)
handler.suffix = "%Y-%m-%d"
handler.setFormatter(formatter)
logger = logging.getLogger(log_file_name)
logger.addHandler(handler)
logger.setLevel(logging_level)

def main_1():
    print("main 1")
    logger.info("A Sample Log Statement main_1 ")
    time.sleep(2)
    logger.info("A Sample Log Statement main_1 ")

def main_2():
    print("main 2")
    logger.info("A Sample Log Statement main_2 ")
    time.sleep(2)
    logger.info("A Sample Log Statement main_2 ")


def isReady():
    while (True):
        p1 = multiprocessing.Process(target=main_1)
        p2 = multiprocessing.Process(target=main_2)
        p1.start()
        p2.start()
        p1.join()
        p2.join()

if __name__ == "__main__":
    isReady()

**Error:**
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\user2\\WorkSapace\\MULTI\\FileTransfer_dec_06\\test1.log' -> 'D:\\venkatareddy.m\\WorkSapace\\Uniper\\FileTransfer_dec_06\\test1.log.2021-12-13'
Call stack:
  File "<string>", line 1, in <module>
  File "C:\Program Files\Python39\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\Program Files\Python39\lib\multiprocessing\spawn.py", line 129, in _main
    return self._bootstrap(parent_sentinel)
  File "C:\Program Files\Python39\lib\multiprocessing\process.py", line 315, in _bootstrap
    self.run()
  File "C:\Program Files\Python39\lib\multiprocessing\process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "D:\user2\WorkSapace\MULTI\FileTransfer_dec_06\logging_2_new.py", line 27, in main_2
    logger.info("A Sample Log Statement main_2 ")
Message: 'A Sample Log Statement main_2 '
Arguments: ()
--- Logging error ---
Traceback (most recent call last):

Answer 1

问题在于记录器使用threading.RLock 在多个线程记录到同一个文件之间提供线程安全性，但不能真正处理多个进程记录到同一个文件。

可能最好的解决方案是创建一个托管日志记录对象，该对象“存在”在multiprocessing.BaseManager 的子类所拥有的单独地址空间中，它启动一个线程来等待对此的方法请求记录对象。实际上传递给您的进程的是一个 proxy 对象，该对象相当于对上述托管记录器的远程方法调用，然后单线程执行这些调用。

我已经修改了代码，以便在出现问题时更有可能失败，也更有可能终止。

import multiprocessing
from multiprocessing.managers import BaseManager
import logging.handlers


def init_log():
    log_file_name = "test.log"
    logging_level = logging.DEBUG
    formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
    handler = logging.handlers.TimedRotatingFileHandler(log_file_name, when='M', interval=1)
    handler.suffix = "%Y-%m-%d"
    handler.setFormatter(formatter)
    logger = logging.getLogger(log_file_name)
    logger.addHandler(handler)
    logger.setLevel(logging_level)
    return logger


class LoggerManager(BaseManager):
    pass


class MyLogger:
    def __init__(self):
        self.logger = init_log()

    def info(self, *args, **kwargs):
        self.logger.info(*args, **kwargs)

    # define other methods such as degug if required:



def main_1(logger):
    print("main 1")
    for _ in range(20):
        logger.info("A Sample Log Statement main_1")

def main_2(logger):
    print("main 2")
    for _ in range(20):
        logger.info("A Sample Log Statement main_2")


def isReady():
    LoggerManager.register('Logger', MyLogger)
    with LoggerManager() as manager:
        my_logger = manager.Logger()
        for _ in range(5):
            p1 = multiprocessing.Process(target=main_1, args=(my_logger,))
            p2 = multiprocessing.Process(target=main_2, args=(my_logger,))
            p1.start()
            p2.start()
            p1.join()
            p2.join()

if __name__ == "__main__":
    isReady()

另一种方法是通过继承logging.handlers.TimedRotatingFileHandler 并用multiprocessing.RLock 实例实例化它来解决问题。然后我通过使用multiprocessing.RLock 实例初始化lock 属性来覆盖基类的createLock 方法。不用说，所有进程都必须使用相同的 multiprocessing.RLock 实例。

import multiprocessing
import logging.handlers


class MyHandler(logging.handlers.TimedRotatingFileHandler):
    def __init__(self, lock, *args, **kwargs):
        self._mp_lock = lock # for multiprocessing "thread" safety
        super().__init__(*args, **kwargs)

    def createLock(self):
        self.lock = self._mp_lock


def init_log(lock):
    global logger

    log_file_name = "test.log"
    logging_level = logging.DEBUG
    formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s')
    handler = MyHandler(lock, log_file_name, when='M', interval=1)
    handler.suffix = "%Y-%m-%d"
    handler.setFormatter(formatter)
    logger = logging.getLogger(log_file_name)
    logger.addHandler(handler)
    logger.setLevel(logging_level)

def main_1(lock):
    init_log(lock)
    print("main 1")
    for _ in range(20):
        logger.info("A Sample Log Statement main_1")

def main_2(lock):
    init_log(lock)
    print("main 2")
    for _ in range(20):
        logger.info("A Sample Log Statement main_2")


def isReady():
    lock = multiprocessing.RLock()
    for _ in range(5):
        p1 = multiprocessing.Process(target=main_1, args=(lock,))
        p2 = multiprocessing.Process(target=main_2, args=(lock,))
        p1.start()
        p2.start()
        p1.join()
        p2.join()

if __name__ == "__main__":
    isReady()