在python中用子进程重定向stdout非常慢

Question

当我在 Python 中使用 subprocess 重定向标准输出时，吞吐量非常慢。我做错了吗？

基本上，我通过管道将外部程序的标准输出放入队列中。然后在另一个函数中，我在控制台中打印它。

这是一个使用 hexdump 生成随机输出的示例代码：

from subprocess import Popen, PIPE
from queue import Queue
import sys
from threading import Thread, Event
import threading

class Buffer(Queue):

    def __init__(self, *args, **kwargs):
        Queue.__init__(self, *args, **kwargs)

    def write(self, line):
        self.put_nowait(line)
        self.join()

    def read(self):
        element = self.get_nowait()
        self.task_done()
        return element

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.returncode is None:
        for line in hexdump.stdout.readlines(8192):
            buffer.write(line)
            if stopped.is_set():
                hexdump.terminate()
                hexdump.wait()
                print('process terminated.')
                break

def read_output(buffer, stopped):
    while not stopped.is_set():
        while not buffer.empty():
            output = buffer.read()
            print('********* output: {}'.format(output))
            sys.stdout.flush()
    print('stopped')
    sys.stdout.flush()


buffer = Buffer()
stopped = Event()


generate_random_output = Thread(target=write_output, args=(buffer, stopped))
generate_random_output.name = 'generate_random_output'
generate_random_output.start()

process_output = Thread(target=read_output, args=(buffer, stopped))
process_output.name = 'process_output'
process_output.start()

try:
    while True:
        continue
except KeyboardInterrupt:
    stopped.set()
    generate_random_output.join()
    process_output.join()
    print('finished generating')
    print('finished processing')

我将不胜感激。

Answer 1

而不是将您的输出重定向到队列 - 直接处理它：

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.poll() is None:
        while not stopped.is_set():
            for line in iter(hexdump.stdout.readline, b''):
                print('********* output: %s' % line.decode(), end='')
                sys.stdout.flush()

        hexdump.terminate()
        hexdump.wait()
        print('process terminated.')
        break