从python中的大量文件中查找单词的最小值？

Question

我有 1000 个 .txt 文件，我在这些文件上运行此代码。我要做的是在文件中找到 ENSG 的最大值，并删除其他小于最大值的 ENSG 值。然后我必须通过查看 utr 长度从同一文本文件中找到最小值并将其保存在我的 output.txt 文件中。我想要 1 个 output.txt 文件中的 1000 个 .txt 文件的输出。此代码几乎可以正常工作，但它会覆盖结果。它只显示最后一个文件的结果：

import glob
f2 = glob.glob("./*.txt")
all_text=""
for fpath in f2:
    f = open(fpath,"r")
    list_lines = f.readlines()
    dic={}
    sent="ENSG"
    temp_list=[]
    for line in list_lines:
        all_text=all_text+line
        name= line.rsplit()[0].strip()
        score=line.rsplit()[1].strip()
        dic[name]=score
    for i in dic.keys():
        if sent in i:
            temp_list.append(dic[i])
            hiegh_score=max(temp_list)
    def check(index):
        reverse_text=all_text[index+1::-1]
        index2=reverse_text.find("\n")
        if sent==reverse_text[:index2+1][::-1][1:len(sent)+1]:
            return False
        else:
            return True

    list_to_min=dic.values()
    for i in temp_list:
        if i!=hiegh_score:
            index=all_text.find(str(i))
            while check(index):
                index=all_text.find(str(i),index+len(str(i)))
            all_text=all_text[0:index]+all_text[index+len(str(i)):]
            list_to_min.remove(str(i))
file2=open("my_try4.txt","w")
file2.write(all_text)
min_score= min(list_to_min)
for j in dic.keys():
    if min_score==dic[j]:
        k="min score is :"+str(min_score)+" for person "+j
        file2.write(k)
print "%6d : %s" % (len(list_lines),fpath)
file2.close()
f.close()

我有这样的文本文件 4.txt:

ENSBTAG00000020679  197
ENSCAFG00000009872  2585
ENSG00000018236 1935
ENSG00000018236 230
ENSG00000018236 257
ENSG00000018236 338
ENSG00000018236 922
ENSG00000018236 922
ENSRNOG00000004438  14
ENSRNOG00000004438  14

现在它应该选择 1935 的 ENSG 并删除 ENSG 的所有其他值。现在文本文件应该如下所示：

    ENSBTAG00000020679  197
    ENSCAFG00000009872  2585
    ENSG00000018236 1935
    ENSRNOG00000004438  14
    ENSRNOG00000004438  14

现在，通过查看这个文本文件，我们找到了最短的值并将其保存在一个文本文件中（我们对 1000 个文件执行此操作，并且输出应该在 1 个文件上）。

output.txt
textfile4 14

Answer 1

重写这个比找出你的代码出了什么问题更容易：

import os.path
import glob
import re
import itertools
from collections import namedtuple, deque
from operator import attrgetter

R_PREFIX_VALUE = re.compile(r'^(?P<prefix>[A-Z]+)(?P<suffix>\d+)\s+(?P<value>\d+)\s*$')

getvalue  = attrgetter('value')

def interleave(seq, val):
    return itertools.chain.from_iterable(itertools.izip(seq, itertools.repeat(val)))

class Fileline(namedtuple('Fileline', 'filename prefix suffix value')):
    @classmethod
    def _fromstr(cls, s, filename=None, rematch=R_PREFIX_VALUE.match):
        m = rematch(s)
        if not m:
            raise ValueError('No valid line found in %r' % s)
        d = m.groupdict()
        d['value'] = int(d['value'])
        d['filename'] = filename
        return cls(**d)

    def _asstr(self):
        return '{}{} {}'.format(self.prefix, self.suffix, self.value)

def max_value_with_prefix(lineseq, prefix, getvalue=getvalue):
    withprefix = (line for line in lineseq if line.prefix==prefix)
    return max_value(withprefix)

def filter_lt_line(lineseq, maxline):
    for line in lineseq:
        if line.prefix != maxline.prefix or line.value >= maxline.value:
            yield line

def extreme_value(fn, lineseq, getvalue=getvalue):
    try:
        return fn((l for l in lineseq if l is not None), key=getvalue)
    except ValueError:
        return None

def max_value(lineseq):
    return extreme_value(max, lineseq)

def min_value(lineseq):
    return extreme_value(min, lineseq)

def read_lines(fn, maker=Fileline._fromstr):
    with open(fn, 'rb') as f:
        return deque(maker(l, fn) for l in f)

def write_file(fn, lineseq):
    lines = (l._asstr() for l in lineseq)
    newlines = interleave(lines, '\n')
    with open(fn, 'wb') as f:
        f.writelines(newlines)

def write_output_file(fn, lineseq):
    lines = ("{} {}".format(l.filename, l.value) for l in lineseq)
    newlines = interleave(lines, "\n")
    with open(fn, 'wb') as f:
        f.writelines(newlines)

def filter_max_returning_min(fn, prefix):
    lineseq = read_lines(fn)
    maxvalue = max_value_with_prefix(lineseq, prefix)
    filteredlineseq = deque(filter_lt_line(lineseq, maxvalue))
    write_file(fn, filteredlineseq)
    minline = min_value(filteredlineseq)
    return minline

def main(fileglob, prefix, outputfile):
    minlines = []
    for fn in glob.iglob(fileglob):
        minlines.append(filter_max_returning_min(fn, prefix))
    write_output_file(outputfile, minlines)

入口点是main()，类似main('txtdir', 'ENSG', 'output.txt')。对于每个文件filter_max_returning_min() 将打开并重写文件并返回最小值。无需保留您访问过的每个文件的每一行的字典或列表。

（顺便说一句，破坏性地覆盖文件似乎是个坏主意！您是否考虑过将它们复制到其他地方？）

当您将单独的关注点隔离到单独的函数中时，为不同的执行行为重新组合它们变得非常容易。例如，通过添加两个小函数在所有文件上并行运行此任务很简单：

def _worker(args):
    return filter_max_returning_min(*args)

def multi_main(fileglob, prefix, outputfile, processes):
    from multiprocessing import Pool
    pool = Pool(processes=processes)
    workerargs = ((fn, prefix) for fn in glob.iglob(fileglob))
    minlines = pool.imap_unordered(_worker, workerargs, processes)
    write_file(outputfile, minlines)

现在您可以启动可配置数量的工作人员，每个工作人员将处理一个文件，并在完成后收集他们的最小值。如果您有非常大的文件或大量文件并且不受 IO 限制，这可能会更快。

只是为了好玩，您还可以轻松地将其转换为 CLI 实用程序：

def _argparse():
    import argparse

    def positive_int(s):
        v = int(s)
        if v < 1:
            raise argparse.ArgumentTypeError('{:r} must be a positive integer'.format(s))
        return v

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""Filter text files and write min value.

    Performs these operations on the text files in supplied `filedir`:

    1. In each file, identify lines starting with the matching `maxprefix`
       which do *not* contain the maximum value for that prefix in that file.
    2. DESTRUCTIVELY REWRITE each file with lines found in step 1 removed!
    3. Write the minimum value (for all lines in all files) to `outputfile`.
    """)
    parser.add_argument('filedir',
        help="Directory containg the text files to process. WILL REWRITE FILES!")
    parser.add_argument('maxprefix', nargs="?", default="ENSG", 
        help="Line prefix which should have values less than max value removed in each file")
    parser.add_argument('outputfile', nargs="?", default="output.txt",
        help="File in which to write min value found. WILL REWRITE FILES!")
    parser.add_argument('-p', '--parallel', metavar="N", nargs="?", type=positive_int, const=10,
        help="Process files in parallel, with N workers. Default is to process a file at a time.")
    return parser.parse_args()

if __name__ == '__main__':
    args = _argparse()
    fileglob = os.path.join(args.filedir, '*.txt')
    prefix = args.maxprefix
    outputfile = args.outputfile
    if args.parallel:
        multi_main(fileglob, prefix, outputfile, args.parallel)
    else:
        main(fileglob, prefix, outputfile)

现在你可以从命令行调用它了：

$ python ENSG.py txtdir ENSCAFG --parallel=4