在Python中删除基于子字符串的重复项

Question

我有以下代码用于检测文件中的重复项并将它们输出到3个单独的文件中，一个用于非重复文件，一个用于重复项（x2），一个用于重复项（> x2）。第一个文件仅保留原始文件中没有重复的行。 （它不会删除找到的任何重复行，它保持单身）。

import os
import sys
import time
import collections


file_in = sys.argv[1]
file_ot = str(file_in) + ".proc"
file_ot2 = str(file_in) + ".proc2"
file_ot3 = str(file_in) + ".proc3"


counter = 0        

dict_in = collections.defaultdict(list)  
with open(file_in, "r") as f:  
    for line in f:  
        #print("read line: " + str(line))
        counter += 1
        fixed_line = line.strip()
        line_list = fixed_line.split(";")
        key = line_list[0][:12]
        print(":Key: " + str(key))
        dict_in[key].append(line)


with open(file_ot, "w") as f1, open(file_ot2, "w") as f2, open(file_ot3, "w") as f3:
    selector = {1: f1, 2: f2}  
    for values in dict_in.values():  
        if len(values) == 1:
            f1.writelines(values)
        elif len(values) == 2:
            f2.writelines(values)
        else:
            f3.writelines(values)



print("Read: " + str(counter) + " lines")

上面的代码可以工作，但是对于v大文件（~1g），在我的系统上通过它们大约需要十分钟。我想知道是否有办法优化此代码的速度，或者在该方向上的任何建议。先感谢您！

输入数据示例：

0000AAAAAAAA;X;;X;
0000AAAAAAAA;X;X;;
0000BBBBBBBB;X;;;
0000CCCCCCCC;;X;;
0000DDDDDDDD;X;;X;
0000DDDDDDDD;X;X;;
0000DDDDDDDD;X;X;X;X
0000EEEEEEEE;X;X;X;X
0000FFFFFFFF;X;;;
0000GGGGGGGG;X;;X;
0000HHHHHHHH;X;X;;
0000JJJJJJJJ;X;X;;

预期产量：

FILE1:
0000BBBBBBBB;X;;;
0000CCCCCCCC;;X;;
0000EEEEEEEE;X;X;X;X
0000FFFFFFFF;X;;;
0000GGGGGGGG;X;;X;
0000HHHHHHHH;X;X;;
0000JJJJJJJJ;X;X;;

FILE2:
0000AAAAAAAA;X;;X;
0000AAAAAAAA;X;X;;

FILE3:
0000DDDDDDDD;X;;X;
0000DDDDDDDD;X;X;;
0000DDDDDDDD;X;X;X;X

Answer 1

我用了543MB的随机文本文件来测试它。

import time

myList = []

start = time.time()
with open("myFile.txt") as f:
    for line in f:
        line = line.replace("\n","")
        myList.insert(len(myList), line)

with open("dupListaOne.txt", "w") as f1, open ("dupListMore.txt","w") as f2, open("UniqueList.txt","w") as f3:
    new_list = sorted(set(myList))
    for i in range(len(new_list)):
            a = myList.count(new_list[i])
            if ((a-1) == 1):
                f1.write("%s\n" % new_list[i] + " " + str(a-1))
            elif ((a-1) > 1):
                f2.write("%s\n" % new_list[i] + " " + str(a-1))
            else:
                f3.write("%s\n" % new_list[i] + " " + str(a-1))
end = time.time()
print("Time: ",end - start)

f1.close()
f2.close()
f3.close()

经过时间：123.82529425621033秒~2分钟