Skip to content

Instantly share code, notes, and snippets.

@arthurbarros
Created October 22, 2015 19:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save arthurbarros/bf6994154cead455456c to your computer and use it in GitHub Desktop.
Save arthurbarros/bf6994154cead455456c to your computer and use it in GitHub Desktop.
Find almost duplicated roms
import glob,os,pdb,time
class File_:
def __init__(self,name_,codes_,codeLength_,shortName_):
self.name=name_
self.codes=codes_
self.codeLength=codeLength_
self.shortName=shortName_
metaFiles=[]
files=[]
delete=[]
noDelete=[]
tmpList=[]
accept=0
while True:
metaFiles[:]=[]
files[:]=[]
delete[:]=[]
noDelete[:]=[]
tmpList[:]=[]
print("Searcing for files...")
files=glob.glob("*")
print("Found "+str(len(files))+" files!")
files.sort()
chars=[]
tmpCodes=0
tmpCodeLength=0
tmpShortName=''
shortNameDone=0
num=0
curFile=1
tmp=''
print("Gathering information...")
for x in files:
print("Analyzing file "+str(curFile)+"/"+str(len(files)))
#pdb.set_trace()
fullName=str(x)
chars=list(fullName)
while num<=len(chars)-1:
#print(chars[num])
if chars[num] =="(" or chars[num]=="[":
tmpCodes+=1
shortNameDone=1
while chars[num]!=")" and chars[num]!="]":
num+=1
tmpCodeLength+=1
else:
if not shortNameDone:
tmpShortName=tmpShortName+chars[num]
num+=1
metaFiles.append(File_(fullName,tmpCodes,tmpCodeLength,tmpShortName))
tmpCodes=0
tmpCodeLength=0
tmpShortName=''
shortNameDone=0
num=0
tmp=''
curFile+=1
print("Searching for duplicates...")
num=0
checking=0
while num<len(metaFiles)-1:
if not checking:
tmp=metaFiles[num].shortName
path=metaFiles[num].name
if num+1!=len(metaFiles)-1:
if metaFiles[num+1].shortName==tmp:
noDelete.append(path)
delete.append(metaFiles[num+1].name)
else:
checking=1
if metaFiles[num+1].name!=tmp:
checking=0
num+=1
if not accept:
print("About to delete "+str(len(delete))+" files. Hit enter to continue...")
input()
accept=1
else:
print("Deleting "+str(len(delete))+" files...")
if len(delete)==0:
break
for x in delete:
try:
if x not in noDelete:
os.remove(str(x))
except:
print("Problem removing "+str(x))
print("Success!")
input()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment