dfs_file_scan.py

import os
import time
import _pickle as pickle

master_list = []

class master_list_item:
    
    TYPE_DIR = b'1'
    TYPE_FILE = b'2'
    
    def __init__(self, type, name):
        self.type = type
        self.name = name
        
    def __str__(self):
        if self.type == self.TYPE_DIR:
            return "{} - {}".format("DIR", self.name)
        else:
            return "{} - {}".format("FILE", self.name)

def scan_files(dir_name, max_depth, current_depth=0):
    try:
        if (current_depth<max_depth):
            if (os.path.isdir(dir_name)):
                for i in os.listdir(dir_name):
                    # print("append file node {}".format(i))
                    if (os.path.isdir(os.path.join(dir_name,i))):
                        master_list.append(master_list_item(master_list_item.TYPE_DIR,i))
                        # print("going deeper into "+i)
                        scan_files(os.path.join(dir_name,i),max_depth,current_depth+1)
                    else:
                        master_list.append(master_list_item(master_list_item.TYPE_FILE,i))
            else:
                master_list.append(master_list_item(master_list_item.TYPE_FILE,dir_name))
                # print("append file node {}".format(str(dir_name)))
        elif (current_depth>=max_depth):
            if os.path.isdir(dir_name):
                master_list.append(master_list_item(master_list_item.TYPE_DIR,dir_name))
            else:
                master_list.append(master_list_item(master_list_item.TYPE_FILE,dir_name))
            # print("appending floor node {}".format(dir_name))
    except PermissionError:
        pass

elapsed_sum = 0

iterations = 10

for j in range(0,iterations):
    master_list = []
    start = time.time()
    scan_files(os.path.expanduser("~"),5)
    elapsed = time.time()-start
    print(f"scan iteration {str(j+1)} complete in "+str(elapsed))
    elapsed_sum += elapsed

# for i in master_list:
#     print(i)
    
print(f"average time = {str(elapsed_sum/iterations)}")

with open('master_list.db','wb') as f:
    pickle.dump(master_list,f)