Preparing Experimental Data

mvunlockedtxt.py

import os

from os import listdir
from os.path import isfile, join


def writeline(f_name,line):
    f = open(f_name,'a')
    line = line.replace("$", "")
    f.write(line)
    f.close()import os

from os import listdir
from os.path import isfile, join

txt_lim = 45000000 # MB
counter = 0
contentseris = 0


def writeline(f_name,line):
    global counter
    f = open(f_name,'a')
    #line = line.replace("$", "")
    counter += len(line)
    f.write(line)
    f.close()

def readNappend(src_name,f_name):
    global counter, contentseris
    try:
        with open(src_name, "r") as f:
            content = f.readlines()
        f.close()
        
        for line in content:
            if counter >= txt_lim:
                counter = 0
                contentseris +=1
            f = f_name+str(contentseris)+".txt"
            writeline(f,line)
    except:
        print("we can not open ----"+src_name+" or write to "+f_name)
    
mypath = "./Cs"
target = "./Codes/"
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] # one level files 
for root, _, files in os.walk(mypath): # or recursively open files
    for f in files:
        filename = root+"/"+f
        print(filename)
        readNappend(filename,target+"Codes")


def readNappend(src_name,f_name):
    with open(src_name, "r") as f:
        content = f.readlines()
    f.close()
    for line in content:
        writeline(f_name,line)

mypath = "./src"
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
for file in onlyfiles:
    filename = mypath+"/"+file
    readNappend(filename,"books.txt")