Divides files into chunks of 1000 VALUES for INSERT INTO and then divide it into chunks of 100k lines

join_sql.py

import glob
from math import floor

def rreplace(s, old, new, count):
  return (s[::-1].replace(old[::-1], new[::-1], count))[::-1]

i = 0
j=0
files = glob.glob("./*.sql")

created = False
creating = False
create_index = False
insert_statment = ""
dump_filename=""

file_in=None
file_out=None

for ffile in files:
  file=ffile.replace('.sql', '')
  ii = floor(i/100000)
  dump_filename=f"dumps/{file}_{ii}.sql"
  
  try:
    if not file_in.closed:
      file_in.close()
  except:
    print("")
    
  try:
    if not file_out.closed:
      file_out.close()
  except:
    print("")
    
  file_in=open(ffile, 'r')
  file_out=open(dump_filename, 'w')

  sb = ""
      
  print(f"Starting with {ffile}")

  for line in file_in:
    if (not created and line.startswith('CREATE TABLE')) or creating:
      creating = True
      file_out.write(line)
      if line.strip().startswith('SELECT'):
        created = True
        creating = False
      else:
        print(f"Creating: {creating} | Created: {created}")
        continue

    if line.startswith('CREATE INDEX') and not create_index:
      print(f"Create index: {create_index}")
      
      create_index = True
      file_out.write(line)
      continue

    if not line.startswith("INSERT INTO"):
      continue
    elif len(insert_statment) == 0:
      insert_statment = line.split(' VALUES ')[0] + ' VALUES '

    if i % 1000 == 0:
      print(f"{i} done!")
      if i > 0:
        file_out.write(sb)
      sb = insert_statment
      j=0
      
    try:
      l = line.split(' VALUES ')[1]
      
      if (i + 1) % 1000 != 0:
        l = l.replace(';', ',')
        
      sb += l
      i += 1
      j += 1
    except Exception as ex:
      print(ex)
      print(f"[{i}] at line: {line}")
    
    iii = floor(i/100000)
    ddump_filename=f"dumps/{file}_{iii}.sql"
    
    if ddump_filename != dump_filename:
      dump_filename = ddump_filename
      file_out.close()
      file_out=open(dump_filename, 'w')

  if j > 0:
    file_out.write(rreplace(sb, ',', ';', 1))
    j = 0
  i=0

  created=False
  creating=False
  create_index=False
  insert_statment=""

file_in.close()
file_out.close()