Skip to content

Instantly share code, notes, and snippets.

@bbarrows
Created February 9, 2015 21:50
Show Gist options
  • Save bbarrows/9695784082d248ba8a4e to your computer and use it in GitHub Desktop.
Save bbarrows/9695784082d248ba8a4e to your computer and use it in GitHub Desktop.
My importer
#!/usr/bin/env python
from os import listdir
from os.path import isfile, join
from subprocess import Popen, PIPE, STDOUT, call
import MySQLdb
import MySQLdb.cursors
import types
import warnings
DB_NAME = "SGTEST"
SQL_PATH = "/tmp"
CREATE_NEW_DB = True
tmp_files = [ f for f in listdir(SQL_PATH) if isfile(join(SQL_PATH,f)) ]
sql_files = [x for x in tmp_files if x.endswith(".sql")]
txt_files = [x for x in tmp_files if x.endswith(".txt") and x[0:x.find('.txt')]+".sql" in sql_files]
if CREATE_NEW_DB:
try:
con = MySQLdb.connect('localhost', 'root', 'password')
with con:
cur = con.cursor()
cur.execute("CREATE DATABASE %s" % DB_NAME)
except Exception:
con = MySQLdb.connect('localhost', 'root', 'password')
with con:
cur = con.cursor()
cur.execute("DROP DATABASE %s" % DB_NAME)
cur.execute("CREATE DATABASE %s" % DB_NAME)
con = MySQLdb.connect('localhost', 'root', 'password', DB_NAME)
file_order = []
do_first = ["PermitApp.sql", "PermitRole.sql", "Avl.sql", "Country.sql", "Customer.sql", "Fleet.sql", "Account.sql", "User.sql", "UserCustomer.sql", "UserFleet.sql", "VehicleType.sql", "Vehicle.sql"]
#for f in do_first:
#call("mysql -u root -ppassword %s < %s" % (DB_NAME, f), shell=True)
for f in do_first:
table_name = f[0:f.find('.sql')]
csv_file_name = table_name + ".txt"
file_order.append((table_name, csv_file_name))
for sql_file in sql_files:
if sql_file not in do_first:
table_name = sql_file[0:sql_file.find('.sql')]
csv_file_name = table_name + ".txt"
file_order.append((table_name, csv_file_name))
for table_name, data_file in file_order:
print "mysql -u root -ppassword %s < %s" % (DB_NAME, table_name+".sql")
call("mysql -u root -ppassword %s < %s" % (DB_NAME, table_name+".sql"), shell=True)
for table_name, data_file in file_order:
print "\nImporting file %s" % data_file
x = "LOAD DATA INFILE \"%s\" INTO TABLE %s COLUMNS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"'ESCAPED BY '\"' LINES TERMINATED BY '\\n';" % (join("/tmp", data_file), table_name)
print x
with con:
cur = con.cursor()
result = cur.execute(x)
print "RESULT:%s" % result
# Check out
# http://dev.mysql.com/doc/refman/5.1/en/load-data.html
# http://stackoverflow.com/questions/13648055/mysql-doesnt-csv-import-n-as-null
# http://dev.mysql.com/doc/refman/5.0/en/mysqlimport.html
# mysqlimport --fields-optionally-enclosed-by=""" --fields-terminated-by=, --lines-terminated-by="\r\n" --user=YOUR_USERNAME --password YOUR_DATABASE YOUR_TABLE.csv
# mysqlimport --fields-optionally-enclosed-by=""" --fields-terminated-by=, --lines-terminated-by="\n" --user=root --password password SGTEST *.csv
# Then I manually run:
# mysql -u root -ppassword SGTEST < Rule.out
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment