Created
September 14, 2011 16:18
-
-
Save Ciantic/1216994 to your computer and use it in GitHub Desktop.
Convert hygxyz.csv to hyg.sqlite
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Public domain | |
# Created by Jari Pennanen | |
# September 2011 | |
# | |
# For viewing SQLite, try one of these | |
# - http://sqliteadmin.orbmu2k.de/ (only win32, better than sqlitebrowser) | |
# - http://sqlitebrowser.sourceforge.net/ | |
import sqlite3 | |
import os | |
import csv | |
# Create HYG table if not existing | |
def create_hyg(conn): | |
cur = conn.cursor() | |
# Create table if not existing | |
cur.execute(''' | |
CREATE TABLE IF NOT EXISTS HYG( | |
StarID INTEGER PRIMARY KEY ASC, | |
HIP INTEGER, | |
HD INTEGER, | |
HR INTEGER, | |
Gliese INTEGER, | |
BayerFlamsteed TEXT, | |
ProperName TEXT, | |
RA REAL, | |
Dec REAL, | |
Distance REAL, | |
PMRA REAL, | |
PMDec REAL, | |
RV REAL, | |
Mag REAL, | |
AbsMag REAL, | |
Spectrum TEXT, | |
ColorIndex REAL, | |
X REAL, | |
Y REAL, | |
Z REAL, | |
VX REAL, | |
VY REAL, | |
VZ REAL | |
); | |
''') | |
conn.commit() | |
# Insert single row to table (does not commit!) | |
def insert_row(conn, headers, row, dry_run=False): | |
cur = conn.cursor() | |
sql = """INSERT OR IGNORE INTO HYG (%(columns)s) VALUES (%(prep)s)""" % dict( | |
columns=",".join(headers), | |
prep=",".join(['?' for t in headers]) | |
) | |
if dry_run: | |
print sql | |
return | |
cur.execute(sql, row) | |
# Import HYG CSV data | |
def import_hygcsv(conn, csvfile, verbose=False, dry_run=False): | |
file = open(csvfile, 'rb') | |
rdr = csv.reader(file) | |
# Get headers row | |
headers = rdr.next() | |
# Debug / preview verbose | |
if verbose: | |
print "Headers:" | |
print headers | |
for n, row in enumerate(rdr): | |
insert_row(conn, headers, row, dry_run=dry_run) | |
# Commit on each 5000th row | |
if n % 5000 == 0: | |
print ".", # Note "," ensures that no newline is printed | |
conn.commit() | |
print "." | |
conn.commit() | |
# Main command line program | |
if __name__ == '__main__': | |
conn = sqlite3.connect('hyg2.sqlite'); | |
create_hyg(conn) | |
import_hygcsv(conn, 'hygxyz.csv', verbose=False, dry_run=False) |
now its running, wrong hygcvs (smaller version)
here is the script for the newer hygdata_v3.csv
https://gist.github.com/sigma001/731942da9f494d405cd51b104f6fb2db
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
i got that error and i'm unfortunately new to sqlite:
File "hygcsv_to_sqlite.py", line 90, in
import_hygcsv(conn, 'hygdata_v3.csv', verbose=False, dry_run=False)
File "hygcsv_to_sqlite.py", line 76, in import_hygcsv
insert_row(conn, headers, row, dry_run=dry_run)
File "hygcsv_to_sqlite.py", line 60, in insert_row
cur.execute(sql, row)
sqlite3.OperationalError: table HYG has no column named id