Skip to content

Instantly share code, notes, and snippets.

@Ciantic
Created September 14, 2011 16:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Ciantic/1216994 to your computer and use it in GitHub Desktop.
Save Ciantic/1216994 to your computer and use it in GitHub Desktop.
Convert hygxyz.csv to hyg.sqlite
# Public domain
# Created by Jari Pennanen
# September 2011
#
# For viewing SQLite, try one of these
# - http://sqliteadmin.orbmu2k.de/ (only win32, better than sqlitebrowser)
# - http://sqlitebrowser.sourceforge.net/
import sqlite3
import os
import csv
# Create HYG table if not existing
def create_hyg(conn):
cur = conn.cursor()
# Create table if not existing
cur.execute('''
CREATE TABLE IF NOT EXISTS HYG(
StarID INTEGER PRIMARY KEY ASC,
HIP INTEGER,
HD INTEGER,
HR INTEGER,
Gliese INTEGER,
BayerFlamsteed TEXT,
ProperName TEXT,
RA REAL,
Dec REAL,
Distance REAL,
PMRA REAL,
PMDec REAL,
RV REAL,
Mag REAL,
AbsMag REAL,
Spectrum TEXT,
ColorIndex REAL,
X REAL,
Y REAL,
Z REAL,
VX REAL,
VY REAL,
VZ REAL
);
''')
conn.commit()
# Insert single row to table (does not commit!)
def insert_row(conn, headers, row, dry_run=False):
cur = conn.cursor()
sql = """INSERT OR IGNORE INTO HYG (%(columns)s) VALUES (%(prep)s)""" % dict(
columns=",".join(headers),
prep=",".join(['?' for t in headers])
)
if dry_run:
print sql
return
cur.execute(sql, row)
# Import HYG CSV data
def import_hygcsv(conn, csvfile, verbose=False, dry_run=False):
file = open(csvfile, 'rb')
rdr = csv.reader(file)
# Get headers row
headers = rdr.next()
# Debug / preview verbose
if verbose:
print "Headers:"
print headers
for n, row in enumerate(rdr):
insert_row(conn, headers, row, dry_run=dry_run)
# Commit on each 5000th row
if n % 5000 == 0:
print ".", # Note "," ensures that no newline is printed
conn.commit()
print "."
conn.commit()
# Main command line program
if __name__ == '__main__':
conn = sqlite3.connect('hyg2.sqlite');
create_hyg(conn)
import_hygcsv(conn, 'hygxyz.csv', verbose=False, dry_run=False)
@sigma001
Copy link

i got that error and i'm unfortunately new to sqlite:

File "hygcsv_to_sqlite.py", line 90, in
import_hygcsv(conn, 'hygdata_v3.csv', verbose=False, dry_run=False)
File "hygcsv_to_sqlite.py", line 76, in import_hygcsv
insert_row(conn, headers, row, dry_run=dry_run)
File "hygcsv_to_sqlite.py", line 60, in insert_row
cur.execute(sql, row)
sqlite3.OperationalError: table HYG has no column named id

@sigma001
Copy link

now its running, wrong hygcvs (smaller version)

@sigma001
Copy link

here is the script for the newer hygdata_v3.csv
https://gist.github.com/sigma001/731942da9f494d405cd51b104f6fb2db

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment