Skip to content

Instantly share code, notes, and snippets.

@sweyrick
Forked from scottynomad/getgeo.sh
Created March 13, 2013 00:40
Show Gist options
  • Save sweyrick/5148442 to your computer and use it in GitHub Desktop.
Save sweyrick/5148442 to your computer and use it in GitHub Desktop.
#!/bin/bash
#===============================================================================
#
# FILE: getgeo.sh
#
# USAGE: ./getgeo.sh
#
# DESCRIPTION: run the script so that the geodata will be downloaded and inserted into your
# database
#
# OPTIONS: ---
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: ---
# AUTHOR: Andreas (aka Harpagophyt )
# COMPANY: <a href="http://forum.geonames.org/gforum/posts/list/926.page" target="_blank" rel="nofollow">http://forum.geonames.org/gforum/posts/list/926.page</a>
# VERSION: 1.3
# CREATED: 07/06/2008
# REVISION: 1.1 2008-06-07 replace COPY continentCodes through INSERT statements.
# 1.2 2008-11-25 Adjusted by Bastiaan Wakkie in order to not unnessisarily
# download.
# 1.3 2011-08-07 Updated script with tree changes. Removes 2 obsolete records from "countryinfo" dump image,
# updated timeZones table with raw_offset and updated postalcode to varchar(20).
# 1.4 2012-10-01 (Scott Wilson) Added is_historical and is_colloquial to alternate names table, and country
# code to time zones. Add column constraints after loading data, runs super fast on my machine.
# Normalized column naming, various other tweaks.
#===============================================================================
#!/bin/bash
WORKPATH="/tmp/geonames.work"
TMPPATH="tmp"
PCPATH="pc"
PREFIX="_"
DBHOST="127.0.0.1"
DBPORT="5432"
DBUSER="postgres"
FILES="allCountries.zip alternateNames.zip userTags.zip admin1CodesASCII.txt admin2Codes.txt countryInfo.txt featureCodes_en.txt iso-languagecodes.txt timeZones.txt"
psql -U $DBUSER -h $DBHOST -p $DBPORT -c "CREATE DATABASE geonames WITH TEMPLATE = template0 ENCODING = 'UTF8';"
psql -U $DBUSER -h $DBHOST -p $DBPORT geonames <<EOT
DROP TABLE IF EXISTS geoname CASCADE;
CREATE TABLE geoname (
id INT,
name VARCHAR,
ascii_name VARCHAR,
alternate_names VARCHAR,
latitude FLOAT,
longitude FLOAT,
fclass CHAR(1),
fcode CHAR(10),
country CHAR(2),
cc2 VARCHAR,
admin1 VARCHAR,
admin2 VARCHAR,
admin3 VARCHAR,
admin4 VARCHAR,
population BIGINT,
elevation INT,
gtopo30 INT,
timezone VARCHAR,
modified_date DATE
);
DROP TABLE IF EXISTS alternatename;
CREATE TABLE alternatename (
id INT,
geoname_id INT,
iso_lang VARCHAR,
alternate_name VARCHAR,
is_preferred_name BOOLEAN,
is_short_name BOOLEAN,
is_colloquial BOOLEAN,
is_historic BOOLEAN
);
DROP TABLE IF EXISTS countryinfo;
CREATE TABLE "countryinfo" (
iso_alpha2 CHAR(2),
iso_alpha3 CHAR(3),
iso_numeric INTEGER,
fips_code VARCHAR(3),
country VARCHAR,
capital VARCHAR,
area DOUBLE PRECISION, -- square km
population INTEGER,
continent CHAR(2),
tld VARCHAR,
currency_code CHAR(3),
currency_name VARCHAR,
phone VARCHAR,
postal VARCHAR,
postal_regex VARCHAR,
languages VARCHAR,
geoname_id INT,
neighbours VARCHAR,
equivalent_fips_code VARCHAR(3)
);
DROP TABLE IF EXISTS iso_languagecodes;
CREATE TABLE iso_languagecodes(
iso_639_3 CHAR(4),
iso_639_2 VARCHAR(50),
iso_639_1 VARCHAR(50),
language_name VARCHAR
);
DROP TABLE IF EXISTS admin1CodesAscii;
CREATE TABLE admin1CodesAscii (
code CHAR(20),
name TEXT,
name_ascii TEXT,
geoname_id INT
);
DROP TABLE IF EXISTS admin2CodesAscii;
CREATE TABLE admin2CodesAscii (
code CHAR(80),
name TEXT,
name_ascii TEXT,
geoname_id INT
);
DROP TABLE IF EXISTS featureCodes;
CREATE TABLE featureCodes (
code CHAR(7),
name VARCHAR,
description TEXT
);
DROP TABLE IF EXISTS timeZones;
CREATE TABLE timeZones (
id VARCHAR,
country_code VARCHAR(2),
GMT_offset NUMERIC(3,1),
DST_offset NUMERIC(3,1),
raw_offset NUMERIC(3,1)
);
DROP TABLE IF EXISTS continentCodes;
CREATE TABLE continentCodes (
code CHAR(2),
name VARCHAR,
geoname_id INT
);
DROP TABLE IF EXISTS postalcodes;
CREATE TABLE postalcodes (
country_code CHAR(2),
postal_code VARCHAR,
place_name VARCHAR,
admin1_name VARCHAR,
admin1_code VARCHAR,
admin2_name VARCHAR,
admin2_code VARCHAR,
admin3_name VARCHAR,
admin3_code VARCHAR,
latitude FLOAT,
longitude FLOAT,
accuracy SMALLINT
);
EOT
# check if needed directories do already exsist
if [ -d "$WORKPATH" ]; then
echo "$WORKPATH exists..."
sleep 0
else
echo "$WORKPATH and subdirectories will be created..."
mkdir -p $WORKPATH/{$TMPPATH,$PCPATH}
echo "created $WORKPATH"
fi
echo
echo ",---- STARTING (downloading, unpacking and preparing)"
cd $WORKPATH/$TMPPATH
for i in $FILES
do
wget -N -q "http://download.geonames.org/export/dump/$i" # get newer files
if [ $i -nt $PREFIX$i ] || [ ! -e $PREFIX$i ] ; then
cp -p $i $PREFIX$i
unzip -u -q $i
case "$i" in
iso-languagecodes.txt)
tail -n +2 iso-languagecodes.txt > iso-languagecodes.txt.tmp;
;;
countryInfo.txt)
grep -v '^#' countryInfo.txt > countryInfo.txt.tmp;
;;
timeZones.txt)
tail -n +2 timeZones.txt > timeZones.txt.tmp;
;;
esac
echo "| $1 has been downloaded";
else
echo "| $i is already the latest version"
fi
done
# download the postalcodes. You must know yourself the url
cd $WORKPATH/$PCPATH
wget -q -N "http://download.geonames.org/export/zip/allCountries.zip"
if [ $WORKPATH/$PCPATH/allCountries.zip -nt $WORKPATH/$PCPATH/allCountries$PREFIX.zip ] || [ ! -e $WORKPATH/$PCPATH/allCountries.zip ]; then
echo "Attempt to unzip $WORKPATH/$PCPATH/allCountries.zip file..."
unzip -u -q $WORKPATH/$PCPATH/allCountries.zip
cp -p $WORKPATH/$PCPATH/allCountries.zip $WORKPATH/$PCPATH/allCountries$PREFIX.zip
echo "| ....zip has been downloaded"
else
echo "| ....zip is already the latest version"
fi
echo "+---- FILL DATABASE ------+"
psql -e -U $DBUSER -h $DBHOST -p $DBPORT geonames <<EOT
copy geoname (id,name,ascii_name,alternate_names,latitude,
longitude,fclass,fcode,country,cc2,admin1,admin2,
admin3,admin4,population,elevation,gtopo30,
timezone,modified_date)
from '${WORKPATH}/${TMPPATH}/allCountries.txt' null as '';
copy postalcodes (country_code,postal_code,place_name,
admin1_name,admin1_code,admin2_name,admin2_code,
admin3_name,admin3_code,latitude,longitude,accuracy)
from '${WORKPATH}/${PCPATH}/allCountries.txt' null as '';
copy timeZones (country_code,id,GMT_offset,DST_offset,raw_offset)
from '${WORKPATH}/${TMPPATH}/timeZones.txt.tmp' null as '';
copy featureCodes (code,name,description)
from '${WORKPATH}/${TMPPATH}/featureCodes_en.txt' null as '';
copy admin1CodesAscii (code,name,name_ascii,geoname_id)
from '${WORKPATH}/${TMPPATH}/admin1CodesASCII.txt' null as '';
copy admin2CodesAscii (code,name,name_ascii,geoname_id)
from '${WORKPATH}/${TMPPATH}/admin2Codes.txt' null as '';
copy iso_languagecodes (iso_639_3,iso_639_2,iso_639_1,language_name)
from '${WORKPATH}/${TMPPATH}/iso-languagecodes.txt.tmp' null as '';
copy countryInfo (iso_alpha2,iso_alpha3,iso_numeric,fips_code,country,
capital,area,population,continent,tld,currency_code,
currency_name,phone,postal,postal_regex,languages,
geoname_id,neighbours,equivalent_fips_code)
from '${WORKPATH}/${TMPPATH}/countryInfo.txt.tmp' null as '';
copy alternatename (id,geoname_id,iso_lang,alternate_name,
is_preferred_name,is_short_name,
is_colloquial,is_historic)
from '${WORKPATH}/${TMPPATH}/alternateNames.txt' null as '';
INSERT INTO continentCodes VALUES ('AF', 'Africa', 6255146);
INSERT INTO continentCodes VALUES ('AS', 'Asia', 6255147);
INSERT INTO continentCodes VALUES ('EU', 'Europe', 6255148);
INSERT INTO continentCodes VALUES ('NA', 'North America', 6255149);
INSERT INTO continentCodes VALUES ('OC', 'Oceania', 6255150);
INSERT INTO continentCodes VALUES ('SA', 'South America', 6255151);
INSERT INTO continentCodes VALUES ('AN', 'Antarctica', 6255152);
CREATE INDEX index_countryinfo_geonameid ON countryinfo (geoname_id);
CREATE INDEX index_alternatename_geonameid ON alternatename (geoname_id);
EOT
psql -U $DBUSER -h $DBHOST -p $DBPORT geonames <<EOT
ALTER TABLE ONLY alternatename
ADD CONSTRAINT pk_alternatenameid PRIMARY KEY (id);
ALTER TABLE ONLY geoname
ADD CONSTRAINT pk_geonameid PRIMARY KEY (id);
ALTER TABLE ONLY countryinfo
ADD CONSTRAINT pk_iso_alpha2 PRIMARY KEY (iso_alpha2);
ALTER TABLE ONLY countryinfo
ADD CONSTRAINT fk_geonameid FOREIGN KEY (geoname_id)
REFERENCES geoname(id);
ALTER TABLE ONLY alternatename
ADD CONSTRAINT fk_geonameid FOREIGN KEY (geoname_id)
REFERENCES geoname(id);
EOT
echo "'----- DONE ( have fun... )"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment