Skip to content

Instantly share code, notes, and snippets.

@chiensiTB
Created November 4, 2014 17:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save chiensiTB/0b3b20f2ba6f09962a30 to your computer and use it in GitHub Desktop.
Save chiensiTB/0b3b20f2ba6f09962a30 to your computer and use it in GitHub Desktop.
uploading worldweather data to Ruby on Rails, without permissions failure on Heroku
class CreatePlaces < ActiveRecord::Migration
# Rails migration for GeoWorldMap db into postgresql db
#(inspired by http://blog.inspired.no/populate-your-database-with-free-world-cities-countries-regions-in-2-minutes-using-a-rails-migration-273/ post)
#modified from the original gist, because of Heroku errors due to copy
#thanks to this post for helping me out:
#http://www.kadrmasconcepts.com/blog/2013/12/15/copy-millions-of-rows-to-postgresql-with-rails/
#extract files from GeoWorldMap.zip archive from here
# http://www.geobytes.com/GeoWorldMap.zip
#
#and place them into #{Rails.root}/db/migrate/
##the archive has 'cities.txt' file, rename it 'Cities.txt'
#mv cities.txt Cities.txt
def up
create_table :countries, :id => false do |t|
t.primary_key :CountryId
t.string :name, :limit => 50, :null => false
t.string :fips104, :limit => 2, :null => false
t.string :iso2, :limit => 2, :null => false
t.string :iso3, :limit => 3, :null => false
t.string :ison, :limit => 4, :null => false
t.string :internet, :limit => 2, :null => false
t.string :capital, :limit => 25
t.string :map_reference, :limit => 50
t.string :nationality_singular, :limit => 35
t.string :nationality_plural, :limit => 35
t.string :currency, :limit => 30
t.string :currency_code, :limit => 3
t.integer :population
t.string :title, :limit => 50
t.string :comment, :limit => 255
end
create_table :regions, :id => false do |t|
t.primary_key :RegionId
t.references :country, :null => false
t.string :name, :limit => 45, :null => false
t.string :code, :limit => 8, :null => false
t.string :adm1code, :limit => 4, :null => false
end
create_table :cities, :id => false do |t|
t.primary_key :CityId
t.references :country, :null => false
t.references :region, :null => false
t.string :name, :limit => 45, :null => false
t.float :latitude, :null => false
t.float :longitude, :null => false
t.string :timezone, :limit => 10, :null => false
t.integer :dma_id
t.string :code, :limit => 4
end
add_index :countries, :name
add_index :regions, :name
add_index :cities, :name
#change encoding to the one used into files
execute "set client_encoding to 'latin1'"
conn = ActiveRecord::Base.connection
rc = conn.raw_connection
rc.exec("COPY countries FROM STDIN WITH CSV")
filepath = File.join(File.dirname(__FILE__),'Countries.txt')
file = File.open(filepath, 'r')
while !file.eof?
rc.put_copy_data(file.readline)
end
rc.put_copy_end
while res = rc.get_result
if e_message = res.error_message
p e_message
end
end
conn2 = ActiveRecord::Base.connection
rc2 = conn2.raw_connection
rc2.exec("COPY regions FROM STDIN WITH CSV")
filepath = File.join(File.dirname(__FILE__),'Regions.txt')
file2 = File.open(filepath, 'r')
while !file2.eof?
rc2.put_copy_data(file2.readline)
end
rc2.put_copy_end
while res = rc2.get_result
if e_message = res.error_message
p e_message
end
end
conn3 = ActiveRecord::Base.connection
rc3 = conn3.raw_connection
rc3.exec("COPY cities FROM STDIN WITH CSV")
filepath = File.join(File.dirname(__FILE__),'cities.txt')
file3 = File.open(filepath, 'r')
while !file3.eof?
rc3.put_copy_data(file3.readline)
end
rc3.put_copy_end
while res = rc3.get_result
if e_message = res.error_message
p e_message
end
end
#change encoding back to UTF8
execute "set client_encoding to 'UTF8'"
rename_column :countries, 'CountryId', :id
rename_column :regions, 'RegionId', :id
rename_column :cities, 'CityId', :id
#to generate foreign keys add
#gem 'foreigner' into Gemfile
add_foreign_key(:cities, :countries, :dependent => :delete)
add_foreign_key(:cities, :regions, :dependent => :delete)
add_foreign_key(:regions, :countries, :dependent => :delete)
end
def down
drop_table :cities
drop_table :regions
drop_table :countries
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment