change_data.rb

require 'ostruct'
require 'csv'

require 'rubygems'
require 'kdtree'

class GeoName < OpenStruct
end

class GeoNames
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        g = GeoName.new
        s = line.chomp.split("\t")
        g.geonameid = s[0]
        g.name = s[1]
        g.asciiname = s[2]
        g.alternatenames = s[3]
        g.latitude = s[4]
        g.longitude = s[5]
        g.feature_class = s[6]
        g.feature_code = s[7]
        g.country_code = s[8]
        g.cc2 = s[9]
        g.admin1 = s[10]
        g.admin2 = s[11]
        g.admin3 = s[12]
        g.admin4 = s[13]
        g.population = s[14]
        g.elevation = s[15]
        g.gtopo30 = s[16]
        g.timezone = s[17]
#        g.modification_date = Time.parse(s[18])

        yield g
      end
    end
  end
end


class Timezones
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        t = GeoName.new
        s = line.chomp.split("\t")
        unless s[0].to_s == "CountryCode"
          t.countrycode = s[0]
          t.timezoneid  = s[1]
          t.gmt_offset  = s[2]
          t.dst_offset  = s[3]
          t.raw_offset  = s[4]
        end

        yield t
      end
    end
  end
end

citiesfile   = "data/allCountries.txt"
kdtreefile   = "data/allCountries.kdtree"
idtzfile     = "data/id_tz_all.csv"

####################
# geonameidを取得するために、Kdtree.new(f)用なファイルを生成する
###################

points = []
GeoNames.parse(citiesfile) do |place|
  points << [place.latitude.to_f, place.longitude.to_f, place.geonameid.to_i]
end
kd = Kdtree.new(points)
File.open(kdtreefile, "w") {|f| kd.persist(f)}

###################################
# geonameidより、タイムゾーンを取得するために、geonameid(int), timezoneid
###################################

CSV.open(idtzfile, "wb") do |csv|
  GeoNames.parse(citiesfile) do |place|
    csv << [place.geonameid, place.timezone]
  end
end

compaire_from_countries15000.rb

require 'ostruct'
require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'


require 'rubygems'
require 'kdtree'
require 'sqlite3'

class GeoName < OpenStruct
end

class GeoNames
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        g = GeoName.new
        s = line.chomp.split("\t")
        g.geonameid = s[0]
        g.name = s[1]
        g.asciiname = s[2]
        g.alternatenames = s[3]
        g.latitude = s[4]
        g.longitude = s[5]
        g.feature_class = s[6]
        g.feature_code = s[7]
        g.country_code = s[8]
        g.cc2 = s[9]
        g.admin1 = s[10]
        g.admin2 = s[11]
        g.admin3 = s[12]
        g.admin4 = s[13]
        g.population = s[14]
        g.elevation = s[15]
        g.gtopo30 = s[16]
        g.timezone = s[17]
#        g.modification_date = Time.parse(s[18])

        yield g
      end
    end
  end
end


class Timezones
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        t = GeoName.new
        s = line.chomp.split("\t")
        unless s[0].to_s == "CountryCode"
          t.countrycode = s[0]
          t.timezoneid  = s[1]
          t.gmt_offset  = s[2]
          t.dst_offset  = s[3]
          t.raw_offset  = s[4]
        end

        yield t
      end
    end
  end
end

class Tzin

  def initialize(kdf, idtzf)
    @tz = {}
    CSV.foreach(idtzf) do |csv|
      @tz[csv[0].to_i] = csv[1]
    end
    @kd = File.open(kdf) { |f| Kdtree.new(f) }
  end

  def timezoneid(lat, lng)
    id = @kd.nearest(lat.to_f, lng.to_f)
    @tz[id]
  end
end

class Tzout
  GEOURL   = 'api.geonames.org'
  USERNAME = 'alinspoon'

  def self.timezoneid(lat, lng)
    begin
      if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
        uri = URI.parse(proxy)
        proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
        response = Net::HTTP.Proxy(uri.host, uri.port,
                                   proxy_user, proxy_pass).start(GEOURL) do |http|
                     http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
                   end
      else
         response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
      end
      js = JSON.parse(response)
      [js['timezoneId'], js['gmtOffset']]
    rescue Exception => e
      p e
    end
  end
end

def get_points
  citiesfile   = "data/cities15000.txt"
  points = []
  GeoNames.parse(citiesfile) do |place|
    if rand(100) == 1
      points << [place.latitude.to_i, place.longitude.to_i]
    end
  end
  points
end

#def test_get_points
#  points = get_points
#end

def test
  allkd    = "data/allCountries.kdtree"
  allidtz  = "data/id_tz_all.csv"
  citykd   = "data/cities15000.kdtree"
  cityidtz = "data/id_tz.csv"

  tzin_c = Tzin.new(citykd, cityidtz)
  tzin_a = Tzin.new(allkd, allidtz)

  errors_c = 0
  errors_a = 0
  errors_tz2nil = 0
  points = get_points
p "points.size:"
p points.size
  points.each do |p|
      lat, lng = p
      tz1_c = tzin_c.timezoneid(lat, lng)
      tz1_a = tzin_a.timezoneid(lat, lng)
      tz2, gmt = Tzout.timezoneid(lat, lng)
      errors_c += 1 unless tz1_c == tz2
      if tz2.nil?
        ENV['TZ'] = tz1_a
        errors_a += 1 unless gmt.to_i == Time.now.gmtoff/3600
      elsif tz1_a != tz2
        errors_a += 1
      end
      sleep 1
  end
  p "errors_c:"
  p errors_c
  p "errors_a:"
  p errors_a
  p "errors_tz2nil:"
  p errors_tz2nil
end

test

comparison.rb

require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'

require 'rubygems'
require 'kdtree'


class Tzin

  def initialize(kdf, idtzf)
    @tz = {}
    CSV.foreach(idtzf) do |csv|
      @tz[csv[0].to_i] = csv[1]
    end
    @kd = File.open(kdf) { |f| Kdtree.new(f) }
  end

  def timezoneid(lat, lng)
    id = @kd.nearest(lat.to_f, lng.to_f)
    @tz[id]
  end
end

class Tzout
  GEOURL   = 'api.geonames.org'
  USERNAME = 'alinspoon'

  def self.timezoneid(lat, lng)
    begin
      if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
        uri = URI.parse(proxy)
        proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
        response = Net::HTTP.Proxy(uri.host, uri.port,
                                   proxy_user, proxy_pass).start(GEOURL) do |http|
                     http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
                   end
      else
         response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
      end
      js = JSON.parse(response)
      [js['timezoneId'], js['gmtOffset']]
    rescue Exception => e
      p e
    end
  end
end

class GenRand
  class << self
    def latlng(ca)
      case ca
      when "ne"
        [rand90, rand180]
      when "nw"
        [rand90, -rand180]
      when "se"
        [-rand90, rand180]
      when "sw"
        [-rand90, -rand180]
      end
    end

    private

    def rand90
      Random.new.rand 89.9
    end

    def rand180
      Random.new.rand 179.9
    end
  end
end

def test
  allkd    = "data/allCountries.kdtree"
  allidtz  = "data/id_tz_all.csv"
  citykd   = "data/cities15000.kdtree"
  cityidtz = "data/id_tz.csv"

  tzin_c = Tzin.new(citykd, cityidtz)
  tzin_a = Tzin.new(allkd, allidtz)

  errors_c = 0
  errors_a = 0
  errors_tz2nil = 0
  kaisu = 100
  kaisu.times do
    ["ne","nw","se","sw"].each do |p|
      lat, lng = GenRand.latlng(p)
      tz1_c = tzin_c.timezoneid(lat, lng)
      tz1_a = tzin_a.timezoneid(lat, lng)
      tz2, gmt = Tzout.timezoneid(lat, lng)
      errors_c += 1 unless tz1_c == tz2
      unless tz1_a == tz2
        if tz2.nil?
          ENV['TZ'] = tz1_a
          errors_tz2nil += 1 unless gmt.to_i == Time.now.gmtoff/3600
        end
        errors_a += 1
      end
      sleep 1
    end
  end
  p "errors_c:"
  p errors_c
  p "errors_a:"
  p errors_a
  p "errors_tz2nil:"
  p errors_tz2nil
  p "kaisu"
  p kaisu*4
end

test

comparison_isan.rb

require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'

require 'rubygems'
require 'kdtree'


class Tzin

  def initialize(kdf, idtzf)
    @tz = {}
    CSV.foreach(idtzf) do |csv|
      @tz[csv[0].to_i] = [csv[1],csv[2],csv[3]]
    end
    @kd = File.open(kdf) { |f| Kdtree.new(f) }
  end

  def timezoneid(lat, lng)
    id = @kd.nearest(lat.to_f, lng.to_f)
    @tz[id]
  end
end

class Tzout
  GEOURL   = 'api.geonames.org'
  USERNAME = 'alinspoon'

  def self.timezoneid(lat, lng)
    begin
      if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
        uri = URI.parse(proxy)
        proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
        response = Net::HTTP.Proxy(uri.host, uri.port,
                                   proxy_user, proxy_pass).start(GEOURL) do |http|
                     http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
                   end
      else
         response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
      end
      js = JSON.parse(response)
      [js['timezoneId'], js['gmtOffset']]
    rescue Exception => e
      p e
    end
  end
end


def test
  allkd    = "data/allCountries.kdtree"
  allidtz  = "data/id_tz.csv"
  isanfile = "data/isan.csv"

  tzin_a = Tzin.new(allkd, allidtz)


  errors = 0
  CSV.foreach(isanfile) do |csv|
    lat, lng = csv[1], csv[2]
    tz1_a, tz1_gmt, tz1_dst = tzin_a.timezoneid(lat, lng)
    tz2, gmt = Tzout.timezoneid(lat, lng)
    if tz1_a != tz2
      unless tz1_a.nil? || tz1_a.empty?
        ENV['TZ'] = tz1_a
        unless gmt.to_i == Time.now.gmtoff/3600
          errors += 1
          puts "tz1_gmt: " + (Time.now.gmtoff/3600).to_s
          puts "gmt:     " + gmt.to_s
          puts csv[0].to_s + "  " + csv[3].to_s
          puts "-----------------"
        end
      end
    elsif tz1_a.nil?
      unless gmt.to_i == tz1_gmt.to_i
        errors += 1
        puts "tz1_gmt: " + tz1_gmt.to_s
        puts "gmt:     " + gmt.to_s
        puts csv[0].to_s + "  " + csv[3].to_s
        puts "-----------------"
      end
    end
    sleep 0.3
  end
  p "errors:"
  p errors
end

test

id_tz_data_hosei.rb

equire 'ostruct'
require 'csv'
require 'date'
require 'net/http'
require 'uri'
require 'json'

require 'rubygems'
require 'kdtree'

class GeoName < OpenStruct
end

class GeoNames
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        g = GeoName.new
        s = line.chomp.split("\t")
        g.geonameid = s[0]
        g.name = s[1]
        g.asciiname = s[2]
        g.alternatenames = s[3]
        g.latitude = s[4]
        g.longitude = s[5]
        g.feature_class = s[6]
        g.feature_code = s[7]
        g.country_code = s[8]
        g.cc2 = s[9]
        g.admin1 = s[10]
        g.admin2 = s[11]
        g.admin3 = s[12]
        g.admin4 = s[13]
        g.population = s[14]
        g.elevation = s[15]
        g.gtopo30 = s[16]
        g.timezone = s[17]
#        g.modification_date = Time.parse(s[18])

        yield g
      end
    end
  end
end


class Timezones
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        t = GeoName.new
        s = line.chomp.split("\t")
        unless s[0].to_s == "CountryCode"
          t.countrycode = s[0]
          t.timezoneid  = s[1]
          t.gmt_offset  = s[2]
          t.dst_offset  = s[3]
          t.raw_offset  = s[4]
        end

        yield t
      end
    end
  end
end

class Tzin
  KDFILE   = "data/allCountries.kdtree"
  IDTZFILE = "data/id_tz.csv"

  def initialize
    @tz = {}
    CSV.foreach(IDTZFILE) do |csv|
      @tz[csv[0].to_i] = csv[1]
    end
    @kd = File.open(KDFILE) { |f| Kdtree.new(f) }
  end

  def timezoneid(lat, lng)
    id = @kd.nearest(lat.to_f, lng.to_f)
    @tz[id]
  end
end

class Tzout
  GEOURL   = 'api.geonames.org'
  USERNAME = 'alinspoon'

  def self.timezoneid(lat, lng)
    begin
      if proxy = ENV["http_proxy"] || ENV["HTTP_PROXY"]
        uri = URI.parse(proxy)
        proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
        response = Net::HTTP.Proxy(uri.host, uri.port,
                                   proxy_user, proxy_pass).start(GEOURL) do |http|
                     http.get("/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}").body
                   end
      else
         response = Net::HTTP.get(GEOURL, "/timezoneJSON?lat=#{lat}&lng=#{lng}&username=#{USERNAME}")
      end
      js = JSON.parse(response)
      [js['timezoneId'], js['gmtOffset'], js['dstOffset']]
    rescue Exception => e
      p e
    end
  end
end

citiesfile   = "data/allCountries.txt"
kdtreefile   = "data/allCountries.kdtree"
idtzfile     = "data/id_tz.csv"
idtzhosei    = "data/id_tz_hosei.csv"

####################
# geonameidを取得するために、Kdtree.new(f)用なファイルを生成する
###################

timesnil = 0
total = 0
tzoutnil = 0

p Time.now
CSV.open(idtzhosei, "wb") do |csv|
  GeoNames.parse(citiesfile) do |place|
    if place.timezone.nil? || place.timezone == ""
      tz, gmtoff, dstoff = Tzout.timezoneid(place.latitude, place.longitude)
      if tz
        csv << [place.geonameid, tz, gmtoff, dstoff]
      end
      timesnil += 1
    else
      csv << [place.geonameid, place.timezone, nil, nil]
    end
    total += 1
  end
end
p Time.now
p tzoutnil
p timesnil
p total

local2utc.rb

require 'csv'
require 'date'

require 'rubygems'
require 'kdtree'

tmpfile  = "data/cities15000.kdtree"
idtzfile = "data/id_tz.csv"

tz = {}
CSV.foreach(idtzfile) do |csv|
  tz[csv[0].to_i] = csv[1]
end

kd = File.open(tmpfile) { |f| Kdtree.new(f) }
loop do
  p "input lat or input q for quit"
  lat = gets.chomp
  if lat == "q"
    "input over........"
    break
  else
    p "input lng"
    lng = gets.chomp
    p "input localtime ex. 2000-1-1 15:00:00"
    lt = gets.chomp
  end
  p id = kd.nearest(lat.to_f, lng.to_f)
  p tz[id]
  ENV['TZ'] = tz[id]
  d = DateTime.parse(lt)
  p Time.local(d.year, d.month, d.day, d.hour, d.min, d.sec).utc
end

test_allCountries_data.rb

require 'ostruct'
require 'csv'

require 'rubygems'
require 'kdtree'

class GeoName < OpenStruct
end

class GeoNames
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        g = GeoName.new
        s = line.chomp.split("\t")
        g.geonameid = s[0]
        g.name = s[1]
        g.asciiname = s[2]
        g.alternatenames = s[3]
        g.latitude = s[4]
        g.longitude = s[5]
        g.feature_class = s[6]
        g.feature_code = s[7]
        g.country_code = s[8]
        g.cc2 = s[9]
        g.admin1 = s[10]
        g.admin2 = s[11]
        g.admin3 = s[12]
        g.admin4 = s[13]
        g.population = s[14]
        g.elevation = s[15]
        g.gtopo30 = s[16]
        g.timezone = s[17]
#        g.modification_date = Time.parse(s[18])

        yield g
      end
    end
  end
end


class Timezones
  class << self
    def parse(file)
      File.new(file).each_line do |line|
        t = GeoName.new
        s = line.chomp.split("\t")
        unless s[0].to_s == "CountryCode"
          t.countrycode = s[0]
          t.timezoneid  = s[1]
          t.gmt_offset  = s[2]
          t.dst_offset  = s[3]
          t.raw_offset  = s[4]
        end

        yield t
      end
    end
  end
end

citiesfile   = "data/allCountries.txt"
kdtreefile   = "data/allCountries.kdtree"
idtzfile     = "data/id_tz.csv"

####################
# geonameidを取得するために、Kdtree.new(f)用なファイルを生成する
###################

timesnil = 0
total = 0
GeoNames.parse(citiesfile) do |place|
  if place.timezone.nil? || place.timezone == ""
    p place.geonameid
    timesnil += 1
  end
  total += 1
end

p timesnil
p total
p timesnil.to_f/total