Skip to content

Instantly share code, notes, and snippets.

@whitequark
Last active October 6, 2022 22:44
Show Gist options
  • Save whitequark/cd89594b797217992d8263325cb77d75 to your computer and use it in GitHub Desktop.
Save whitequark/cd89594b797217992d8263325cb77d75 to your computer and use it in GitHub Desktop.
Strava archiver
1. install postgres
2. run makedb.rb >tiles.csv
3. run tiles.sql
4. run archive.rb
5. enjoy
#!/usr/bin/env ruby
require 'pg'
require 'fileutils'
require 'net/http'
db = {}
nthr = 64
def domain
s = %w{a b c}.sample
"heatmap-external-#{s}.strava.com"
end
def url(z, x, y)
"http://#{domain}/tiles/all/hot/#{z}/#{x}/#{y}@2x.png?v=19"
end
queue = []
nthr.times do
Thread.new do
conn = PG::Connection.open(db)
Net::HTTP.start domain do |http|
loop do
z, x, y = nil
loop do
task = queue.pop
if task.nil?
sleep rand
else
z, x, y = task
break
end
end
tile = http.get(url(z, x, y)).body
FileUtils.mkdir_p "data/#{z}/#{x}"
File.write("data/#{z}/#{x}/#{y}.png", tile)
conn.exec %{UPDATE tiles SET done = 't', size = #{tile.length}
WHERE z = #{z} AND x = #{x} AND y = #{y}}
puts "#{z} #{x.rjust(4)} #{y.rjust(4)}"
end
end
end
end
conn = PG::Connection.open(db)
loop do
sleep 1 while queue.length > nthr
conn.exec %{SELECT z, x, y FROM tiles
WHERE done = 'f' ORDER BY z
LIMIT 1000} do |rows|
break unless rows.any?
rows.each do |row|
queue.push([row["z"], row["x"], row["y"]])
end
end
end
#!/usr/bin/env ruby
14.times do |z|
(2**z).times do |x|
(2**z).times do |y|
puts %{#{z},#{x},#{y},0,}
end
end
end
#!/usr/bin/env ruby
require 'pg'
conn = PG::Connection.open
total_count = 0
total_size = 0
last_z = 0
last_avg = 0
14.times do |z|
size, = *conn.exec(%{SELECT MIN(size), MAX(size), AVG(size)
FROM tiles WHERE size > 0 AND z = #{z}})
break if size["avg"].nil?
count = 2**z*2**z
avg_size = size["avg"].to_i
puts("Z=#{z}:")
puts(" File size:")
puts(" min #{size["min"]}")
puts(" max #{size["max"]}")
puts(" mean #{size["avg"]}")
puts(" median #{size["med"]}")
puts(" Count:")
puts(" #{count}")
puts(" Predicted size:")
puts(" #{count*avg_size}")
puts
total_count += count
total_size += count*avg_size
last_z = z
last_avg = avg_size
end
(last_z+1...14).each do |z|
count = 2**z*2**z
puts("Z=#{z}: (extrapolated)")
puts(" File size:")
puts(" mean #{last_avg}")
puts(" Count:")
puts(" #{count}")
puts(" Predicted size:")
puts(" #{count*last_avg}")
puts
total_count += count
total_size += count*last_avg
end
puts("Grand total:")
puts(" Count:")
puts(" #{total_count}")
puts(" Predicted size:")
puts(" #{total_size}")
CREATE TABLE tiles (z INT, x INT, y INT, done BOOLEAN, size INT);
CREATE INDEX coords ON tiles (z, x, y);
COPY tiles FROM 'tiles.csv' WITH (FORMAT csv);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment