Strava archiver
1. install postgres | |
2. run makedb.rb >tiles.csv | |
3. run tiles.sql | |
4. run archive.rb | |
5. enjoy |
#!/usr/bin/env ruby | |
require 'pg' | |
require 'fileutils' | |
require 'net/http' | |
db = {} | |
nthr = 64 | |
def domain | |
s = %w{a b c}.sample | |
"heatmap-external-#{s}.strava.com" | |
end | |
def url(z, x, y) | |
"http://#{domain}/tiles/all/hot/#{z}/#{x}/#{y}@2x.png?v=19" | |
end | |
queue = [] | |
nthr.times do | |
Thread.new do | |
conn = PG::Connection.open(db) | |
Net::HTTP.start domain do |http| | |
loop do | |
z, x, y = nil | |
loop do | |
task = queue.pop | |
if task.nil? | |
sleep rand | |
else | |
z, x, y = task | |
break | |
end | |
end | |
tile = http.get(url(z, x, y)).body | |
FileUtils.mkdir_p "data/#{z}/#{x}" | |
File.write("data/#{z}/#{x}/#{y}.png", tile) | |
conn.exec %{UPDATE tiles SET done = 't', size = #{tile.length} | |
WHERE z = #{z} AND x = #{x} AND y = #{y}} | |
puts "#{z} #{x.rjust(4)} #{y.rjust(4)}" | |
end | |
end | |
end | |
end | |
conn = PG::Connection.open(db) | |
loop do | |
sleep 1 while queue.length > nthr | |
conn.exec %{SELECT z, x, y FROM tiles | |
WHERE done = 'f' ORDER BY z | |
LIMIT 1000} do |rows| | |
break unless rows.any? | |
rows.each do |row| | |
queue.push([row["z"], row["x"], row["y"]]) | |
end | |
end | |
end |
#!/usr/bin/env ruby | |
require 'pg' | |
conn = PG::Connection.open | |
total_count = 0 | |
total_size = 0 | |
last_z = 0 | |
last_avg = 0 | |
14.times do |z| | |
size, = *conn.exec(%{SELECT MIN(size), MAX(size), AVG(size) | |
FROM tiles WHERE size > 0 AND z = #{z}}) | |
break if size["avg"].nil? | |
count = 2**z*2**z | |
avg_size = size["avg"].to_i | |
puts("Z=#{z}:") | |
puts(" File size:") | |
puts(" min #{size["min"]}") | |
puts(" max #{size["max"]}") | |
puts(" mean #{size["avg"]}") | |
puts(" median #{size["med"]}") | |
puts(" Count:") | |
puts(" #{count}") | |
puts(" Predicted size:") | |
puts(" #{count*avg_size}") | |
puts | |
total_count += count | |
total_size += count*avg_size | |
last_z = z | |
last_avg = avg_size | |
end | |
(last_z+1...14).each do |z| | |
count = 2**z*2**z | |
puts("Z=#{z}: (extrapolated)") | |
puts(" File size:") | |
puts(" mean #{last_avg}") | |
puts(" Count:") | |
puts(" #{count}") | |
puts(" Predicted size:") | |
puts(" #{count*last_avg}") | |
puts | |
total_count += count | |
total_size += count*last_avg | |
end | |
puts("Grand total:") | |
puts(" Count:") | |
puts(" #{total_count}") | |
puts(" Predicted size:") | |
puts(" #{total_size}") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment