Skip to content

Instantly share code, notes, and snippets.

@attilaolah
Last active January 12, 2024 19:17
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save attilaolah/6a5a9c1f54463dcb9fda6188d856c2a2 to your computer and use it in GitHub Desktop.
Save attilaolah/6a5a9c1f54463dcb9fda6188d856c2a2 to your computer and use it in GitHub Desktop.
Mapbox Static Tile Downloader
// Package main downloads tiles from the Mapbox Static Tiles API.
// See https://docs.mapbox.com/api/maps/static-tiles/#retrieve-raster-tiles-from-styles.
//
// Usage:
// go run mapbox_dl.go -h
package main
import (
"flag"
"fmt"
"image"
"image/draw"
"io"
"io/ioutil"
"math"
"net/http"
"os"
"path/filepath"
"strings"
"time"
// Supported tile extensions:
"image/jpeg"
"image/png"
)
var (
token = flag.String("access_token", "", "Mapbox API access token.")
username = flag.String("username", "mapbox", "Mapbox style owner's username.")
style = flag.String("style_id", "satellite-v9", "Mapbox map style ID.")
ext = flag.String("ext", ".jpg", "File extension (usually .png for generated maps, .png for satellite imagery).")
zoom = flag.Int("zoom", 0, "Zoom level")
x2 = flag.Bool("2x", true, "Fetch double-resolution tiles (@2x).")
printURLs = flag.Bool("print", false, "Whether to print URLs to download.")
download = flag.Bool("download", false, "Whether to download tiles or just print the URLs.")
interval = flag.Duration("download_interval", 0, "How much time to wait between downloads (or printouts).")
overwrite = flag.Bool("overwrite", false, "Whether to re-download existing tiles.")
merge = flag.Bool("merge", false, "Whether to merge all tiles into one big square image.")
quality = flag.Int("merge_quality", jpeg.DefaultQuality, "Quality for merging when encoding to JPEG.")
)
// Tileset contains all tiles for a zoom level.
type Tileset struct {
username, style, ext string
zoom int
x2 bool
}
// Tile represents a single tile.
type Tile struct {
*Tileset
x, y int
}
// NewTileset creates a new Tileset from flag values.
func NewTileset() *Tileset {
return &Tileset{
username: *username,
style: *style,
ext: *ext,
zoom: *zoom,
x2: *x2,
}
}
// Size returns the number of of rows/cols.
func (s *Tileset) Size() int {
return int(math.Pow(2, float64(s.zoom)))
}
// Tiles returns a channel providing every tile in the tileset.
func (s *Tileset) Tiles() <-chan Tile {
ch := make(chan Tile)
go func() {
for x := 0; x < s.Size(); x++ {
for y := 0; y < s.Size(); y++ {
ch <- Tile{
Tileset: s,
x: x,
y: y,
}
}
}
close(ch)
}()
return ch
}
// Image returns a blank Image capable to holde every tile.
func (s *Tileset) Image() draw.Image {
size := s.Size() * 512
if s.x2 {
size *= 2
}
return image.NewRGBA(image.Rect(0, 0, size, size))
}
// Path returns the path to the merged image.
func (s *Tileset) Path() string {
x2 := ""
if s.x2 {
x2 = "@2x"
}
return fmt.Sprintf("styles/v1/%s/%s/tiles/%d%s%s", s.username, s.style, s.zoom, x2, s.ext)
}
// URL returns the URL of the tile.
func (t *Tile) URL() string {
p := t.Path()
p = strings.TrimSuffix(p, filepath.Ext(p))
return fmt.Sprintf("https://api.mapbox.com/%s?access_token=%s", p, *token)
}
// Path returns the path to the local file.
func (t *Tile) Path() string {
x2 := ""
if t.x2 {
x2 = "@2x"
}
return fmt.Sprintf("styles/v1/%s/%s/tiles/%d/%d/%d%s%s", t.username, t.style, t.zoom, t.x, t.y, x2, t.ext)
}
// Image reads the downloaded tile as an image.
func (t *Tile) Image() (image.Image, error) {
f, err := os.Open(t.Path())
if err != nil {
return nil, fmt.Errorf("error reading tile %q: %w", t.Path(), err)
}
defer func() {
cerr := f.Close()
if err == nil {
err = cerr
}
}()
var img image.Image
if img, _, err = image.Decode(f); err != nil {
return nil, fmt.Errorf("error decoding tile %q: %w", t.Path(), err)
}
// Allow the defer above to set the error.
return img, err
}
// Rect identifies the tile's position in the merged tileset.
func (t *Tile) Rect() image.Rectangle {
size := 512
if t.x2 {
size *= 2
}
return image.Rect(t.x*size, t.y*size, (t.x+1)*size, (t.y+1)*size)
}
// Download fetches and stores the tile locally.
// Returns true if the file was downloaded, false if failed or cached.
func (t *Tile) Download() (bool, error) {
p := t.Path()
dir := filepath.Dir(p)
if _, err := os.Stat(dir); os.IsNotExist(err) {
// NOTE: ModePerm will have the umask removed.
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return false, fmt.Errorf("error creating directory: %w", err)
}
} else if err != nil {
return false, fmt.Errorf("error checking directory: %w", err)
}
if _, err := os.Stat(p); os.IsNotExist(err) || *overwrite {
err = t.DownloadTo(p)
return err == nil, err
} else if err != nil {
return false, fmt.Errorf("error checking file: %w", err)
}
return false, nil
}
// DownloadTo fetches and stores the tile in 'path'.
func (t *Tile) DownloadTo(path string) error {
res, err := http.Get(t.URL())
if err != nil {
return fmt.Errorf("error downloading tile: %w", err)
}
defer func() {
cerr := res.Body.Close()
if err == nil {
err = cerr
}
}()
if res.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(res.Body)
return fmt.Errorf("error downloading tile: %q: %s", res.Status, body)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("error creating file: %w", err)
}
defer func() {
cerr := f.Close()
if err == nil {
err = cerr
}
}()
_, err = io.Copy(f, res.Body)
return err
}
func main() {
flag.Parse()
if (*download || *printURLs) && *token == "" {
println("If -download or -print is set, -access_token must also be set.")
os.Exit(1)
}
s := NewTileset()
for t := range s.Tiles() {
if *download {
fmt.Printf("%s… ", t.Path())
if dl, err := t.Download(); err != nil {
println("ERROR:")
println(err.Error())
} else if dl {
fmt.Println("OK")
time.Sleep(*interval)
} else {
fmt.Println("CACHED")
}
} else if *printURLs {
fmt.Println(t.URL())
}
}
if !*merge {
return
}
img := s.Image()
f, err := os.Create(s.Path())
if err != nil {
print("error creating merged file: ")
println(err.Error())
os.Exit(1)
}
defer func() {
if err = f.Close(); err != nil {
print("error closing merged file: ")
println(err.Error())
os.Exit(1)
}
}()
fmt.Printf("%s… ", s.Path())
for t := range s.Tiles() {
i, err := t.Image()
if err != nil {
print("error reading tile: ")
println(err.Error())
os.Exit(1)
}
draw.Draw(img, t.Rect(), i, image.Point{}, draw.Over)
}
if *ext == ".png" {
if err = png.Encode(f, img); err != nil {
print("error saving merged file: ")
println(err.Error())
os.Exit(1)
}
fmt.Println("OK")
return
}
if *ext == ".jpg" {
if err = jpeg.Encode(f, img, &jpeg.Options{
Quality: *quality,
}); err != nil {
print("error saving merged file: ")
println(err.Error())
os.Exit(1)
}
fmt.Println("OK")
return
}
print("extension not supported: ")
println(*ext)
os.Exit(1)
}
@MaxVero
Copy link

MaxVero commented Jan 12, 2024

Hi Mr. Attilaolah, I ask you a simple question. In your opinion with this code is it possibile to download hundreds of images and use in a computer vision task? I've read the limit but I'm not sure about these kind of algorithm can be tolerate by term of service, for example in google maps is it not possibile. What do you know about this? Thank you in advance.

@attilaolah
Copy link
Author

This is quite old and not really the best idea for mass downloading. Maybe you could use it to just generate the URLs, then use scrapy or something to fetch them in parallel. Even then, (a) Mapbox probably won't let you do it, and (b) you'll use up your API tokens quite quickly if you want mass download. I'd suggest finding a ready-made archive of pre-rendered images. I haven't read the ToS though. Indeed, Google won't let you even store the map tiles, not even in a cache if I'm not mistaken, but I'm not sure about Mapbox.

Alternatively, you could render the tiles yourself using mapnik or some more modern alternative.

Sorry if this didn't help much. It also depends on how many tiles you need, if it's <100k then maybe you should just go ahead and try, and see what happens. You say "hundreds" of images so that doesn't sound like something that would even be noticed anywhere.

@MaxVero
Copy link

MaxVero commented Jan 12, 2024 via email

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment