Given a list of OSM meta-tiles, render the meta-tile using Mapnik, and chop up the tile using ImageMagick.

README.md

About

This program is like the more well known generate_tiles.py however rather than rendering all tiles within a predefined bounding box, it will render tiles listed in a file.

If you have a file name tileList.txt with contents

0/0/0
1/0/0
1/0/1
1/1/1
2/3/3

You can render all these tiles using the command,

./render_mapnik_osm_metatiles -m MAPNIK_MAP.xml -j NUM_THREADS -f tileList.txt -t META_TILES

Internally it will render these tiles as larger as metatiles of size given by -t where the tile reference in your tile list is the top left tile of the metatile. You will not see these metatiles saved to disk though, only the final 256x256 will be present.

LICENSE

I release this program under the GNU Lesser General Public License version 2.1 or later.

control

Source: mapnik-osm-metatiles
Section: util
Priority: extra
Maintainer: Andrew Harvey <andrew.harvey4@gmail.com>
Build-Depends:
 libmapnik2-dev
 , libmagick++-dev
 , libboost-program-options-dev
Standards-Version: 3.9.2
Vcs-Git: git://gist.github.com/1397377.git
Vcs-Browser: https://gist.github.com/1397377

Package: mapnik-osm-metatiles
Architecture: all
Depends:
 libmagick++4
 , libmapnik2-2.0
 , libboost-program-options1.46.1 | libboost-program-options1.42.0
Description: Renders OSM tiles using mapnik.
 Given a list of OSM tiles to render, this program renders them using Mapnik
 as meta-tiles. It then chops up the metatile producing tiles in an z/x/y
 structure.

Makefile

CXX = g++

CXXFLAGS = $(shell mapnik-config --cflags) $(shell Magick++-config --cxxflags --cppflags) -Wall
LDFLAGS = $(shell mapnik-config --libs --dep-libs --ldflags) $(shell Magick++-config --ldflags --libs) -lboost_program_options -lpthread

OBJ = mapnik_osm_metatiles.o

BIN = render_mapnik_osm_metatiles

all : $(BIN)

$(BIN) : $(OBJ)
	$(CXX) $(OBJ) $(LDFLAGS) -o $@

.c.o :
	$(CXX) -c $(CXXFLAGS) $<

.PHONY : clean

clean:
	rm -f $(OBJ)
	rm -f $(BIN)

mapnik_osm_metatiles.cpp

/*****************************************************************************
 *
 * Copyright (C) 2011 Andrew Harvey <andrew.harvey4@gmail.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/


/* This program takes a list of OSM meta-tiles, and renders them using Mapnik,
   then chops up the meta-tiles into regular 256x256 size tiles. */


// IO libraries
#include <iostream>
#include <fstream>
#include <string>

#include <math.h>

#include <assert.h>

// queue for keeping store of which tiles to render
#include <queue>

// filesystem libraries for making new directories
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>

// use Boost to parse program arguments/options
#include <boost/program_options.hpp>
namespace po = boost::program_options;

// use pthreads to multithread the rendering engine
#include <pthread.h>

// Mapnik includes
#include <mapnik/map.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/load_map.hpp>
#include <mapnik/font_engine_freetype.hpp>
#include <mapnik/agg_renderer.hpp>
#include <mapnik/image_util.hpp>
#include <mapnik/config_error.hpp>

// ImageMagick for splitting meta tiles
#include <Magick++.h>

/* Default width and height of each individual tile in pixels.

   This value is defined by the specification at 
   http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames */
#define TILE_SIZE 256

/* limits of the Spherical Mercator projection that OSM uses on the default web
   map */
#define MAX_X 20037508
#define MAX_Y 20037508

/* A Meta-OSM Slippy map tile structure */
struct Metatile {
    unsigned int z, x, y;
    bool notNull;
};

/* An arbitary geometry bounding box with no coordinate system */
struct Bbox {
    double left, bottom, right, top;
};

// function declarations
Bbox tileToMercBounds(int z, int x, int y);
void *renderThread(void *argument);
Metatile getNextTile();

// empty global tile queue
std::queue<Metatile> tileQueue;
pthread_mutex_t tileQueueMutex = PTHREAD_MUTEX_INITIALIZER;

// The buffer in pixels to render beyond the extent of the meta tile.
int buffer = 128;

// mapnik map.xml file
std::string mapfile;

/* metaTiles^2 will equal the number of tiles to render at a time as one large
   meta tile, which will then be split up and save separately */
unsigned int metaTiles = 4;

std::string tileFileName;

int main ( int argc , char** argv)
{
    try {

        /*     configuration     */

        // number of rendering threads
        int numThreads = 1;

        // read program options
        po::options_description desc("Allowed options");
        desc.add_options()
            ("help,h", "produce help message")
            ("buffer,b", po::value<int>(&buffer)->default_value(128), "set rendering buffer")
            ("metaTiles,t", po::value<unsigned int>(&metaTiles)->default_value(4), "set number of metaTiles")
            ("mapfile,m", po::value(&mapfile), "set number of metaTiles")
            ("threads,j", po::value<int>(&numThreads)->default_value(1), "set number of rendering threads")
            ("tileFile,f", po::value(&tileFileName), "file with list of tiles to render")
        ;

        po::variables_map vm;
        po::store(po::parse_command_line(argc, argv, desc), vm);
        po::notify(vm);

        if (vm.count("help")) {
            std::cout << desc << "\n";
            return EXIT_SUCCESS;
        }

        if (!vm.count("mapfile")) {
            std::cout << desc << "\n";
            return EXIT_FAILURE;
        }

        // read all tiles into queue
        std::string line;
        std::ifstream tileFile(tileFileName.c_str());
        if (tileFile.is_open()) {
            while (tileFile.good()) {
                getline(tileFile, line);

                // skip empty lines
                if (line.size() > 0) {
                    // parse the line for an z/x/y tile reference
                    int slash1 = line.find('/');
                    int z = atoi(line.substr(0, slash1).c_str());
                    int slash2 = line.find('/', slash1 + 1);
                    int x = atoi(line.substr(slash1 + 1, slash2 - slash1 - 1).c_str());
                    int slash3 = line.find('/', slash2 + 1);
                    int y = atoi(line.substr(slash2 + 1, slash3 - slash2 - 1).c_str());

                    std::stringstream tileDir;
                    tileDir << "tiles/" << z << "/" << x;
                    std::stringstream tileRefStream;
                    tileRefStream << "tiles/" << z << "/" << x << "/" << y << ".png";
                    std::string tileRef = tileRefStream.str();

                    // check z/x/y tile looks like a valid tile
                    if (!((z >= 0) && (z <= 30))) {
                        fprintf(stderr, "Unexpected tile reference.Does not satisfy 0 <= z <= 30, found: %d", z);
                        continue;
                    }

                    if (!((x >= 0) && (x < pow(2, z)))) {
                        fprintf(stderr, "Unexpected tile reference. Does not satisfy 0 <= x < 2^%d, found: %d", z, x);
                        continue;
                    }

                    if (!((y >= 0) && (y < pow(2, z)))) {
                        fprintf(stderr, "Unexpected tile reference. Does not satisfy 0 <= y < 2^%d, found: %d", z, y);
                        continue;
                    }

                    struct Metatile tile = {z, x, y, TRUE};
                    tileQueue.push(tile);
                }
            }
        }else{
            std::cout << "Unable to open file.\n";
            return EXIT_FAILURE;
        }

        // register mapnik datasources
        mapnik::datasource_cache::instance()->register_datasources("/usr/lib/mapnik/2.0/input/");

        // register all truetype fonts recursively below the Debian font directory
        mapnik::freetype_engine::register_fonts("/usr/share/fonts/truetype/", true);

        // span rendering threads
        pthread_t renderingThreads[numThreads];
        int threadID[numThreads];

        for (int i = 0; i < numThreads; i++) {
            threadID[i] = i;
            printf("Spawning rendering thread %d\n", i);
            int rc = pthread_create(&renderingThreads[i], NULL, renderThread, (void *) &threadID[i]);
            assert(rc == 0);
        }

        /* wait for all threads to complete */
        for (int i = 0; i < numThreads; i++) {
            int rc = pthread_join(renderingThreads[i], NULL);
            assert(0 == rc);
        }

        std::cout << "done";
        exit(EXIT_SUCCESS);
    }
    catch ( const mapnik::config_error & ex )
    {
        std::cerr << "### Configuration error: " << ex.what() << std::endl;
        return EXIT_FAILURE;
    }
    catch ( const std::exception & ex )
    {
        std::cerr << "### std::exception: " << ex.what() << std::endl;
        return EXIT_FAILURE;
    }
    catch ( ... )
    {
        std::cerr << "### Unknown exception." << std::endl;
        return EXIT_FAILURE;
    }
    return EXIT_SUCCESS;
}

/* returns the next tile from the tileQueue for a rendering thread */
Metatile getNextTile() {
    Metatile tile;

    pthread_mutex_lock(&tileQueueMutex);
        if (tileQueue.empty()) {
            tile.notNull = FALSE;
        }else{
            tile = tileQueue.front();
            tileQueue.pop();
        }
    pthread_mutex_unlock(&tileQueueMutex);

    return tile;
}

void *renderThread(void *argument) {
    int tid = *((int *) argument);

    // create a new meta tile as a mapnik map to render on
    mapnik::Map m(TILE_SIZE * metaTiles, TILE_SIZE * metaTiles);

    // load our map stylesheet
    mapnik::load_map(m, mapfile);

    Metatile tile = getNextTile();
    while (tile.notNull) {
        unsigned int x = tile.x;
        unsigned int y = tile.y;
        unsigned int z = tile.z;

        printf("%d: Rendering %d/%d/%d * %d\n", tid, z, x, y, metaTiles);

        Bbox bound_top_left = tileToMercBounds(z, x, y);

        int brx = x + metaTiles - 1;
        int bry = y + metaTiles - 1;

        // ensure the metatile doesn't extend beyond normal tile limits
        assert(brx < pow(2, z));
        assert(bry < pow(2, z));

        Bbox bound_bottom_right = tileToMercBounds(z, brx, bry);
        m.set_buffer_size(buffer);
        m.zoom_to_box(mapnik::box2d<double>(bound_top_left.left, bound_bottom_right.bottom, bound_bottom_right.right, bound_top_left.top));

        mapnik::image_32 buf(m.width(),m.height());
        mapnik::agg_renderer<mapnik::image_32> ren(m,buf);
        ren.apply();

        // make parent tile directories
        if (mkdir("tiles", S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) && (errno != EEXIST)) {
            fprintf(stderr, "Can't make tile directory: tiles (%s)\n", strerror(errno));
            return NULL;
        }

        char tilesDir[200];

        // make z directory
        sprintf(tilesDir, "tiles/%d", z);
        if (mkdir(tilesDir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) && (errno != EEXIST)) {
            fprintf(stderr, "Can't make tile directory: tiles (%s)\n", strerror(errno));
            return NULL;
        }

        // make x directories (enough for all the split tiles too)
        for (unsigned int i = 0; i < metaTiles; i++) {
            sprintf(tilesDir, "tiles/%d/%d", z, x + i);
            if (mkdir(tilesDir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) && (errno != EEXIST)) {
                fprintf(stderr, "Can't make tile directory: tiles (%s)\n", strerror(errno));
                return NULL;
            }
        }

        // save the tile image in memory encoded using PNG
        std::string png_metaTile = mapnik::save_to_string<mapnik::image_data_32>(buf.data(), "png");

        // split the metatile up into 256*256 size tiles

        // grab the in memory PNG encoded image as a Magick::Image
        Magick::Image IM_metaTile;
        IM_metaTile.magick("PNG");
        Magick::Blob IM_blob(png_metaTile.data(), png_metaTile.size());
        IM_metaTile.read(IM_blob);

        for (unsigned int mi = 0; mi < metaTiles; mi++) {
            for(unsigned int mj = 0; mj < metaTiles; mj++) {
                char tileName[200];
                sprintf(tileName, "tiles/%d/%d/%d.png", z, x + mi, y + mj);
                Magick::Image IM_tile = IM_metaTile;
                IM_tile.crop( Magick::Geometry(256, 256, 256*mi, 256*mj) );
                IM_tile.write(tileName);
            }
        }

        tile = getNextTile();
    }

    return NULL;
}

/* For a given OSM z/x/y tile, find the bounds of this tile in the Spherical
   Mercator projection which OSM uses. */
Bbox tileToMercBounds(int z, int x, int y) {
    // special case for zoom 0 which spans the while projection space
    if (z == 0) {
        Bbox bbox = {-MAX_X, -MAX_Y, MAX_X, MAX_Y};
        return bbox;
    }

    /* the reference number for the tile directly after the centre axis of the
       projected space */
    int tilePastCentreAxis = pow(2,z) / 2;

    /* if tile is beyond expected extents cap it
         x = 0 .. (2^z - 1)
         y = 0 .. (2^z - 1)   */
    assert((x >= 0) && (x < pow(2, z)));
    assert((y >= 0) && (y < pow(2, z)));

    /* flip all tiles left/above of the centre axis across to the right/bottom
       side for now in other words move everything to the bottom right quadrant */
    bool mirrored_x = false;
    bool mirrored_y = false;

    if (x < tilePastCentreAxis) {
        mirrored_x = true;
        x = (pow(2, z) - 1) - x;
    }

    if (y < tilePastCentreAxis) {
        mirrored_y = true;
        y = (pow(2, z) - 1) - y;
    }

    // 2^z / 2 <= x < 2^z
    assert ((tilePastCentreAxis <= x) && (x < (pow(2, z))));
    assert ((tilePastCentreAxis <= y) && (y < (pow(2, z))));

    /* we need to cast the numerator to double in case both the numerator and
       and denominator would have been integers, but true answer should have
       been a floating point number.

       we also need to cast the tile number to double so that when we multiply
       it by MAX_X, we don't overfloat the int type */
    double bound_left = ((double)(((double)x - tilePastCentreAxis) * (MAX_X))) / tilePastCentreAxis;
    double bound_right = ((double)(((double)(x + 1) - tilePastCentreAxis)  * (MAX_X))) / tilePastCentreAxis;

    double bound_top = (((double)(((double)y - tilePastCentreAxis) * (MAX_Y))) / tilePastCentreAxis) * -1;
    double bound_bottom = (((double)(((double)(y + 1) - tilePastCentreAxis) * (MAX_Y))) / tilePastCentreAxis) * -1;

    /* if we mirrored the tiles earlier, fix them up now that we have the merc
       bounds */
    if (mirrored_x) {
        double br = bound_right;
        double bl = bound_left;

        bound_left = -1 * br;
        bound_right = -1 * bl;
    }

    if (mirrored_y) {
        double bb = bound_bottom;
        double bt = bound_top;

        bound_bottom = -1 * bt;
        bound_top = -1 * bb;
    }

    // return the bounds of this tile in the Spherical Mercator projection
    struct Bbox bbox = {bound_left, bound_bottom, bound_right, bound_top};
    return bbox;
}