A quick and dirty script for restructing GeServer GWC created tilesets from internal directory structure to TMS-like z/x/y structure. Optionally discarding "empty" files and dividing files into separate subdirectories (e.g based on admin-units) based on an index file Raw

gwc_restructure_tiledir.py

import json
import os
import shutil
from time import time

# absolute path to gwc dir, e.g '/opt/geoserver/data/gwc' or 'c:/geoserver/data_dir/gwc'
TILEPATH = '<path_to_geoserver_gwc_dir>' 

# All zoomfolders in os.path.join(TILPATH, TILELAYER) path that should be scanned,
ZOOMFOLDERS = [
    '<folder_name_at_zoom_0>',
    '<folder_name_at_zoom_1>',
    ...
    '<folder_name_at_zoom_n>',
    ]


def get_zoom(folder):
    """Extracts zoom value from zoom folder name.

    The folder name is expected to be of the form: 
    {{whatever_name_zoomnumber}}.
    """
    return '%s' % int(folder.split('_')[-1])

def get_column_row(filepath, flavour='vanilla'):
    """Extracts tile row folder name + tile filename.

    Flavour can be of type {gwc} or {diskcache}.
    """
    if flavour == 'gwc':
        _x, _y = [n for n in filepath.split('_')]
        x = int(_x)
        y, ext = os.path.splitext(_y)
    elif flavour == 'diskcache':
        x0, x1, x2, y0, y1, y2 = filepath.split(os.path.sep)
        x = int('%s%s%s' % (x0, x1, x2))
        y, ext = os.path.splitext('%s%s%s' % (y0, y1, y2))
    else:
        raise IOError('Unknown tileindex flavour - "%s"' % flavour)
    return '%s' % x, '%s%s' % (int(y), ext)

def walk_tile_folders(tilepath, layername, zoomfolders, outfolder, ext, idxpath, 
    minsize=None, flavour=None):
    """Walks all given zoomfolders and copies tiles to TMS-like 
    `<outfolder>/<fileextension>/{z}/{x}/{y}.<fileextension>` folder structure.

    Optionally:
    
    - minimum file size in bytes can be specified - files smaller 
    than that will not be copied.
    - idxpath can specify a directory with zoom-based index-files. The file name
    must follow a pattern of `z<number>_index.json` with <number> indicating the
    specific zoom level, e.g z10_index.json for zoom level 10, z1_index.json for
    zoom level 1. The distribution index must be encoded in json and be in the 
    form:

    ```
    {
        "z:x:y" : ["subdirs", "that this", "tile", "needs", "to be copied"],
        ...
        "z:m:n" : ["this goes to one subdir only"],
    }
    ```
    If using the idxpath option and a tile is not specified in the zoomlevel's 
    indexfile, it will be discarded. Required subdirs are created automagically.

    NB! will not reverse y-axis!
    """"
    _extdir = os.path.join(outfolder, ext)
    if not os.path.exists(_extdir):
        os.mkdir(_extdir)
    for zoomfolder in zoomfolders:
        start = time()
        zoom = get_zoom(zoomfolder)
        if idxpath:
            idxfile = os.path.join(idxpath, 'z%s_index.json' % zoom)
            with open(idxfile) as i:
                idx = json.loads(i.read())
            print 'Z=%s index loaded in %.2f seconds' % (zoom, time() - start, )
            _def = []
        else:
            idx = {}
            print 'Z=%s no index loaded' % zoom
            _def = ['']
        start = time()
        inzpath = os.path.join(tilepath, layername, zoomfolder)
        numtiles = 0
        discarded = 0
        print 'Copy %s -> %s:' % (inzpath, zoom),
        for thisdir, subdirs, files in os.walk(inzpath, topdown=False):
            for tilefile in files:
                if os.path.splitext(tilefile)[-1] == '.%s' % ext:
                    src = os.path.join(thisdir, tilefile)
                    size = os.path.getsize(src)
                    if (minsize != None and size > minsize) or minsize == None:
                        x, y = get_column_row(
                            os.path.relpath(src, inzpath), flavour)
                        key = '%s:%s:%s' % (zoom, x, y.rstrip('.%s' % ext))
                        distdirs = idx.get(key, _def)
                        for distdir in distdirs:
                            _distdir = os.path.join(_extdir, distdir)
                            if not os.path.exists(_distdir):
                                os.mkdir(_distdir)
                            _zdir = os.path.join(_distdir, zoom)
                            if not os.path.exists(_zdir):
                                os.mkdir(_zdir)
                            _xdir = os.path.join(_zdir, x)
                            if not os.path.exists(_xdir):
                                os.mkdir(_xdir)
                            dst = os.path.join(_xdir, y)
                            shutil.copy2(src, dst)
                            numtiles += 1
                    else:
                        discarded += 1

        print 'copied %s files, discarded %s files in %.2f seconds' % (
            numtiles, discarded, time() - start)


if __name__ == '__main__':
    walk_tile_folders(
        TILEPATH, 
        '<tilelayer_name>', # e.g 'topp_states' 
        ZOOMFOLDERS, 
        '<outfolder>', # The path where tiles will be copied, e.g 'z:/data/tiles/mylayer'
        '<file_format>', # only files w/ this extension will be considered, e.g 'jpeg' or 'geojson' or 'png'
        None, # index file path if tiles need to be divided in separate directories, e.g based on admin units
        None, # min file size thresold in bytes. Files smaller than this will not be copied.
        'gwc' # or 'diskcache', the flavour of directory structure used for original tiles.
    )