Skip to content

Instantly share code, notes, and snippets.

@andrewharvey
Created August 8, 2012 12:03
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save andrewharvey/3294610 to your computer and use it in GitHub Desktop.
Save andrewharvey/3294610 to your computer and use it in GitHub Desktop.
Tools, utilities and stylesheets for working with the Geoscience Australia National Dynamic Land Cover Dataset
#!/bin/sh
./scripts/01-download.sh
./scripts/02-unzip.sh
./scripts/03-reproject.sh
./scripts/04-transform-classification.pl
#!/bin/sh
# This script is licensed CC0 by Andrew Harvey <andrew.harvey4@gmail.com>
#
# To the extent possible under law, the person who associated CC0
# with this work has waived all copyright and related or neighboring
# rights to this work.
# http://creativecommons.org/publicdomain/zero/1.0/
mkdir -p downloads-zip
if test -z "$1" ; then
n=2 # by default just download the first two files in the list
else
n="$1"
fi
wget -O - 'https://www.ga.gov.au/products/servlet/controller?product_id=51382&product_id=51383&product_id=51384&product_id=51386&product_id=51385&sector=General+public&occupation=Spatial+information+services&email=&co=Continue+to+File+Download+-%3E&event=FILE_LISTING' | \
grep 'DOWNLOAD_FILE' | grep -o "\"[^\"]*\"" | sed 's/^"/https:\/\/www.ga.gov.au/' | sed 's/"$//' | \
head -n $n | \
wget -i - --directory-prefix=downloads-zip --trust-server-names --no-clobber
#!/bin/sh
# This script is licensed CC0 by Andrew Harvey <andrew.harvey4@gmail.com>
#
# To the extent possible under law, the person who associated CC0
# with this work has waived all copyright and related or neighboring
# rights to this work.
# http://creativecommons.org/publicdomain/zero/1.0/
for f in downloads-zip/*.zip ; do
z=`basename "$f" .zip`;
mkdir -p "downloads-unzip/$z";
unzip -n -d "downloads-unzip/$z" "$f";
done
#!/bin/sh
# This script is licensed CC0 by Andrew Harvey <andrew.harvey4@gmail.com>
#
# To the extent possible under law, the person who associated CC0
# with this work has waived all copyright and related or neighboring
# rights to this work.
# http://creativecommons.org/publicdomain/zero/1.0/
mkdir -p raster-web-mercator
gdalwarp -t_srs "EPSG:900913" -r near -of VRT downloads-unzip/Scene01-DLCDv1_Class/DLCDv1_Class.tif raster-web-mercator/DLCDv1_Class.vrt
gdalwarp -t_srs "EPSG:900913" -r near -of VRT downloads-unzip/Scene01-trend_evi_max/trend_evi_max.tif raster-web-mercator/trend_evi_max.vrt
gdalwarp -t_srs "EPSG:900913" -r near -of VRT downloads-unzip/Scene01-trend_evi_mean/trend_evi_mean.tif raster-web-mercator/trend_evi_mean.vrt
gdalwarp -t_srs "EPSG:900913" -r near -of VRT downloads-unzip/Scene01-trend_evi_min/trend_evi_min.tif raster-web-mercator/trend_evi_min.vrt
gdal_translate -of GTiff -co COMPRESS=DEFLATE -co ZLEVEL=9 raster-web-mercator/DLCDv1_Class.vrt raster-web-mercator/DLCDv1_Class.tiff
gdal_translate -of GTiff -co COMPRESS=DEFLATE -co ZLEVEL=9 raster-web-mercator/trend_evi_max.vrt raster-web-mercator/trend_evi_max.tiff
gdal_translate -of GTiff -co COMPRESS=DEFLATE -co ZLEVEL=9 raster-web-mercator/trend_evi_mean.vrt raster-web-mercator/trend_evi_mean.tiff
gdal_translate -of GTiff -co COMPRESS=DEFLATE -co ZLEVEL=9 raster-web-mercator/trend_evi_min.vrt raster-web-mercator/trend_evi_min.tiff
rm -f raster-web-mercator/*.vrt
#!/usr/bin/perl -w
# This script will read the classification metadata for the DLCD image
# from the provided Excel spreadsheet and produce a "QGIS Color map" and
# a Mapnik RasterColorizer element (just the one XML element is written
# not the whole stylesheet).
# This script is licensed CC0 by Andrew Harvey <andrew.harvey4@gmail.com>
#
# To the extent possible under law, the person who associated CC0
# with this work has waived all copyright and related or neighboring
# rights to this work.
# http://creativecommons.org/publicdomain/zero/1.0/
use strict;
use Spreadsheet::ParseExcel::Simple;
my $xls = Spreadsheet::ParseExcel::Simple->read('downloads-unzip/Scene01-DLCDv1_Class/DLCD_Colours_Labels.xls');
my @dlcd_data;
foreach my $sheet ($xls->sheets) {
if ($sheet->sheet->get_name() eq "Sheet1") {
# consume the first header row
my @header = $sheet->next_row;
while ($sheet->has_data) {
my @data = $sheet->next_row;
push @dlcd_data, \@data;
}
}
}
print "Producing styles for:\n";
open(my $qgis_fh, ">", "stylesheets/qgis.txt") or die $!;
print " * QGIS\n";
open(my $mapnik_fh, ">", "stylesheets/DLCDv1_Class-RasterColorizer.xml") or die $!;
print " * Mapnik (XML)\n";
open(my $leaflet_legend_fh, ">", "stylesheets/leaflet-legend.js") or die $!;
print " * Leaflet (JavaScript)\n";
open(my $html_legend_fh, ">", "stylesheets/html-legend.html") or die $!;
print " * HTML\n";
print $qgis_fh "# GA DLCD Color Map\n";
print $qgis_fh "INTERPOLATION:DISCRETE\n";
# add an extra entry as transparent white for nodata values (cell value of 0)
# format of lines is: value,red,green,blue,alpha (transparency),label
print $qgis_fh "0,255,255,255,0,nodata\n";
print $html_legend_fh "<div class=\"info legend\">\n";
my %legend;
foreach my $row (@dlcd_data) {
my ($class, $red, $green, $blue, $iso_class, $label) = @{$row};
print $qgis_fh "$class,$red,$green,$blue,255,$label\n";
print $mapnik_fh "<stop value=\"$class\" color=\"rgb($red,$green,$blue)\"/>\n";
my $hex_col = sprintf ("#%.2x%.2x%.2x", $red, $green, $blue);
push @{$legend{"color"}}, $hex_col;
push @{$legend{"label"}}, $label;
print $html_legend_fh " <i style=\"background: $hex_col\"></i> $label<br>\n";
}
print $html_legend_fh "</div>";
foreach my $element ("color", "label") {
print $leaflet_legend_fh "var ${element}s = [\n";
my $num_values = scalar @{$legend{$element}};
for (my $i = 0; $i < $num_values; $i++) {
print $leaflet_legend_fh " \"$legend{$element}[$i]\"";
if ($i != $num_values - 1 ) { # not the last entry
print $leaflet_legend_fh ",";
}
print $leaflet_legend_fh "\n";
}
print $leaflet_legend_fh "];\n\n";
}
close $qgis_fh;
close $mapnik_fh;
close $leaflet_legend_fh;
close $html_legend_fh;

About

This package contains a set of tools, utilities and stylesheets for working with the Geoscience Australia National Dynamic Land Cover Dataset.

Using the Scripts

You can run the scripts individually as described in the sections below, or just as,

./scripts/00-all.sh

Dependencies

To run through all the steps provided by these script you will need,

gdal-bin, libspreadsheet-parseexcel-simple-perl, unzip

Downloading the Dataset

You can download the Reference documents and final 34 DLC classes landcover TIFF using,

./scripts/01-download.sh

If you want the TIFF's showing trends from 2000-2008 as well (adds ~1.5GB) you will need specify the first program argument as 5 (because you want to download the first 5 files from GA catalogue number 71071.

./scripts/01-download.sh 5

Unzipping

To unzip these downloads run,

./scripts/02-unzip.sh

Reproject

Next to reproject the TIFF to web mercator use,

./scripts/03-reproject.sh

Generate Stylesheets

Then finally to generate the QGIS and Mapnik stylesheets (only the RasterColorizer stop elements) run,

./scripts/04-transform-classification.pl

Loading into QGIS

To use this stylesheet in QGIS, select the properties of the tiff layer you have just opened.

Then under style use Band 1 and Colormap. Then under the Colormap tab, choose "Load color map from file" and select stylesheet/qgis.txt.

Previewing the Mapnik Style

There are many ways to use the Mapnik stylesheet. If you just want to preview it locally then liteserv may be your best option.

There is a live preview of this stylesheet.

<?xml version="1.0" encoding="utf-8"?>
<Map srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over">
<Style name="landcover">
<Rule>
<!--
Scaling options are fast or bilinear. Fast is fast, but doesn't look nice at low zooms when the image size
is reduced. Bilinear is slow, looks nicer at low zooms, however if you allow overzoom then using bilinear
(as opposed to fast) won't look nice.
It is up to you, but idealy I would like bilinear for underzoom (low zoom levels; image size reduced) and
linear for overzoom (high zoom levels; image size enlarged).
You could do this using two rules and use the min and max scale denominators.
-->
<RasterSymbolizer scaling="bilinear">
<RasterColorizer default-mode="exact" default-color="transparent" epsilon="0.001">
<!--
The following classification values were obtained from the documentation accompanying the source dataset.
The classification is © Commonwealth of Australia, (Geoscience Australia) 2011
and is part of the "The National Dynamic Land Cover Dataset" product, GA Catalogue Number: 71071.
https://www.ga.gov.au/products/servlet/controller?event=GEOCAT_DETAILS&catno=71071
It's use here is in accordance with the Creative Commons Attribution 2.5 Australia license
http://creativecommons.org/licenses/by/2.5/au/
Furthermore these step values have been automatically generated using the 04-transform-classification.pl script.
-->
<stop value="1" color="rgb(130,130,130)"/>
<stop value="2" color="rgb(0,0,0)"/>
<stop value="3" color="rgb(0,70,173)"/>
<stop value="4" color="rgb(150,225,255)"/>
<stop value="5" color="rgb(90,36,90)"/>
<stop value="6" color="rgb(166,38,170)"/>
<stop value="7" color="rgb(183,18,52)"/>
<stop value="8" color="rgb(198,141,153)"/>
<stop value="9" color="rgb(226,194,199)"/>
<stop value="10" color="rgb(219,77,105)"/>
<stop value="11" color="rgb(0,178,160)"/>
<stop value="12" color="rgb(255,255,115)"/>
<stop value="13" color="rgb(255,255,203)"/>
<stop value="14" color="rgb(255,121,0)"/>
<stop value="15" color="rgb(255,255,255)"/>
<stop value="16" color="rgb(255,255,115)"/>
<stop value="17" color="rgb(225,225,225)"/>
<stop value="18" color="rgb(255,169,82)"/>
<stop value="19" color="rgb(247,232,89)"/>
<stop value="20" color="rgb(251,206,146)"/>
<stop value="21" color="rgb(249,229,38)"/>
<stop value="22" color="rgb(255,255,203)"/>
<stop value="23" color="rgb(253,196,128)"/>
<stop value="24" color="rgb(175,136,80)"/>
<stop value="25" color="rgb(193,168,117)"/>
<stop value="26" color="rgb(125,50,40)"/>
<stop value="27" color="rgb(221,204,165)"/>
<stop value="28" color="rgb(234,170,122)"/>
<stop value="29" color="rgb(209,191,145)"/>
<stop value="30" color="rgb(140,95,70)"/>
<stop value="31" color="rgb(0,133,0)"/>
<stop value="32" color="rgb(20,194,0)"/>
<stop value="33" color="rgb(214,255,138)"/>
<stop value="34" color="rgb(186,232,96)"/>
</RasterColorizer>
</RasterSymbolizer>
</Rule>
</Style>
<Layer name="dlcd">
<StyleName>landcover</StyleName>
<Datasource>
<Parameter name="file">raster-web-mercator/DLCDv1_Class.tiff</Parameter>
<Parameter name="type">gdal</Parameter>
<Parameter name="band">1</Parameter>
</Datasource>
</Layer>
</Map>
<?xml version="1.0" encoding="utf-8"?>
<Map srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over">
<Style name="trend_evi_max">
<Rule>
<RasterSymbolizer scaling="bilinear">
<RasterColorizer default-mode="linear" default-color="transparent">
<stop value="-0.15" color="red"/>
<stop value="0" color="yellow"/>
<stop value="0.15" color="green"/>
</RasterColorizer>
</RasterSymbolizer>
</Rule>
</Style>
<Layer name="trend_evi_max">
<StyleName>trend_evi_max</StyleName>
<Datasource>
<Parameter name="file">raster-web-mercator/trend_evi_max.tiff</Parameter>
<Parameter name="type">gdal</Parameter>
<Parameter name="band">1</Parameter>
</Datasource>
</Layer>
</Map>
<?xml version="1.0" encoding="utf-8"?>
<Map srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over">
<Style name="trend_evi_mean">
<Rule>
<RasterSymbolizer scaling="bilinear">
<RasterColorizer default-mode="linear" default-color="transparent">
<stop value="-0.1" color="red"/>
<stop value="0" color="yellow"/>
<stop value="0.1" color="green"/>
</RasterColorizer>
</RasterSymbolizer>
</Rule>
</Style>
<Layer name="trend_evi_mean">
<StyleName>trend_evi_mean</StyleName>
<Datasource>
<Parameter name="file">raster-web-mercator/trend_evi_mean.tiff</Parameter>
<Parameter name="type">gdal</Parameter>
<Parameter name="band">1</Parameter>
</Datasource>
</Layer>
</Map>
<?xml version="1.0" encoding="utf-8"?>
<Map srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over">
<Style name="trend_evi_min">
<Rule>
<RasterSymbolizer scaling="bilinear">
<RasterColorizer default-mode="linear" default-color="transparent">
<stop value="-0.1" color="red"/>
<stop value="0" color="yellow"/>
<stop value="0.1" color="green"/>
</RasterColorizer>
</RasterSymbolizer>
</Rule>
</Style>
<Layer name="trend_evi_min">
<StyleName>trend_evi_min</StyleName>
<Datasource>
<Parameter name="file">raster-web-mercator/trend_evi_min.tiff</Parameter>
<Parameter name="type">gdal</Parameter>
<Parameter name="band">1</Parameter>
</Datasource>
</Layer>
</Map>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment