lossyrob

package geotrellis.spark.io.accumulo

import geotrellis.spark._
import geotrellis.spark.io._

import spray.json._

import scala.collection.JavaConversions._

import org.apache.spark.Logging

lossyrob

### INGEST GEOTIFFS INTO HDFS ###

# geotrellis-spark JAR. Shouldn't have to change this one if running in the root folder (remember to run ./sbt "project spark" assembly)
JAR=spark/target/scala-2.10/geotrellis-spark-assembly-0.10.0-SNAPSHOT.jar 

# Amount of memory for the driver
DRIVER_MEMORY=3G

# Amount of memory per executor. If in local mode, change the DRIVER_MEMORY instead.
EXECUTOR_MEMORY=512G

lossyrob

import geotrellis.raster._
import geotrellis.vector._
import geotrellis.spark._
import geotrellis.spark.io.accumulo._

import org.apache.accumulo.core.client.security.tokens.PasswordToken

implicit val _sc = sc

// Get accumulo instance and the catalog.

lossyrob

### INGEST SPATIAL GEOTIFF IN ACCUMULO ###

# geotrellis-spark JAR. Shouldn't have to change this one if running in the root folder (remember to run ./sbt "project spark" assembly)
JAR=/Users/rob/proj/gt/geotrellis/spark/target/scala-2.10/geotrellis-spark-assembly-0.10.0-SNAPSHOT.jar 

# Directory with the input tiled GeoTIFF's 
INPUT=file:/Users/rob/proj/climate/data/one-month-tiles

# Table to store tiles
TABLE=nexmonth

lossyrob

package geotrellis.spark.io.hadoop

import geotrellis.spark._
import geotrellis.raster._

// "com.google.uzaygezen" % "uzaygezen-core" % "0.2"
import com.google.uzaygezen.core.CompactHilbertCurve
import com.google.uzaygezen.core.MultiDimensionalSpec
import com.google.uzaygezen.core.BitVector
import com.google.uzaygezen.core.BitVectorFactories

lossyrob

package geotrellis.benchmark

import geotrellis.vector._
import geotrellis.raster._
import geotrellis.raster.op.local._

import com.google.caliper.Param

import scala.util.Random
import scala.annotation.tailrec

lossyrob

package geotrellis.raster.io.geotiff.reader.decompression

import geotrellis.raster.io.geotiff.reader._
import geotrellis.raster.io.geotiff.reader.tags.Tags

import java.nio.{ByteBuffer, ByteOrder}

import monocle.syntax._

import spire.syntax.cfor._

lossyrob

Bit rasters

[info] Running geotrellis.benchmark.ReadAspectBitUncompressedStriped 
[info]  0% Scenario{vm=java, trial=0, benchmark=NativeReadAspectTif} 4605736.18 ns; σ=183873.35 ns @ 10 trials
[info] 50% Scenario{vm=java, trial=0, benchmark=NewReadAspectTif} 19903200.00 ns; σ=128275.29 ns @ 3 trials
[info] 
[info]           benchmark    ms linear runtime
[info] NativeReadAspectTif  4.61 ======
[info]    NewReadAspectTif 19.90 ==============================

lossyrob

/*
 * Copyright (c) 2014 Azavea.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software

lossyrob

from collections import namedtuple
import re, sys
import csv
import os
from tempfile import mkdtemp
from zipfile import ZipFile
import os.path

def parse_file(filepath):