Skip to content

Instantly share code, notes, and snippets.

@wnina
Last active May 17, 2016 15:51
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save wnina/727734af3324d00767c6520d3edccb95 to your computer and use it in GitHub Desktop.
Save wnina/727734af3324d00767c6520d3edccb95 to your computer and use it in GitHub Desktop.
package tutorial
import geotrellis.raster._
import geotrellis.raster.io.geotiff._
import geotrellis.raster.render._
import geotrellis.raster.resample._
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.file._
import geotrellis.spark.io.avro.codecs._
import geotrellis.vector._
import akka.actor._
import akka.io.IO
import spray.can.Http
import spray.routing.{HttpService, RequestContext}
import spray.routing.directives.CachingDirectives
import spray.http.MediaTypes
import scala.concurrent._
import com.typesafe.config.ConfigFactory
// more
import geotrellis.spark.io.hadoop._
import org.apache.hadoop.fs.Path
import org.apache.hadoop.conf.Configuration
import org.apache.spark._
import geotrellis.spark.util.SparkUtils
object ServeNDVI {
//val catalogPath = new java.io.File("data/catalog").getAbsolutePath
val inputPath = "hdfs://master:9000/datalandsat"
val rootPath = new Path(inputPath)
val hadoopConfiguration = new Configuration
val conf = new SparkConf()
.setMaster("spark://master:7077")
.setAppName("Spark Tunki - Serve")
implicit val sc = new SparkContext(conf)
val hadoopAttributeStore = new HadoopAttributeStore(rootPath, hadoopConfiguration)
// Create a reader that will read in the indexed tiles we produced in IngestImage.
val hadoopValueReader = HadoopValueReader(hadoopAttributeStore)
//val fileValueReader = FileValueReader(catalogPath)
def reader(layerId: LayerId) = hadoopValueReader.reader[SpatialKey, MultibandTile](layerId)
def main(args: Array[String]): Unit = {
implicit val system = akka.actor.ActorSystem("tutorial-system")
println(">>>>>> inside main, begin")
// create and start our service actor
val service =
system.actorOf(Props(classOf[NDVIServiceActor]), "tutorial")
// start a new HTTP server on port 8080 with our service actor as the handler
IO(Http) ! Http.Bind(service, "localhost", 8082)
println(">>>>>> inside main, end")
}
}
class NDVIServiceActor extends Actor with HttpService {
import scala.concurrent.ExecutionContext.Implicits.global
def actorRefFactory = context
def receive = runRoute(root)
val colorMap =
ColorMap.fromStringDouble(ConfigFactory.load().getString("tutorial.colormap")).get
def root =
pathPrefix(IntNumber / IntNumber / IntNumber) { (zoom, x, y) =>
respondWithMediaType(MediaTypes.`image/png`) {
complete {
println(">>>>>> inside complete")
future {
// Read in the tile at the given z/x/y coordinates.
val tileOpt: Option[MultibandTile] =
try {
println(">>>> zoom: " + zoom + "x: "+ x + "y: "+ y)
Some(ServeNDVI.reader(LayerId("nlcd-tms",zoom)).read(x, y))
} catch {
case _: TileNotFoundError =>
None
}
tileOpt.map { tile =>
// Compute the NDVI
val ndvi =
tile.convert(DoubleConstantNoDataCellType).combineDouble(0, 1) { (r, ir) =>
if(isData(r) && isData(ir)) {
(ir - r) / (ir + r)
} else {
Double.NaN
}
}
// Render as a PNG
ndvi.renderPng(colorMap).bytes
}
}
}
}
}
}
@wnina
Copy link
Author

wnina commented May 16, 2016

The script that use to ingest was:

./spark-submit --class geotrellis.spark.etl.MultibandIngest --master spark://master:7077 --driver-memory 2G  /opt/geotrellis-github/new-geotrellis/geotrellis/spark-etl/target/scala-2.10/geotrellis-spark-etl-assembly-0.10.0-SNAPSHOT.jar --input hadoop --format multiband-geotiff --cache NONE -I path="file:///opt/spark-1.6.0-bin-hadoop2.6/datalandsat" --output hadoop -O path="hdfs://master:9000/datalandsat/" --layer nlcd-tms --crs EPSG:3857 --pyramid --layoutScheme tms

@wnina
Copy link
Author

wnina commented May 16, 2016

The error is:

[info] >>>>>> inside complete
[info] >>>> zoom: 8x: 76y: 138
[info] [ERROR] [05/16/2016 15:52:16.905] [ForkJoinPool-4-worker-1] [akka://tutorial-system/user/tutorial] Error during processing of request HttpRequest(GET,http://localhost:8082/8/79/138,List(Host: localhost:8082, User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0, Accept-Language: en-US, en;q=0.5, Accept-Encoding: gzip, deflate, Connection: keep-alive, Pragma: no-cache, Cache-Control: no-cache),Empty,HTTP/1.1)
[info] java.lang.UnsupportedOperationException: empty collection
[info]  at org.apache.spark.rdd.RDD$$anonfun$first$1.apply(RDD.scala:1330)
[info]  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
[info]  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
[info]  at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
[info]  at org.apache.spark.rdd.RDD.first(RDD.scala:1327)
[info]  at geotrellis.spark.io.hadoop.HadoopValueReader$$anon$1.read(HadoopValueReader.scala:54)
[info]  at tutorial.NDVIServiceActor$$anonfun$root$1$$anonfun$apply$1$$anonfun$apply$2.apply(ServeNDVI.scala:85)
[info]  at tutorial.NDVIServiceActor$$anonfun$root$1$$anonfun$apply$1$$anonfun$apply$2.apply(ServeNDVI.scala:79)
[info]  at scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
[info]  at scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
[info]  at scala.concurrent.impl.ExecutionContextImpl$$anon$3.exec(ExecutionContextImpl.scala:107)
[info]  at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
[info]  at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
[info]  at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
[info]  at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
[info] 

@wnina
Copy link
Author

wnina commented May 16, 2016

The output of json metadata (nlcd-tms___0___metadata.json and nlcd-tms___15___metadata.json) and when i ingested are:

[{"name":"nlcd-tms","zoom":0},{"header":{"format":"hdfs","keyClass":"geotrellis.spark.SpatialKey","valueClass":"geotrellis.raster.MultibandTile","path":"hdfs://master:9000/datalandsat/nlcd-tms/0"},"metadata":{"extent":{"xmin":-7931295.914632329,"ymin":-1812178.4956779396,"xmax":-7904412.241232644,"ymax":-1785130.8165442757},"layoutDefinition":{"extent":{"xmin":-20037508.342789244,"ymin":-20037508.342789244,"xmax":20037508.342789244,"ymax":20037508.342789244},"tileLayout":{"layoutCols":1,"layoutRows":1,"tileCols":256,"tileRows":256}},"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":0,"row":0}},"cellType":"int32","crs":"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs "},"keyIndex":{"type":"zorder","properties":{"keyBounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":0,"row":0}}}},"schema":{"type":"record","name":"KeyValueRecord","namespace":"geotrellis.spark.io","fields":[{"name":"pairs","type":{"type":"array","items":{"type":"record","name":"Tuple2","namespace":"scala","fields":[{"name":"_1","type":{"type":"record","name":"SpatialKey","namespace":"geotrellis.spark","fields":[{"name":"col","type":"int"},{"name":"row","type":"int"}]}},{"name":"_2","type":{"type":"record","name":"ArrayMultibandTile","namespace":"geotrellis.raster","fields":[{"name":"bands","type":{"type":"array","items":[{"type":"record","name":"ByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":-128}]},{"type":"record","name":"FloatArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"float"}},{"name":"noDataValue","type":["boolean","float"],"default":true}]},{"type":"record","name":"DoubleArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"double"}},{"name":"noDataValue","type":["boolean","double"],"default":true}]},{"type":"record","name":"ShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-32768}]},{"type":"record","name":"IntArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-2147483648}]},{"type":"record","name":"BitArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"}]},{"type":"record","name":"UByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":0}]},{"type":"record","name":"UShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":0}]}]}}]}}]}}}]}}]

[{"name":"nlcd-tms","zoom":15},{"header":{"format":"hdfs","keyClass":"geotrellis.spark.SpatialKey","valueClass":"geotrellis.raster.MultibandTile","path":"hdfs://master:9000/datalandsat/nlcd-tms/15"},"metadata":{"extent":{"xmin":-7931295.914632329,"ymin":-1812178.4956779396,"xmax":-7904412.241232644,"ymax":-1785130.8165442757},"layoutDefinition":{"extent":{"xmin":-20037508.342789244,"ymin":-20037508.342789244,"xmax":20037508.342789244,"ymax":20037508.342789244},"tileLayout":{"layoutCols":32768,"layoutRows":32768,"tileCols":256,"tileRows":256}},"bounds":{"minKey":{"col":9898,"row":17843},"maxKey":{"col":9920,"row":17865}},"cellType":"int32","crs":"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs "},"keyIndex":{"type":"zorder","properties":{"keyBounds":{"minKey":{"col":9898,"row":17843},"maxKey":{"col":9920,"row":17865}}}},"schema":{"type":"record","name":"KeyValueRecord","namespace":"geotrellis.spark.io","fields":[{"name":"pairs","type":{"type":"array","items":{"type":"record","name":"Tuple2","namespace":"scala","fields":[{"name":"_1","type":{"type":"record","name":"SpatialKey","namespace":"geotrellis.spark","fields":[{"name":"col","type":"int"},{"name":"row","type":"int"}]}},{"name":"_2","type":{"type":"record","name":"ArrayMultibandTile","namespace":"geotrellis.raster","fields":[{"name":"bands","type":{"type":"array","items":[{"type":"record","name":"ByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":-128}]},{"type":"record","name":"FloatArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"float"}},{"name":"noDataValue","type":["boolean","float"],"default":true}]},{"type":"record","name":"DoubleArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"double"}},{"name":"noDataValue","type":["boolean","double"],"default":true}]},{"type":"record","name":"ShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-32768}]},{"type":"record","name":"IntArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":-2147483648}]},{"type":"record","name":"BitArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"}]},{"type":"record","name":"UByteArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":"bytes"},{"name":"noDataValue","type":["int","null"],"default":0}]},{"type":"record","name":"UShortArrayTile","fields":[{"name":"cols","type":"int"},{"name":"rows","type":"int"},{"name":"cells","type":{"type":"array","items":"int"}},{"name":"noDataValue","type":["int","null"],"default":0}]}]}}]}}]}}}]}}]

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment