Skip to content

Instantly share code, notes, and snippets.

@moradology

moradology/whatever.json

Created July 18, 2018 21:04
Show Gist options
  • Save moradology/44ff7958437a611c7049324eaf541029 to your computer and use it in GitHub Desktop.
Save moradology/44ff7958437a611c7049324eaf541029 to your computer and use it in GitHub Desktop.
Download ZIP
json
Raw
whatever.json
{"paragraphs":[{"text":"import com.azavea._\n\nimport geotrellis.raster._\nimport geotrellis.spark._\nimport geotrellis.spark.io._\nimport geotrellis.spark.io.cog.COGLayerStorageMetadata\nimport geotrellis.spark.io.s3._\nimport geotrellis.spark.io.s3.cog._\nimport geotrellis.vector.Extent\nimport cats.effect.IO\nimport cats.implicits._\nimport com.amazonaws.services.s3.AmazonS3URI\nimport spire.syntax.cfor._\nimport org.apache.spark.SparkContext\nimport scala.util.Try\n\nimport java.util.concurrent.Executors\nimport scala.concurrent.ExecutionContext\n\n","user":"anonymous","dateUpdated":"2018-07-18T19:44:34+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nimport com.azavea._\n\nimport geotrellis.raster._\n\nimport geotrellis.spark._\n\nimport geotrellis.spark.io._\n\nimport geotrellis.spark.io.cog.COGLayerStorageMetadata\n\nimport geotrellis.spark.io.s3._\n\nimport geotrellis.spark.io.s3.cog._\n\nimport geotrellis.vector.Extent\n\nimport cats.effect.IO\n\nimport cats.implicits._\n\nimport com.amazonaws.services.s3.AmazonS3URI\n\nimport spire.syntax.cfor._\n\nimport org.apache.spark.SparkContext\n\nimport scala.util.Try\n\nimport java.util.concurrent.Executors\n\nimport scala.concurrent.ExecutionContext\n"}]},"apps":[],"jobName":"paragraph_1531940306467_527520497","id":"20180718-185826_1443777371","dateCreated":"2018-07-18T18:58:26+0000","dateStarted":"2018-07-18T19:44:34+0000","dateFinished":"2018-07-18T19:45:13+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:318"},{"text":"val bench = new Bench { }\nimport bench._","user":"anonymous","dateUpdated":"2018-07-18T19:45:18+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nbench: com.azavea.Bench = $$$$e78674de3c900ed66c46641937e0d6$$$$anon$1@a4783a7\n\nimport bench._\n"}]},"apps":[],"jobName":"paragraph_1531940551273_826422775","id":"20180718-190231_1268988488","dateCreated":"2018-07-18T19:02:31+0000","dateStarted":"2018-07-18T19:45:18+0000","dateFinished":"2018-07-18T19:45:34+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:319"},{"text":"\n def availableZoomLevels(path: String)(name: String): List[Int] = {\n val s3Path = new AmazonS3URI(path)\n val attributeStore = S3AttributeStore(s3Path.getBucket, s3Path.getKey)\n attributeStore.availableZoomLevels(name).toList\n }\n\n def runValueReader(path: String)(name: String, zoomLevels: List[Int], extent: Option[Extent] = None, threads: Int = 16, targetBands: Option[Seq[Int]] = None)(implicit sc: SparkContext): List[(Long, Long)] = {\n val pool = Executors.newFixedThreadPool(threads)\n implicit val ec = ExecutionContext.fromExecutor(pool)\n\n val s3Path = new AmazonS3URI(path)\n val attributeStore = S3AttributeStore(s3Path.getBucket, s3Path.getKey)\n\n val COGLayerStorageMetadata(cogLayerMetadata, _) = attributeStore.readMetadata[COGLayerStorageMetadata[SpatialKey]](LayerId(name, 0))\n\n val layersData: List[(LayerId, KeyBounds[SpatialKey], TileLayerMetadata[SpatialKey])] =\n zoomLevels\n .map(LayerId(name, _))\n .map { layerId =>\n val metadata = cogLayerMetadata.tileLayerMetadata(layerId.zoom)\n (layerId, metadata.bounds match { case kb: KeyBounds[SpatialKey] => kb }, metadata)\n }\n\n val valueReader = new S3COGValueReader(attributeStore)\n\n val res: IO[List[(Long, Long)]] =\n layersData\n .map { case (layerId, kb, metadata) =>\n IO.shift(ec) *> IO {\n val gb @ GridBounds(minCol, minRow, maxCol, maxRow) = extent.map(metadata.mapTransform.extentToBounds).getOrElse(kb.toGridBounds)\n val reader = valueReader.reader[SpatialKey, MultibandTile](layerId)\n val buf: scala.collection.mutable.ArrayBuffer[SpatialKey] = scala.collection.mutable.ArrayBuffer()\n cfor(minCol)(_ < maxCol, _ + 1) { col =>\n cfor(minRow)(_ < maxRow, _ + 1) { row =>\n buf += SpatialKey(col, row)\n }\n } \n targetBands match {\n case Some(bands) => IO {\n val (time, _) = timedC reateLong {\n buf.toList.parTraverse { sk => IO { reader.readSubsetBands(sk, bands) }.handleErrorWith { error => println(\"I failed\"); IO.pure(Array.empty) } }.unsafeRunSync\n }\n\n (time, gb.size)\n case None => IO {\n val (time, _) = timedCreateLong {\n buf.toList.parTraverse { sk => IO { reader.read(sk).some }.handleErrorWith { error => IO.pure { None } } }.unsafeRunSync\n }\n\n (time, gb.size)\n }\n }\n }\n }.parSequence\n\n res.unsafeRunSync()\n }","user":"anonymous","dateUpdated":"2018-07-18T20:49:49+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala","lineNumbers":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\navailableZoomLevels: (path: String)(name: String)List[Int]\n\n\n\n\n<console>:427: warning: match may not be exhaustive.\nIt would fail on the following input: EmptyBounds\n (layerId, metadata.bounds match { case kb: KeyBounds[SpatialKey] => kb }, metadata)\n ^\n\nrunValueReader: (path: String)(name: String, zoomLevels: List[Int], extent: Option[geotrellis.vector.Extent], threads: Int, targetBands: Option[Seq[Int]])(implicit sc: org.apache.spark.SparkContext)List[(Long, Long)]\n"}]},"apps":[],"jobName":"paragraph_1531940643233_1742680117","id":"20180718-190403_2051636321","dateCreated":"2018-07-18T19:04:03+0000","dateStarted":"2018-07-18T20:42:26+0000","dateFinished":"2018-07-18T20:42:33+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:320"},{"text":"val multibandValueExt = Extent(-7386874.413479432, 2015491.5618235283, -7308602.896515412, 2093763.0787875466).some","user":"anonymous","dateUpdated":"2018-07-18T19:45:45+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nmultibandValueExt: Option[geotrellis.vector.Extent] = Some(Extent(-7386874.413479432, 2015491.5618235283, -7308602.896515412, 2093763.0787875466))\n"}]},"apps":[],"jobName":"paragraph_1531940948751_1638057420","id":"20180718-190908_884489917","dateCreated":"2018-07-18T19:09:08+0000","dateStarted":"2018-07-18T19:45:45+0000","dateFinished":"2018-07-18T19:45:49+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:321"},{"text":"val cogBandLayer = \"cog-multiband-band-layer\"","user":"anonymous","dateUpdated":"2018-07-18T19:48:39+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\ncogBandLayer: String = cog-multiband-band-layer\n"}]},"apps":[],"jobName":"paragraph_1531941081356_1657061713","id":"20180718-191121_2040184142","dateCreated":"2018-07-18T19:11:21+0000","dateStarted":"2018-07-18T19:48:39+0000","dateFinished":"2018-07-18T19:48:43+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:322"},{"text":" val cogPath = \"s3://geotrellis-test/daunnc/cog-benchmark/cog-3\"","user":"anonymous","dateUpdated":"2018-07-18T19:48:49+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\ncogPath: String = s3://geotrellis-test/daunnc/cog-benchmark/cog-3\n"}]},"apps":[],"jobName":"paragraph_1531941249069_-1261802142","id":"20180718-191409_1541060757","dateCreated":"2018-07-18T19:14:09+0000","dateStarted":"2018-07-18T19:48:49+0000","dateFinished":"2018-07-18T19:48:53+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:323"},{"text":"runValueReader(cogPath)(name = cogBandLayer, extent = multibandValueExt, zoomLevels = List(13), targetBands = Seq(0, 1, 3, 2, 123123113, 8, 9, 4).some)(sc)","user":"anonymous","dateUpdated":"2018-07-18T20:42:39+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1531941100988_-1024421938","id":"20180718-191140_225969885","dateCreated":"2018-07-18T19:11:40+0000","dateStarted":"2018-07-18T20:42:39+0000","dateFinished":"2018-07-18T20:37:18+0000","status":"RUNNING","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:324"},{"user":"anonymous","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{}},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1531941315124_-2004074715","id":"20180718-191515_1699988980","dateCreated":"2018-07-18T19:15:15+0000","status":"READY","progressUpdateIntervalMs":500,"$$hashKey":"object:325"}],"name":"Untitled Note 2","id":"2DJ9DEUF5","angularObjects":{"2BRWU4WXC:shared_process":[],"2AM1YV5CU:shared_process":[],"2AJXGMUUJ:shared_process":[],"2ANGGHHMQ:shared_process":[],"2AKK3QQXU:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment