| USR_LOCAL="/usr/local" | |
| USR="/usr" | |
| TIFF_HOME=$USR_LOCAL | |
| LASZIP_HOME=$USR_LOCAL | |
| LIBXML2_HOME=/usr/local/Cellar/libxml2/2.9.4/ | |
| WEBSOCKETPP_HOME=/Users/hobu/dev/git/websocketpp | |
| GEOTIFF_HOME=$USR_LOCAL | |
| P2G_HOME=$USR_LOCAL | |
| HEXER_HOME="/usr/local" | |
| SQLITE_HOME="/usr/local/opt/sqlite" |
| USR_LOCAL="/usr/local" | |
| USR="/usr" | |
| TIFF_HOME=$USR_LOCAL | |
| LASZIP_HOME=$USR_LOCAL | |
| LIBXML2_HOME=/usr/local/Cellar/libxml2/2.9.4/ | |
| WEBSOCKETPP_HOME=/Users/hobu/dev/git/websocketpp | |
| GEOTIFF_HOME=$USR_LOCAL | |
| P2G_HOME=$USR_LOCAL | |
| HEXER_HOME="/usr/local" | |
| SQLITE_HOME="/usr/local/opt/sqlite" |
| object Main { | |
| def main(args: Array[String]): Unit = { | |
| val bucket = "foo" | |
| val prefix = "bar" | |
| // Read all geotiffs in s3://foo/bar | |
| // but these are large geotiffs. | |
| // I choose to read them as 1024x1024 tiles | |
| def loadFromS3(bucket: String, prefix: String, partitionCount: Int, dimensions: (Int, Int))(implicit sc: SparkContext): RDD[(ProjectedExtent, Tile)] = { |
| import os | |
| import csv | |
| path = "gwgm-ca-run-results-initial-geolife-Sept18.csv" | |
| fields = ["uuid","duration","system","timeAtFirstResult","clusterId","startTime","endTime","result","queryName"] | |
| # clusterIds = ['j-2XPAUZO5DFFGZ', 'j-2D7LTJ7YTZSVR'] | |
| # clusterIds = ['j-224ANWOQ9RL3V', 'j-3G1WOHPKBA7HO'] | |
| clusterIds = ['j-11FRN8ESB4T4O', |
| ==Running tests against France polygon and BBOX, for six months and one month, over whole datasets== | |
| Both returned the same number of results for all non-loose queries. | |
| RUNNING FRANCE-BBOX-ONE-MONTH 1980 | |
| GDELT-IN-FRANCE-BBOX-ONE-MONTH-1980-JAN GW = 384 GM = 124 RESULTS = 0 MMMMMMMMMMMMMMMM | |
| GDELT-IN-FRANCE-BBOX-ONE-MONTH-1980-FEB GW = 366 GM = 78 RESULTS = 0 MMMMMMMMMMMMMMMM | |
| GDELT-IN-FRANCE-BBOX-ONE-MONTH-1980-MAR GW = 346 GM = 78 RESULTS = 0 MMMMMMMMMMMMMMMM | |
| GDELT-IN-FRANCE-BBOX-ONE-MONTH-1980-APR GW = 393 GM = 80 RESULTS = 2 MMMMMMMMMMMMMMMM |
| class ShardingKeyIndex(inner: KeyIndex[K], shardCount: Int, maxBytes: Int) extends KeyIndex[K] { | |
| def combine(shardBytes: Array[Byte], indexBytes: Array[Byte]): Array[Byte] = { | |
| // Combine the shard bytes and the index bytes to something, not longer than 64 bytes | |
| ??? | |
| } | |
| def prefixWithShard(i: Long, shard: Int): Long = { | |
| val iAsBytes = toBytes(i) | |
| val shardAsBytes = toBytes(shard) | |
| combine(shardAsBytes, iAsBytes, shardCount, maxBytes).toLong |
by Annie Wilson
the local acro-theater group that Rob toured to Mexico with! A space-age retelling of the story of Io, accompanied by a local folk duo.
a prolific and well-respected American choreographer. I've never seen his work and I'm excited to finally get to it (if it's not sold out).
| import com.amazonaws.auth.{DefaultAWSCredentialsProviderChain, AWSCredentials, AWSCredentialsProvider} | |
| import com.amazonaws.services.elasticmapreduce._ | |
| import com.amazonaws.services.elasticmapreduce.model._ | |
| import scala.collection.JavaConverters._ | |
| case class ClusterDefinition(masterDns: String, instanceType: String, count: Int) | |
| object Emr { | |
| val credentials = new DefaultAWSCredentialsProviderChain() |
| // 1. Implement window reading as a "crop" operation: rename mutable(e: Extent) => crop(e: Extent) | |
| // 2. Add logic to geotrellis.raster.crop logic to account for GeoTiffTile having it's own crop mechanism | |
| object Foo { | |
| def crop(gb: GridBounds, options: Options): Tile = { | |
| val cropBounds = | |
| if(options.clamp) | |
| gb.intersection(self) match { | |
| case Some(intersection) => intersection |
| val pointFeatures: RDD[PointFeature[Double]] = ??? | |
| val layoutDefinition: LayoutDefinition = ??? // The target layout of raster tiles | |
| val keyed: RDD[(SpatialKey, PointFeature[Double])] = | |
| pointFeatures.flatMap { pf => | |
| val centerKey @ SpatialKey(col, row) = layoutDefinition.mapTransform(pf.geom) | |
| Seq( | |
| (centerKey, pf), | |
| (SpatialKey(col - 1, row), pf), // Left | |
| (SpatialKey(col + 1, row), pf), // Right | |
| (SpatialKey(col, row - 1), pf), // Upper |