idigbio-spark scripts

create-checklist-cluster.sh

#!/bin/bash
#
#

WKT_STRING="POLYGON ((-72.77293810620904 -33.196074154826235, -72.77293810620904 6.59516197881252, -28.12450060620904 6.59516197881252, -28.12450060620904 -33.196074154826235, -72.77293810620904 -33.196074154826235))"

spark-submit \
  --master mesos://zk://mesos01:2181,mesos02:2181,mesos03:2181/mesos \
  --driver-memory 4G \
  --conf spark.sql.caseSensitive=true \
  --class ChecklistGenerator \
  iDigBio-LD-assembly-1.5.9.jar \
  -c hdfs:///guoda/data/source=preston.acis.ufl.edu/dwca/core.parquet \
  -t hdfs:///guoda/data/traits \
  -o hdfs:///user/$USER/checklist \
  "Anas|Anura" \
  "$WKT_STRING" \
  ""

create-checklist-local.sh

#!/bin/bash
#
#

WKT_STRING="POLYGON ((-72.77293810620904 -33.196074154826235, -72.77293810620904 6.59516197881252, -28.12450060620904 6.59516197881252, -28.12450060620904 -33.196074154826235, -72.77293810620904 -33.196074154826235))"

spark-submit \
  --driver-memory 4G \
  --conf spark.sql.caseSensitive=true \
  --class ChecklistGenerator \
  iDigBio-LD-assembly-1.5.9.jar \
  -c file:///$HOME/preston-amazon/data-processed/core.parquet \
  -t file:///$HOME/traits \
  -o file:///$HOME/checklist \
  "Animalia" \
  "$WKT_STRING" \
  ""

preston2parquet.sh

#!/bin/bash
#
#

spark-submit \
  --driver-memory 4G \
  --conf spark.sql.caseSensitive=true \
  --class  bio.guoda.preston.spark.PrestonUtil \
  iDigBio-LD-assembly-1.5.9.jar \
  file:///$HOME/preston-amazon/data \
  file:///$HOME/preston-amazon/data-processed