jhpoelen

#!/bin/bash
#
# streams Wikidata taxon items (or items containing https://www.wikidata.org/wiki/Q16521)
# from latest data dump in line json (one json object per line)
#

curl --silent "https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.bz2"\
| bunzip2\
| grep -E "Q16521[^0-9]"\
| sed 's/,$//g'\

jhpoelen

OPEN_ALEX_VERSION=hash://sha256/f19011fe72234cb22f4326fcc95752647a11628cdaa0d24f7d55033b06cb5653

REMOTES=https://biokic6.rc.asu.edu/preston/openalex,https://linker.bio\


preston ls\
 --anchor "${OPEN_ALEX_VERSION}"\
 --remote "${REMOTES}"\
 | grep hasVersion\
 | grep -v csv\

jhpoelen

#!/bin/bash
#
# Find publications (aka "works") that mention "Bombus" (bumblebees) in a versioned copy of OpenAlex.
#
# Requirements:
#   preston - https://github.com/bio-guoda/preston
#   grep (comes with linux distro)
#   gunzip (comes with linux distro)
#   jq - https://jqlang.github.io/jq/
#   mlr - https://miller.readthedocs.io/

jhpoelen

#!/bin/bash
#
#
curl "https://depot.globalbioticinteractions.org/snapshot/target/data/tsv/datasets.tsv"\
| awk '{ print "https://depot.globalbioticinteractions.org/reviews/" $1 "/indexed-interactions.tsv" }'\
| xargs -L1 curl\
| xargs -L1 DiscretePowerLawfitter.sh\
> angel-review-of-globi-datasets.tsv

jhpoelen

#!/bin/bash
#
# related to https://github.com/Big-Bee-Network/UCSB-IZC00012194
#

preston ls\
| grep hasVersion\
| preston grep 'UCSB-IZC00012194.*body\ssize' --log tsv\
| grep value\
| cut -f1

jhpoelen

#!/bin/bash
#
# prerequisites
# * preston https://github.com/bio-guoda/preston
# * pv pipeviewer https://linux.die.net/man/1/pv
# * mlr https://miller.readthedocs.io/en/6.7.0/
#
# executed/tested on 22.04.1-Ubuntu
# 

jhpoelen

#!/bin/bash
#
# 2020-10-15
#
# This script is a way to select pollination and flower visits record from 
# one of the data products provided via https://globalbioticinteractions.org/data . 
#
# This particular example uses a July 2020 data publication. 
#
# For more recent data, see https://globalbioticinteractions.org/data . 

jhpoelen

Script used for counting records / taxa

Attached big-bee-globi-stats.log was generated on 2020-09-25 using latest interactions.tsv.gz with

$ sha256sum interactions.tsv.gz 
436f1249dc71bc948483bac0d6f13c667e9d69456ef727037637516468e9d29d

To reproduce:

jhpoelen

prepare_ebird_2018_id <- function() {
  ebird_data_location <- "http://ebirddata.ornith.cornell.edu/downloads/gbiff/dwca-1.0.zip"
  # unfortunately, the eBird URL no longer work, but, 
  # using a time machine, we went back in time and republished data via Zenodo from 2018
  ebird_data_location <- "https://zenodo.org/record/3858251/files/dwca-1.0.zip"

  ebird_data_id <- contentid::register(ebird_data_location)
  ebird_data_id
}

jhpoelen

#!/bin/bash
#
#

WKT_STRING="POLYGON ((-72.77293810620904 -33.196074154826235, -72.77293810620904 6.59516197881252, -28.12450060620904 6.59516197881252, -28.12450060620904 -33.196074154826235, -72.77293810620904 -33.196074154826235))"

spark-submit \
  --master mesos://zk://mesos01:2181,mesos02:2181,mesos03:2181/mesos \
  --driver-memory 4G \
  --conf spark.sql.caseSensitive=true \