GBIF_azure_stats.scala

import org.apache.spark.sql.functions._
val wasbs_path = "wasbs://gbif@ai4edataeuwest.blob.core.windows.net/occurrence/20210413/occurrence.parquet/*"

val df = spark.read.parquet(wasbs_path)

// Number species total
df.select("specieskey").distinct().count()
// Number species by Kingdom
df.select("kingdom","specieskey").distinct().groupBy("kingdom").count().orderBy(desc("count")).show()
// Number records total
df.count()
// Number records by Kingdom
df.groupBy("kingdom").count().orderBy(desc("count")).show()
// Number of datasets
df.select("datasetkey").distinct().count()
// Number of publishers
df.select("publishingorgkey").distinct().count()
// Number of observations, specimens etc
df.groupBy("basisofrecord").count().orderBy(desc("count")).show()

Thanks John