get_openneuro_subjcnt.bash

#!/usr/bin/env bash
#
# 20200414WF - init
#    get all neuro study subjects by cursor
#


CURSORS=(
  "eyJvZmZzZXQiOjI1fQ==" \
  "eyJvZmZzZXQiOjEwMH0=" \
  "eyJvZmZzZXQiOjEyNX0=" \
  "eyJvZmZzZXQiOjE1MH0=" \
  "eyJvZmZzZXQiOjE3NX0=" \
  "eyJvZmZzZXQiOjIwMH0=" \
  "eyJvZmZzZXQiOjIyNX0=" \
  "eyJvZmZzZXQiOjI1MH0=" \
  "eyJvZmZzZXQiOjI3NX0=" \
  "eyJvZmZzZXQiOjMwMH0=" \
  "eyJvZmZzZXQiOjMyNX0=" \
  "eyJvZmZzZXQiOjM1MH0=" \
  "eyJvZmZzZXQiOjM3NX0=" \
  "eyJvZmZzZXQiOjQwMH0=" \
  "eyJvZmZzZXQiOjQyNX0=" \
  "eyJvZmZzZXQiOjQ1MH0=" \
  "eyJvZmZzZXQiOjQ3NX0=" \
  "eyJvZmZzZXQiOjUwMH0=" \
  "eyJvZmZzZXQiOjUyNX0=" \
  "eyJvZmZzZXQiOjU1MH0=" \
  "eyJvZmZzZXQiOjU3NX0=" \
  "eyJvZmZzZXQiOjYwMH0=" \
  "eyJvZmZzZXQiOjYyNX0=" \
  "eyJvZmZzZXQiOjY1MH0=" \
  "eyJvZmZzZXQiOjY3NX0=" \
  "eyJvZmZzZXQiOjcwMH0=" \
  "eyJvZmZzZXQiOjcyNX0=" \
  "eyJvZmZzZXQiOjc1MH0=" \
  "eyJvZmZzZXQiOjc3NX0=" \
  "eyJvZmZzZXQiOjgwMH0=" \
  "eyJvZmZzZXQiOjgyNX0=" \
  "eyJvZmZzZXQiOjg1MH0=" \
)
curlme (){
curl 'https://openneuro.org/crn/graphql' -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:75.0) Gecko/20100101 Firefox/75.0' -H 'Accept: */*' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/json' -H 'Origin: https://openneuro.org' -H 'Referer: https://openneuro.org/public/datasets' -H 'Connection: keep-alive' -H 'TE: Trailers' --data \
   '{"operationName":"getDatasets","variables":{"orderBy":{"created":"descending"},"filterBy":{"public":true},"myDatasets":false,"cursor":"'$1'"},"query":"query getDatasets($cursor: String, $orderBy: DatasetSort = {created: descending}, $filterBy: DatasetFilter = {}, $myDatasets: Boolean = false) {\n  datasets(first: 25, after: $cursor, orderBy: $orderBy, filterBy: $filterBy, myDatasets: $myDatasets) {\n    edges {\n      node {\n        id\n        created\n        uploader {\n          id\n          name\n          __typename\n        }\n        public\n        permissions {\n          id\n          userPermissions {\n            userId\n            level\n            access: level\n            user {\n              id\n              name\n              email\n              provider\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        draft {\n          id\n          partial\n          summary {\n            modalities\n            sessions\n            subjects\n            subjectMetadata {\n              participantId\n              age\n              sex\n              group\n              __typename\n            }\n            tasks\n            size\n            totalFiles\n            dataProcessed\n            __typename\n          }\n          issues {\n            severity\n            __typename\n          }\n          description {\n            Name\n            __typename\n          }\n          __typename\n        }\n        analytics {\n          views\n          downloads\n          __typename\n        }\n        stars {\n          userId\n          datasetId\n          __typename\n        }\n        followers {\n          userId\n          datasetId\n          __typename\n        }\n        snapshots {\n          id\n          created\n          tag\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    pageInfo {\n      startCursor\n      endCursor\n      hasPreviousPage\n      hasNextPage\n      count\n      __typename\n    }\n    __typename\n  }\n}\n"}'
}

echo -e "id\tnsubj" > subjcnt.txt
for C in ${CURSORS[@]}; do
 echo "$C"
 curlme $C | jq -r '.data|.datasets|.edges|.[]|.node| .id as $id |.draft |.summary| .subjects| [$id, length]|@tsv ' | tee -a subjcnt.tsv
 sleep 5
done