Skip to content

Instantly share code, notes, and snippets.

@nickdos
Last active April 22, 2020 00:22
Show Gist options
  • Save nickdos/fbe3c2cb633a877292e28254042195ac to your computer and use it in GitHub Desktop.
Save nickdos/fbe3c2cb633a877292e28254042195ac to your computer and use it in GitHub Desktop.
Collectory GBIF DOI citations lookup
#!/usr/local/bin/bash
#
# Iterate over list of GBIF DOI keys from data_resource table and get a count of the
# distinct publication citations via GBIF API by storing each citation DOI (ID) in an
# associative array and then reading the length of the this array.
#
# Requires BASH version > 4 and JQ command line tool (https://github.com/stedolan/jq)
#
# SQL: select gbif_registry_key from data_resource where gbif_registry_key <> '';
#
# run as ./collectory-doi-counts.sh gbif_registry_key_list.txt
#
GBIF_URL_PREFIX="https://www.gbif.org/api/resource/search?limit=999&contentType=literature&gbifDatasetKey="
if [[ ! -r "$1" ]]; then
echo "unable to read file '$1'"
exit 1
fi
declare -Ag doiHashMap
count=0
while read key; do
echo -n "$((count = count + 1)) $key "
citations=$(curl -s "$GBIF_URL_PREFIX$key" | jq -r '.results[].identifiers.doi')
while IFS= read -r doi ; do echo -n "."; key=${doi:-2}; doiHashMap["$key"]=1; done <<< "$citations"
echo "unique DOIs = ${#doiHashMap[@]}"
done < <(cat $1)
echo "Total DOIs = ${#doiHashMap[@]}"
printf "%s\n" "${!doiHashMap[@]}"
82cd8df8-f762-11e1-a439-00145eb45e9a
82dfda58-f762-11e1-a439-00145eb45e9a
82e0ee52-f762-11e1-a439-00145eb45e9a
82e33d42-f762-11e1-a439-00145eb45e9a
82e45d26-f762-11e1-a439-00145eb45e9a
82e57c42-f762-11e1-a439-00145eb45e9a
82e6a41e-f762-11e1-a439-00145eb45e9a
82e7db90-f762-11e1-a439-00145eb45e9a
82ea2cf6-f762-11e1-a439-00145eb45e9a
82eb67a6-f762-11e1-a439-00145eb45e9a
82eee8cc-f762-11e1-a439-00145eb45e9a
853006c0-f762-11e1-a439-00145eb45e9a
c60d0b4f-4e51-4a5a-95dc-babc74ba3db0
beed1b50-8c73-11dc-aaed-b8a03c50a862
85ff6f64-f762-11e1-a439-00145eb45e9a
8600992a-f762-11e1-a439-00145eb45e9a
86442d3e-f762-11e1-a439-00145eb45e9a
8645653c-f762-11e1-a439-00145eb45e9a
8647dc9a-f762-11e1-a439-00145eb45e9a
86b65c92-f762-11e1-a439-00145eb45e9a
88dfd534-f762-11e1-a439-00145eb45e9a
88e120c4-f762-11e1-a439-00145eb45e9a
88e24a94-f762-11e1-a439-00145eb45e9a
f900725d-db2f-4771-9330-7960ce68d4b5
895ca488-f762-11e1-a439-00145eb45e9a
895de76c-f762-11e1-a439-00145eb45e9a
95ddbdfa-f762-11e1-a439-00145eb45e9a
95deedf6-f762-11e1-a439-00145eb45e9a
95e02f86-f762-11e1-a439-00145eb45e9a
95e29b40-f762-11e1-a439-00145eb45e9a
95e3d104-f762-11e1-a439-00145eb45e9a
dce8feb0-6c89-11de-8225-b8a03c50a862
0debafd0-6c8a-11de-8225-b8a03c50a862
39905320-6c8a-11de-8226-b8a03c50a862
793c3890-6c8a-11de-8226-b8a03c50a862
a79c2b50-6c8a-11de-8226-b8a03c50a862
d0e133c0-6c8a-11de-8226-b8a03c50a862
fa375330-6c8a-11de-8226-b8a03c50a862
4ffe66a0-6c8b-11de-8226-b8a03c50a862
7c93d290-6c8b-11de-8226-b8a03c50a862
18c93d12-34fb-4d3f-903c-b77215a1dcc9
c8bc80f2-e9e3-4431-9998-6202be1132f3
2c0eeb7f-5c59-4e12-8aa2-d02c88bcf5c5
2812fba1-cf15-471e-88f0-1a4e6f25f72c
b5aaed9c-4f09-4cfc-9259-2dc65536a604
4bf1cca8-832c-4891-9e17-7e7a65b7cc81
358fbd61-739f-4102-afc9-9d468f571e4d
7565c55d-9dc8-442d-95cf-19d5ebb3d2b0
84a649ce-ff81-420d-9c41-aa1de59e3766
40c0f670-ee87-4576-be9d-2725e0b47035
2a513d16-f4ce-4252-974f-2486ce70b788
2a796d4b-d12f-4806-876a-4fbb5e0a63ca
0645ccdb-e001-4ab0-9729-51f1755e007e
4ce8e3f9-2546-4af1-b28d-e2eadf05dfd4
03ce64e0-7906-465f-afa8-1bfb6597aeaa
96a2390b-6137-4cf8-8323-aac518faab2b
75a06b5c-c16e-448f-9f7c-5d92be6667b6
e731a329-1fe3-4497-9e3c-24cb9235b5a1
522ff2e2-94f4-456f-b540-32f65f4b51ff
ad5b3c99-3877-4a24-9a26-6ab55fb9c39f
72fde3a9-f64a-46f2-a9ee-7994b53038f3
e697698b-17ec-48a3-aa60-273b24640013
94e857d3-74a7-44a3-ab0a-e033aba3986a
834442e7-c3ba-4ff9-bd1f-3288989f7725
492d70bb-fe52-4575-9b35-ce4282f41d7d
543c7a69-38bc-4506-9d20-aa10e4ae2876
70501d5b-609e-4d11-a8c5-88960b40ae10
1cfb1900-5735-4f47-9f97-cb99a1aa7211
62ce31e7-c3b5-477a-b97b-32d040ccaecc
4a2fdba3-6aa5-4a28-b0f3-ae8f1a0aa532
d7d8cce3-a4f3-4f64-9871-dd7283c85f91
ce58b7bb-fed6-46b9-9424-135499c7afdf
ee412fa2-edc9-4c6b-91f3-ff2a02c245e0
7d36731e-f762-11e1-a439-00145eb45e9a
82edc37a-f762-11e1-a439-00145eb45e9a
964cb534-f762-11e1-a439-00145eb45e9a
7f56df30-f762-11e1-a439-00145eb45e9a
96aa3736-f762-11e1-a439-00145eb45e9a
969e8a30-f762-11e1-a439-00145eb45e9a
96a682ee-f762-11e1-a439-00145eb45e9a
964f1f04-f762-11e1-a439-00145eb45e9a
7f5a4c74-f762-11e1-a439-00145eb45e9a
7f5b699c-f762-11e1-a439-00145eb45e9a
7f5da78e-f762-11e1-a439-00145eb45e9a
7f5c7c38-f762-11e1-a439-00145eb45e9a
7b6f0500-f762-11e1-a439-00145eb45e9a
7b702b2e-f762-11e1-a439-00145eb45e9a
599ba2ea-f3c1-4885-934a-7f5cad372c43
a5fd2bf1-fcba-4ea9-8edb-7b2421ad4c38
47abf4a5-0eb2-4373-bc51-61196e7346d2
73d4365a-8868-4d84-a6fd-51098b968170
920aa899-7c31-44a4-8a73-c3644f570228
b6465deb-2071-445a-9ebd-d5722e1ed7e2
167f1948-8f8e-4561-8f44-6184f3c79ef4
cd174407-5632-4090-ba61-9ff3fe41bc03
c1fc2df7-223b-4472-8998-70afb3b749ab
17812adb-60bb-4feb-b074-46e55eeab4e3
8007b00d-0834-4bea-b02e-eb34a6c937a5
06cb2d65-3dc1-4f9f-8e73-c4348abeba77
e2b51a4e-af48-4d33-b7c3-8c5549cba3ad
0cf9329b-036b-42f8-aa60-e58b6577711d
1ba242f4-6bff-4735-bdec-89357c997da5
d0582993-28ac-4ac4-937d-cb61497e532c
a855a185-cf36-4b06-89f6-bbbce2e2805d
16e7e918-38c8-4679-a612-22d933d34946
376a2b64-3c48-4b99-a38e-a5e3dd694414
d073b090-c76b-4d0b-81db-874298cd7181
be0ae621-f624-4a42-8936-304286bda625
e8443518-d916-47aa-9ae9-405333745b66
75bfed9c-679d-4c8f-ab46-feb2eb98512a
694b8003-0ee9-4360-982e-cb7dfb0cafe5
470cccd6-0a5b-4839-a2d1-8ae5d0bf9926
7cede6e3-c7f9-401d-9226-c9ce34a73f7d
b58d2f54-4d35-405b-8982-c3b77c64ddd2
aea7520f-8c5d-4a7a-9240-4e7fbeaefe86
21bfb712-d63c-4ef3-9627-bab7b928fb13
9236fad5-b80c-4630-86e0-b289812c2016
e3e48696-f083-4c3a-8c5b-efb1589c54db
4c7fac39-7664-43a9-a01d-6eac3305784e
cee9cad3-172b-475d-ab04-871c7617bbe8
d170f08a-3fe7-478a-9a0a-a68544f15f2f
2985efd1-45b1-46de-b6db-0465d2834a5a
02abb9d1-7d81-42b3-ac9a-3b3d0c7a5280
7447a228-a4cf-414b-87e4-3aba15275580
368caf55-2fda-4005-aa53-6a4a1c9f6465
8805b51e-d8e8-41d0-8976-d653d39c7a91
5cf2e899-2c17-499d-8b55-357d50d7a453
4b79a668-7e87-4154-ba53-51ee79a76599
118796ad-498c-41fc-a8c1-eb4bec1006ba
e8d7be1a-865f-4dee-bf10-a042ef43c89f
a3d3150a-6a85-4e42-885f-c273509a808e
3d5d2fc8-05bb-4d16-afb2-9d61a891e2d5
2c57f133-13e3-4016-ac63-d3f9299da2cc
4b75c0ac-f7c4-4883-942d-d1dfe6939754
edb501e5-93b6-45b2-b2be-b4705587a78b
9cc2cbfc-1002-4d0d-b262-cc3c5b268757
4e9aecfe-6aa6-467b-a874-a6a0d89242a6
d510984b-1cf9-43ba-94d0-a949f43d82f8
de077267-677e-4b3e-a40e-753a2d1c7b6a
5d11316c-4237-41cc-aba0-1288b532d5c2
226064ec-8b44-44c5-ae67-b98a9598e58c
bf7cb159-620a-4d8b-8c62-e2658712b3ae
bd93635d-c4ad-46ad-a285-aad2c3bfd408
30d9b134-c7df-4e90-8142-02d7a54f47b8
84a47405-034b-4ab3-bb49-b2ab36a21a88
63fa6322-9962-44a5-8691-24cdb26e1370
3d93e795-10d9-41fd-8eca-579c2aca513e
aeb6345b-aa74-44b8-989b-1b6fe7be6ab5
1173740d-43d0-4469-95ec-4a8a52895d6a
b3d40286-603b-47ee-bcaa-e829c383675d
eebc1080-cec1-4d28-a5d3-ba8694114d12
72e7571d-6d9c-4a87-94ea-7c5732bab796
c6ca96bc-7cf5-4fe7-ac44-d01e06394ef3
19369ff7-ef17-44ef-8df2-c02b3132fffd
5e7c3e74-2aaa-42a5-b93d-5e6b8418bd30
f66a607a-6658-4855-aaa1-291bd31c6d82
edd9ef2e-2229-406f-8dcc-4eab3cea3009
e7facd4a-7108-4506-b446-e7d69612d9f7
db8f0895-3023-4e03-9f37-2c27bb2e5fc7
7a78957a-ec61-4ff2-bd45-830006371692
64823037-f86c-4bc9-9cc5-439cf0b448df
8cf29185-bc61-4d54-980f-407cf8bea5c2
5d243972-6de3-4917-abfc-a2cd0c8438a9
8037efce-aa87-49fe-a7ad-1962f53b2f51
4b09a9fa-7189-45e3-af0c-e31e909c5001
2eb5a1d1-0436-4d3d-af87-2842ddd86c36
765feb50-40bd-4638-8e4f-a5f11b7a0676
3fa9349b-6507-4019-b956-fa3f989c854b
5cff5cac-c511-4cb4-89b1-c9a49cc77f49
4790dd85-b9d2-4a40-b3f6-dc4010cf3be3
dad84e72-95f7-4709-b564-72a4429942b1
3fab912e-e927-4f1c-a97c-eb446cd609e0
989a39ef-6e04-4439-83db-78bc286f19fa
32e8c7c1-b6a8-41b3-8379-6528200ffcff
c3f5731f-4ebc-48a2-83ba-3fb5c895f6c1
37beb6ea-e591-41e3-b781-e384250dc42c
ecadd1f4-5d30-489c-ba85-a47be708005e
557146f2-4f90-43ad-a2eb-c4f9ace7a2df
e4473544-f4cd-4429-b791-39d6e1fdb0a4
b6455ad6-a663-4047-8f9c-02dd5ab61ea6
884cb99c-95a0-4dac-94dd-d57dc509c560
0b9d46d1-4e57-4114-987a-083815522cf4
f7d6a733-fa5e-43ba-b06e-e70778953ead
7e231320-9118-42a7-a4d7-39bf182dbc4a
4287ce1e-b34e-48d4-b7aa-7fbfb9111607
170fd706-23a1-4036-b29e-57f4b2cb7bed
79b52ea9-6834-4822-90f8-b4497c259783
0471f28e-b58b-4c02-89e4-d2a1d621a9d8
a3fbbbf6-01ce-45f5-8cba-2a285af48200
e3ce628e-9683-4af7-b7a9-47eef785d3bb
402c0a2b-5101-4df7-a066-36af0b190d8f
c19a8f6e-c368-494d-942e-10108d8867a7
4677560c-a8d9-4e3e-b883-d3c6d97ded49
ea00f908-0fa5-469a-b9d3-5eb76136d998
8810b12b-7a24-403e-8b44-bf3848badb35
b0361e24-d3c6-4229-9c07-671d0ee428b0
a20a695a-df93-4bd8-98dd-33e1a37e2f7f
6f4fc23c-09fb-4307-8079-15eb970f7c9f
ea2fdd57-4632-4e7c-a20d-5d88982d55d1
b2c37c6b-ab2f-4eff-93a6-ecfd3676e69d
1be46db6-0035-47f9-abaf-f0c018f7f573
847375df-8ac8-4189-a8d6-deaefb90355c
04106094-0ced-419a-8216-17efdae78b90
f38cc07c-49a8-4ca4-8882-5cdde1733c12
bfbac936-6715-4f50-9b18-175bffb85025
6bfeecf3-bdb0-4deb-83d9-8f07029f2bbc
5d7f5915-0561-4107-aa20-20a4267c203f
87dcc098-729a-4444-8f0f-4c2971fe07e5
d3e6fa33-75a8-481b-bae4-7bafd683da67
c9d02825-4ff4-4483-bdd6-7bce652826d3
98a2fca0-4bfe-4d6e-85bc-b50d58891639
11ba0f8a-b084-47f6-b711-9f995c6f8e79
31cc8b55-0455-4faa-aa50-e135d1dcacf0
dcce70f7-06be-4de3-aed1-d811343566ea
0a9ada31-8f3e-412b-ac1c-2881e00ad3d9
37952dee-f155-4501-938f-3f1085cc404a
8442816f-cf5f-4c21-9d3d-5c368b4006d8
28ff4b21-bc9d-4c4e-91a5-e424ab301353
4896f82a-ff72-4d14-bf2a-780b86ab03f7
5a889070-20bd-11dd-bc28-b8a03c50a862
93788243-3b92-47f5-8658-d04868b5b00e
138cc86a-61be-4e89-86a7-737086398cbf
0e5c0373-f957-4ce3-b8d9-a9054df51fc5
84149905-2fc7-4cc6-a9c8-f4186e0a62a8
cbd1b8af-9457-483c-b0dd-6916ef98e6cb
3ea595af-7af0-44b6-9253-f5c120dbac73
63a4d812-1a6e-4c69-a47f-7493405bdba8
b92284e8-cf98-4827-8fa4-7cb764cfaca7
8a0e70b1-dd09-4c74-85f1-68554108ba7b
57ea6e10-3b95-4770-ab39-2111a06ddc7c
1bc719fd-c4e1-410f-b8c1-518cc1addcb5
792e64d5-c1ea-4bf2-b462-b377067cbdd1
4797da32-5693-4281-a8f1-bf99bae66ddb
465e33dd-59ec-4e55-b2c0-1edde45d757e
ce1faba4-d8d9-46ac-ba26-cffe03eadc99
390fe826-ee2f-4d82-ba45-01d5747c3e6c
e6e6cdd6-964c-406c-bd9b-b989e95640c5
fce24703-ad84-453c-8b52-f0763eb71d2d
c848f703-f22d-4517-892d-d2fbbbb8269f
6aebb29a-4dcd-41cb-be99-e08865e422d8
1f396a46-b83e-4255-a840-859985f73ded
0a6a9256-73be-4f7c-83af-a29db0a222c6
2cc3887f-73f3-4466-97f8-c85011314c2b
f906d1c4-6e1a-46dd-8805-ec98b5091908
9380202e-de78-488f-ada2-d0000e2e8a04
35e8d7cb-256c-4470-b2e6-79adadc308c5
9940b3a6-32a4-4f00-815e-f8492bd64a0e
d9d37e23-acee-436b-adc6-3659606b6830
0d031c51-08fa-4c31-aced-2250d458acd6
f2069019-06c4-40fc-97a4-6672134a1f1f
496f88c8-7c67-4e93-8f85-bc954e78debc
e671c044-762c-4db2-bbba-07175f5f3c10
62dbb284-ccc3-486b-a1f0-128a872da7ab
a8d6ee00-7297-425e-bcac-9b3b5cc093aa
68fc81fa-d63e-4cf1-a0e5-9e5464641166
12d4db8d-5529-43da-b790-2796cfd5c09b
a9a01e83-571b-42ff-b1ef-dab048029b32
0cb30c4a-55c7-481c-ac46-88c7d51a4dd1
e8b047d5-14de-4c08-b876-ee4d9e444bc2
e774b35d-60af-4063-9f40-b22695a1e3dc
ff6a915a-43f4-4e42-b2f9-6dc5fe577d0d
0ec475b6-11e4-41b4-904c-3d6a951b28eb
4a23152b-456e-48e9-9ebb-236e304a6378
f338c61f-4861-4dea-9379-96d98d8ede85
2754fff5-18b1-4706-bc8c-a647f08b665b
456619c2-c56e-4d4b-86df-ddc80750bda8
29b28617-c91c-4bc9-b3aa-c97960a8b5c8
0a147ef3-8436-44a2-8f06-7820daf54d30
c83a2438-0703-47fa-8b95-17d84e8c762f
a81bea66-5d00-4fc7-a8d8-4c873d916f06
f9452582-daf8-4724-a217-4e99af1aea8a
2ab285bd-1e2c-4e93-b321-ef1c5160730f
c2295938-45f9-473f-bfcc-453dfc590acf
b6162578-e753-40b4-bd69-0658221aee4c
86cd718d-eb8d-46a0-aeec-3b0089a69ab9
42793207-9afe-4991-bff4-db2be2e6182e
9419c0b3-4c23-40de-b96d-c4e55a23975e
6affc398-dfcb-4ac1-86a1-d8b9566e2dac
3b7f6307-2f03-4f25-9f64-6aa4c3a3aea1
51c51934-43f1-467c-b1b5-2e1c48fdd51b
b5267bb3-217b-49a1-b9c9-9024cdad2666
b5e6a82a-7781-497b-b2aa-118b8890c3fa
8c748691-6253-43fc-a851-86da40fa0021
68733b96-0244-495c-af7e-1c8d6a4a8dd3
59df094c-e61d-4cc4-a779-970fab936dba
46881d90-bf4d-49f4-bbb5-a1af323e00d3
8b70ab8e-c0d3-464b-8ce5-3f8d29470aca
8102988d-73b9-4ee0-8cd5-c73e2378b5d3
dca145de-af79-41c4-9dbb-9c61d4fb92b2
bdda202f-3b58-4cbc-828a-12ea52253d62
e11af578-d507-42a6-a34e-aaec39ee55bc
70c9ed78-fa16-4e18-95c5-80f46419f585
8f44aaa6-e2b0-4f8b-bae6-61b485f9f37f
af194990-12ae-4403-a66b-c1499d2f756b
6d4427fc-932b-4814-b323-ee7980788f3b
8226ec75-9bba-4351-a101-79306a7dcd55
fba07657-9a38-4cae-9797-d0febebfca24
592d585d-6186-4093-8375-77ad80da7c9a
9824fc39-2ab3-44b8-b4a8-530e18c01db4
2f5fa57d-1fae-444c-9669-258b7fb918b4
6e4b8bf0-7442-4faf-b94a-2987e89ebd37
7b582e40-5480-4da6-95b3-7d2386f5cac2
63cee3e6-4a0f-4469-928f-962b20872e48
a16a504d-de4b-4a26-9590-ab18da7e41f9
701888b9-26b1-4e7f-b226-a0766c6ec7f5
1864778a-0d5a-4b59-9396-33fdf6539b65
86b7940e-f762-11e1-a439-00145eb45e9a
def80289-b974-48d3-8ff1-7e91c82b5bfd
aeaef0c7-f842-4bca-a287-fa8e22f0086b
38f06820-08c5-42b2-94f6-47cc3e83a54a
f8f683a8-ab6c-4c4b-a6ec-53b23d5404d0
96d36be8-f65b-4bce-ac40-05ad8cadc9a1
5c88c333-373d-46bb-8e22-5a2f50913ed0
64ec1ba4-08d4-46ef-b40a-68965728e182
8bdcf8fe-a73b-4456-a5c3-8b20279f0521
f8ceb4e6-82ff-4325-afc8-eb5e64b5f842
d7349b4a-5b9e-414f-94a2-418af65425ce
58f732a6-b36e-4d86-bb67-22fe034ba196
51ffcd3a-f71f-491e-b75a-505657c3770e
9ecef2f5-217f-4f02-b403-425f2f6ed240
c7fcf26c-bd16-4b8e-9903-f871a03f424b
19b8cae9-a4f2-49ca-b687-cc297726e462
c03fe10c-2f92-49c8-9eb5-e81ca25a839c
8531183a-86b5-459b-a93e-37198f38f8a6
42301ebf-d36a-4e4f-a061-d37d778c2365
0ceb2c40-4134-482f-9d19-15560e3f1a97
85afbe6b-163d-4a81-ba18-d0a552e8c1f2
eb477342-875e-450a-b377-cf405f315f16
8db49bef-d87e-403d-ae19-32db175ab91e
6f19bb5e-ef43-492e-af23-608f2d84e064
9654e40e-c632-40b4-ba1a-df0f4537b8bd
0d979703-5f6b-424e-aa10-eb611202fe8e
cbee6026-cfa5-476f-aaa6-7884ab9eb964
ccf7853b-477b-47cf-ab2a-cb5cc2ac92c2
8642acf1-68f2-42b2-8ce3-97798e383cb2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment