Skip to content

Instantly share code, notes, and snippets.

@jonchurch
Last active April 12, 2024 08:00
Show Gist options
  • Save jonchurch/7b449632ecb532871f061411d499801c to your computer and use it in GitHub Desktop.
Save jonchurch/7b449632ecb532871f061411d499801c to your computer and use it in GitHub Desktop.
Given a github org, get the npm registry download stats for the past week
#!/bin/bash
# Exit on error, uninitialized variable use, and catch errors in pipelines
set -euo pipefail
# set -x
# Initialize the DEBUG flag
DEBUG=false
# Process command-line arguments
while getopts "d" opt; do
case $opt in
d)
DEBUG=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
exit 1
;;
esac
done
# Remove the parsed options from the positional parameters
shift $((OPTIND-1))
# Check if an organization name is provided as an argument
if [ "$#" -ne 1 ]; then
echo "Usage: $0 -d <org-name>"
exit 1
fi
# Enable debugging if the debug flag is set
if [ "$DEBUG" = true ]; then
# Print each command before executing for debugging
set -x
fi
# Assign the first argument as the organization name
ORG=$1
# Output file
OUTPUT_FILE="${ORG}_stats.csv"
# Write CSV header
echo "Repository Name, Package Name, Total Downloads" > $OUTPUT_FILE
# Fetch repositories and iterate over them
gh repo list $ORG --limit 1000 --json name,url | jq -c '.[]' | while read -r repo; do
REPO_NAME=$(echo $repo | jq -r '.name')
REPO_URL=$(echo $repo | jq -r '.url')
# Initialize package name and download stats as null
PACKAGE_NAME=null
DOWNLOAD_STATS=null
# Construct the raw content URL for package.json
RAW_URL="${REPO_URL/github.com/raw.githubusercontent.com}/master/package.json"
# Fetch package.json content if it exists
PACKAGE_JSON=$(curl -s $RAW_URL || true)
if echo "$PACKAGE_JSON" | jq . > /dev/null 2>&1; then
# Valid JSON, extract package name
PACKAGE_NAME=$(echo $PACKAGE_JSON | jq -r '.name' || true)
fi
# Fetch last week of download stats from npm if the package name is valid
if [ ! -z "$PACKAGE_NAME" ] && [ "$PACKAGE_NAME" != "null" ]; then
DOWNLOAD_JSON=$(curl -s "https://api.npmjs.org/downloads/range/last-week/$PACKAGE_NAME")
if echo "$DOWNLOAD_JSON" | jq . > /dev/null 2>&1 && [ "$(echo "$DOWNLOAD_JSON" | jq 'type')" = '"object"' ]; then
DOWNLOAD_STATS=$(echo "$DOWNLOAD_JSON" | jq '[.downloads[]?.downloads] | add' || echo "null")
else
DOWNLOAD_STATS=null
fi
else
# If there is no package name, ensure download stats are set to null
DOWNLOAD_STATS=null
fi
# Write to the output file
echo "${REPO_NAME}, ${PACKAGE_NAME}, ${DOWNLOAD_STATS}" >> $OUTPUT_FILE
done
# Sorting the CSV file based on the "Total Downloads" column
# Backup the original file
cp $OUTPUT_FILE "${OUTPUT_FILE}.bak"
# Sort and save
(head -n 1 $OUTPUT_FILE && tail -n +2 $OUTPUT_FILE | awk 'BEGIN{FS=OFS=","} {if($3==" null") $3=0; print $0}' | sort -t, -k3,3nr) > "${OUTPUT_FILE}.sorted"
mv "${OUTPUT_FILE}.sorted" $OUTPUT_FILE
echo "Finished. The output is saved in $OUTPUT_FILE"
Repository Name Package Name Total Downloads
body-parser body-parser 33,226,789
express express 28,984,418
serve-static serve-static 28,352,551
compression compression 17,345,310
cors cors 11,126,011
serve-index serve-index 10,959,460
multer multer 4,539,309
morgan morgan 4,348,628
cookie-parser cookie-parser 2,924,216
serve-favicon serve-favicon 2,512,154
errorhandler errorhandler 2,002,237
session express-session 1,651,732
method-override method-override 802,007
basic-auth-connect basic-auth-connect 498,922
csurf csurf 423,281
response-time response-time 327,456
timeout connect-timeout 252,265
cookie-session cookie-session 192,233
vhost vhost 140,943
connect-multiparty connect-multiparty 63,375
generator express-generator 8,570
express-paginate express-paginate 4,163
express-expose express-expose 2,912
flash flash 1,121
domain-middleware domain-middleware 968
api-error-handler api-error-handler 842
express-namespace express-namespace 782
urlrouter urlrouter 156
connect-rid connect-rid 87
connect-markdown connect-markdown 16
routification routification 6
mime-extended mime-extended 3
restful-router restful-router 3
badgeboard badgeboard 1
vhostess vhostess 1
.github 0
Admin 0
discussions 0
examples 0
expressjs.com expressjs.com 0
expressjs.github.io 0
security-triage 0
security-wg 0
set-type set-type 0
statusboard 0
Repository Name Package Name Total Downloads
statuses statuses 50,086,878
http-errors http-errors 49,953,166
mime-db mime-db 48,505,624
mime-types mime-types 47,834,708
cookie cookie 46,640,731
on-finished on-finished 38,473,623
negotiator negotiator 31,516,160
content-disposition content-disposition 29,633,827
accepts accepts 29,072,232
range-parser range-parser 28,617,123
content-type content-type 28,354,509
vary vary 27,791,502
methods methods 27,379,018
fresh fresh 27,136,751
etag etag 26,999,514
media-typer media-typer 26,688,520
type-is type-is 26,035,874
proxy-addr proxy-addr 25,829,619
forwarded forwarded 25,714,700
on-headers on-headers 19,532,487
compressible compressible 18,775,382
basic-auth basic-auth 6,887,391
http-assert http-assert 1,332,129
spdy-push spdy-push 35
http-push http-push 1
jshttp.github.io badgeboard 1
.github null 0
http-utils http-utils 0
style-guide null 0
Repository Name Package Name Total Downloads
path-to-regexp path-to-regexp 46,680,760
finalhandler finalhandler 31,509,856
send send 29,076,351
encodeurl encodeurl 27,128,990
parseurl parseurl 26,931,206
cookies cookies 1,992,464
resolve-path resolve-path 780,729
csrf csrf 600,881
router router 579,613
path-match path-match 468,694
multiparty multiparty 419,282
hbs hbs 166,698
routington routington 818
templation templation 57
ssl-redirect ssl-redirect 30
qs-strict qs-strict 4
pillarjs.github.io badgeboard 1
.github null 0
discussions null 0
extend-proto extend-proto 0
node-frameworks null 0
request @pillarjs/request 0
understanding-csrf null 0
views null 0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment