Skip to content

Instantly share code, notes, and snippets.

@evanfoster
Last active June 7, 2023 01:09
Show Gist options
  • Save evanfoster/ddbea335a30a47266dc97a8a0fb56464 to your computer and use it in GitHub Desktop.
Save evanfoster/ddbea335a30a47266dc97a8a0fb56464 to your computer and use it in GitHub Desktop.
Grabs the total subscriber count of all protesting subreddits.
#!/usr/bin/env zsh
# set -x
# Putting this in the user's $HOME, since that seems more secure. Probably nobody else will run this,
# so the risk is really just mine.
token_file=~/.reddit_token
user_agent_file=/tmp/user-agent
echoerr () { printf '%s\n' "$@" 1>&2; }
die () {
message="$1"; shift
exit_code="${1:-1}"
echoerr "$message"
exit "$exit_code"
}
needs_new_token () {
[[ -f "$token_file" ]] || return 0
current_time="$(date +%s)"
expiration_time="$(jq '.expiration_time' "$token_file")"
# Let's build in 10 minutes of wiggle room.
((expiration_time -= 600))
(( current_time <= expiration_time )) || return 0
return 1
}
wiki_url='https://oauth.reddit.com/r/modcoord/wiki/index.json'
reddit_post_url='https://oauth.reddit.com/r/ModCoord/comments/1401qw5/incomplete_and_growing_list_of_participating/.json'
data_source="${1:-wiki}"
case "$data_source" in
wiki) source_url="$wiki_url";;
post) source_url="$reddit_post_url";;
*) die "Usage: $0 <source, either 'wiki' or 'post'>";;
esac
type jq 1>&2 || die 'jq is not installed. Please install jq with your package manager'
if needs_new_token; then
read -r '?Username: ' username
read -r -s '?Password: ' reddit_password; echo ''
read -r '?App ID: ' app_id
read -r -s '?App secret: ' app_secret; echo ''
[[ -z "$username" || -z "$reddit_password" || -z "$app_id" || -z "$app_secret" ]] && die 'No empty auth values are allowed'
current_time="$(date +%s)"
if ! auth_response="$(curl --fail-with-body -X POST -d "grant_type=password&username=$username&password=$reddit_password" --user "$app_id:$app_secret" https://www.reddit.com/api/v1/access_token)"; then
die "Reddit returned an error when getting auth: $auth_response"
fi
[[ -z "$auth_response" ]] || die 'Reddit returned no auth data'
expires_in="$(jq '.expires_in' <<<"$auth_response")"
expiration_time=$((current_time + expires_in))
jq --arg "$expiration_time" '. + {expiration_time: $expiration_time}' <<<"$auth_response" > "$token_file"
fi
if ! [[ -f "$user_agent_file" ]]; then
read -r '?User agent: ' user_agent
[[ -z "$user_agent" ]] && die 'User agent must be set'
printf '%s' "$user_agent" > "$user_agent_file"
fi
token_type="$(jq -r '.token_type' "$token_file" | tr -d '\n')"
token="$(jq -r '.access_token' "$token_file" | tr -d '\n')"
user_agent="$(<$user_agent_file)"
if ! post_data="$(curl --fail-with-body -s -L -H "Authorization: $token_type $token" -A "$user_agent" "$source_url")"; then
die "Reddit returned an error when getting subreddit list: $post_data"
fi
[[ -z "$post_data" ]] && die 'Reddit returned no post data'
case "$data_source" in
wiki) data="$(jq -r '.data.content_md' <<<"$post_data")";;
post) data="$(jq -r '.[0].data.children[0].data.selftext' <<<"$post_data")"
esac
# this zsh expansion is cool and also kind of horrible
subreddits=("${(@f)$(sed '/^$/d' <<<"$data" | grep -oE '^r/.*$' | sed 's|/$||g'| tr '[:upper:]' '[:lower:]' | sort -u)}")
mkdir -p /tmp/r || die 'Failed to create destination path'
failed_subreddits=()
for subreddit in "${subreddits[@]}"; do
[[ -f /tmp/"$subreddit" ]] && continue
sleep .5
if ! response="$(curl --fail-with-body -s -L -H "Authorization: $token_type $token" -A "$user_agent" https://oauth.reddit.com/"$subreddit"/about.json)"; then
error_code="$(jq '.error' <<<"$response")"
reason="$(jq '.reason' <<<"$response")"
if [[ "$reason" == null ]]; then
reason="$(jq '.message' <<<"$response")"
fi
echoerr "$subreddit: got $error_code when getting data for reason: $reason"
failed_subreddits+=("$subreddit")
continue
fi
count="$(jq '.data.subscribers' <<<"$response")"
if grep -q -oE '^[0-9]+$' <<<"$count"; then
printf '%s: %i\n' "$subreddit" "$count"
printf '%i' "$count" > /tmp/"$subreddit"
else
echoerr "$subreddit: failed to get data, probably due to a typo"
failed_subreddits+=("$subreddit")
fi
done
printf $'Number of subreddits listed in the %s: %\'d\n' "$data_source" "${#subreddits[@]}"
subreddit_files=(/tmp/r/*)
printf $'Number of subreddits that data was gathered for: %\'d\n' "${#subreddit_files[@]}"
counts=()
for file in "${subreddit_files[@]}"; do
counts+=("$(<"$file")")
done
# Thanks, https://stackoverflow.com/a/13635566 !
sum=$(IFS=+; echo "$((${counts[*]}))")
printf $'Sum of subscribers across %\'d subreddits: %\'d\n' "${#subreddit_files[@]}" "$sum"
printf "Subreddits that data couldn't be gathered for: "; printf '%s, ' "${failed_subreddits[@]}" | sed 's|, $||g'; echo
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment