Unblock everyone on your Twitter account, using twURL and lots of patience

unblock-all.sh

#!/bin/bash

#set -x

: ${num_parallel:=8}
: ${parallel:=parallel}
: ${cycle_time:=10}
# rate_limit_sleep_time=15
# default_rate_limit_sleep_time=$rate_limit_sleep_time
cursor="-1"
url="/1.1/blocks/list.json?count=100"

which jq >/dev/null || {
  echo "* \`jq\` not found, please install from https://stedolan.github.io/jq/" >&2
  exit 1
}
which twurl >/dev/null || {
  echo "* \`twurl\` not found, please install from https://github.com/twitter/twurl" >&2
  exit 2
}
which "${parallel}" >/dev/null || {
  echo "* \`${parallel}\` not found, please install either GNU xargs or GNU parallel" >&2
  exit 3
}

echo "* Using \`${parallel}\` with ${num_parallel} processes, request cycle time of ${cycle_time}s" >&2
echo "* Checking twurl accounts…"
num_twurl_accts=$(twurl accounts | wc -l)
if [ $num_twurl_accts -lt 2 ]; then
  echo "* No twurl accounts configured - please read https://github.com/twitter/twurl#getting-started" >&2
  exit 4
fi
twurl accounts | sed -e 's/^/    /' >&2
echo "* Using twurl default account (shown above)" >&2

if [ -f blocklist.json ]; then
  cursor=$(jq -r <blocklist.json '.next_cursor_str')
fi

echo -n "* Ready to continue from cursor ${cursor}, press ENTER to continue or Ctrl-C to abort" >&2
read

starttime=$(date +%s)
total=0

function cleanup() {
  echo >&2
  echo "* Cleaning up…" >&2
  [ -f blocklist.json.tmp ] && rm blocklist.json.tmp
  [ -f blocklist.request.tmp ] && rm blocklist.request.tmp
  endtime=$(date +%s)
  duration=$((endtime - starttime))

  echo "* Unblocked ${total} users in ${duration}s" >&2
  exit
}
trap cleanup INT

while [ ${cursor:-0} -ne 0 ]; do
  uwc="${url}&cursor=${cursor}"
  while :; do
    twurl -t "${uwc}" >blocklist.json.tmp 2>blocklist.request.tmp
    grep -cq "\"Rate limit exceeded\"" blocklist.json.tmp || break
    rate_limit_count=$(grep 'x-rate-limit-limit' blocklist.request.tmp | sed -e 's/[^0-9]*//g')
    rate_limit_reset=$(grep 'x-rate-limit-reset' blocklist.request.tmp | sed -e 's/[^0-9]*//g')
    echo "* Rate limit of ${rate_limit_count} per 15m exceeded, resetting at ${rate_limit_reset} (it is now $(date +%s))" >&2
    rate_limit_calc_cycle_time=$((60 * 15 / rate_limit_count))
    echo "* Setting cycle time according to rate limit (to ${rate_limit_calc_cycle_time}s)" >&2
    cycle_time=$rate_limit_calc_cycle_time
    rate_limit_diff=$((10 + rate_limit_reset - $(date +%s)))
    echo "* Sleeping for ${rate_limit_diff}s…" >&2
    sleep ${rate_limit_diff}
    # echo "* Rate limit exceeded, sleeping for ${rate_limit_sleep_time}s" >&2
    # sleep ${rate_limit_sleep_time}
    # rate_limit_sleep_time=$((rate_limit_sleep_time * 1.5))
    # rate_limit_sleep_time=${rate_limit_sleep_time/.*}
  done
  # rate_limit_sleep_time=$default_rate_limit_sleep_time
  mv blocklist.json.tmp blocklist.json
  count=$(jq -r <blocklist.json '.users[].id' | wc -l)
  count=$((count + 0))
  total=$((total + count))
  echo "* Got ${count} items at cursor ${cursor}" >&2
  echo -n "* Unblocking: " >&2
  jq -r <blocklist.json '.users[].id' | $parallel -P ${num_parallel} -n 1 -I % "twurl -q -X POST '/1.1/blocks/destroy.json?user_id=%&skip_status=1' && echo -n '.' >&2 || echo -n 'X' >&2" &
  cursor=$(jq -r <blocklist.json '.next_cursor_str')
  sleep ${cycle_time}
  echo -n '|' >&2
  wait
  echo >&2
  echo "* Unblocked ${total} users so far" >&2
done
endtime=$(date +%s)
duration=$((endtime - starttime))

cleanup
rm blocklist.json