scrape-recent-failures

#!/usr/bin/env zsh

# Find recent build failures containing a specific message by scraping the
# website for builds since a specific nightly (exclusive)
#
#     ./scrape-recent-failures 2022-03-23 'no entry found for key' > fails

set -euo pipefail

nightly="${1:?missing argument 1: date of last nightly before the problem}"
message="${2:?missing argument 2: message to check for}"

for page in {1..20}
do
  echo "Listing page $page" >&2
  curl -sS https://docs.rs/releases/recent-failures/$page \
    | pup -p '.release .name text{}' \
    | while read -r l
      do
        crate="${l%-*}"
        version="${l##*-}"
        curl -sS https://docs.rs/crate/$crate/$version/builds \
          | pup 'li:first-of-type .release attr{href}' \
          | while read -r l
            do
              echo "Checking $crate @ $version" >&2
              log="$(curl -sS https://docs.rs$l | pup -p 'pre text{}')"
              if (echo "$log" | egrep -qs "rustc 1\\.[0-9]+\\.[0-9]+-nightly \\([0-9a-f]+ $nightly\\)")
              then
                echo "Completed searching till $nightly" >&2
                exit
              fi
              if (echo "$log" | grep -qsF "$message")
              then
                echo "Matched $crate @ $version" >&2
                echo $crate $version
              fi
              sleep 1
            done
      done
done