generate a list of all tidypub urls in use. would take an estimated 12 years to complete a run on a single machine.

tidypubscraper.pl

#!/usr/bin/perl

use strict;
use warnings;
use LWP::UserAgent;
use Math::BaseArith;

my $browser = LWP::UserAgent->new;

sub get {
  my $key = shift;
  my $url = "http://tidypub.org/$key";
  my $response = $browser->get("http://tidypub.org/$key");
  print "$url\n" if ($response->is_success);
  if (!$response->is_success && $response->status_line ne '404 NOT FOUND') {
    die "$url -- ", $response->status_line, "\nAborted";
  }
}

sub toKey {
  my $n = shift;
  my @digits = encode($n, [52, 52, 52, 52, 52]);
  my $key = '';
  foreach my $digit (@digits) {
    my $char = ($digit < 26) ? chr($digit + 97) : chr(($digit - 26) + 65);
    $key .= "$char";
  }
  return $key;
}

for (my $i = 0; $i < 380204031; $i++) {
  my $key = toKey($i);
  get($key);
}