hoehrmann/gist:2340570

## gistfile1.pl
#!perl -w
use strict;
use warnings;
use LWP::UserAgent;
use HTML::FormatText;

die "Usage: $0 bookid > example.txt\n" unless @ARGV == 1;

my $book = shift @ARGV;
my %seen;
my $next = "PP1";
my $ua = LWP::UserAgent->new;

$ua->agent('Mozilla/5.0');

binmode STDOUT, ':utf8';

$|++;

while (1) {
  last if $seen{ $next }++;
  my $uri = "http://books.EXAMPLE.com/books?id=${book}&pg=${next}&hl=en&output=text";
  warn "Retrieving $next";
  my $response = $ua->get($uri);
  my $content = $response->decoded_content;
  if ($content =~ /pg=([a-zA-Z0-9_-]+)[^"]*" accesskey="n"/) {
    $next = $1;
  } else {
    warn "found no next page!\n";
  }
  $content =~ s/<body.*?<div id='flow-top-div' class='flow-top-div'>/<body><div>/s;
  $content =~ s(</div>\s*</div>\s*</div>\s*</div>\s*.*$)()s;
  my $fmt = HTML::FormatText->new;
  print $fmt->format_string($content);
  sleep 1;
}
	#!perl -w
	use strict;
	use warnings;
	use LWP::UserAgent;
	use HTML::FormatText;

	die "Usage: $0 bookid > example.txt\n" unless @ARGV == 1;

	my $book = shift @ARGV;
	my %seen;
	my $next = "PP1";
	my $ua = LWP::UserAgent->new;

	$ua->agent('Mozilla/5.0');

	binmode STDOUT, ':utf8';

	$\|++;

	while (1) {
	last if $seen{ $next }++;
	my $uri = "http://books.EXAMPLE.com/books?id=${book}&pg=${next}&hl=en&output=text";
	warn "Retrieving $next";
	my $response = $ua->get($uri);
	my $content = $response->decoded_content;
	if ($content =~ /pg=([a-zA-Z0-9_-]+)[^"]*" accesskey="n"/) {
	$next = $1;
	} else {
	warn "found no next page!\n";
	}
	$content =~ s/<body.*?<div id='flow-top-div' class='flow-top-div'>/<body><div>/s;
	$content =~ s(</div>\s</div>\s</div>\s</div>\s.*$)()s;
	my $fmt = HTML::FormatText->new;
	print $fmt->format_string($content);
	sleep 1;
	}