AnyEvent loop

loop.pl

#!/usr/bin/env perl

use strict;
use 5.010;
use AnyEvent;
use AnyEvent::HTTP;
use Time::HiRes 'time';

my $records = [ (1)x4e3 ];
my $i = 0;
say 0+@$records;

<<'WRONG'
	$cv = AE::cv;
	for (@$records) {
		$cv->begin;
		http_request ... sub {
			$cv->end;
		}
	}
	$cv->recv;
WRONG
;

$AnyEvent::HTTP::MAX_PER_HOST = 100;

say "Starting in parallel by $AnyEvent::HTTP::MAX_PER_HOST";
my $job;
my $cv = AE::cv;
my $st = time;
$job = sub {
	if ($i < @$records) {
		if ($i % 1000 == 0) {
			my $d = time - $st;
			printf "Processed %d in %0.2fs (%0.2f/s)\n",$i,$d,$i/$d;
		}
		
		my $record = $records->[$i++];
		# process your record here
		http_request
			GET => 'http://localhost/',
			sub {
				my ($body,$headers) = @_;
				# process results
				
				# call for continuation
				$job->();
			}
		;
	} else {
		# no more jobs
		$cv->end;
	}
};
$cv->begin;
for ( 1 .. $AnyEvent::HTTP::MAX_PER_HOST ) {
	$cv->begin;
	$job->();
}
$cv->end;