Skip to content

Instantly share code, notes, and snippets.

@dolmen
Created April 4, 2012 09:02
Show Gist options
  • Save dolmen/2299812 to your computer and use it in GitHub Desktop.
Save dolmen/2299812 to your computer and use it in GitHub Desktop.
Fetching URLs with AnyEvent::HTTP vs LWP::UserAgent::POE
# Written by Olivier Mengué
use utf8;
use strict;
use warnings;
use AnyEvent;
use AnyEvent::HTTP;
my @urls = qw(
http://perl.org/
http://jobs.perl.org/
http://www.perl.org/
http://blogs.perl.org/
http://moose.perl.org/
http://dbi.perl.org/
http://debugger.perl.org/
);
use constant MAX_CONN => 3;
sub create_conn
{
my ($cv, $ref_handle, $conn_idx, $cnt) = @_;
unless (@urls) {
print "We ran out of urls $conn_idx\n",
"Session $conn_idx has stopped.\n";
return
}
my $url = shift @urls;
$cnt++;
my $t0 = time;
print "Session $conn_idx count $cnt\n";
$cv->begin;
$$ref_handle = http_get $url, timeout => 10, sub {
delete $conn[$conn_idx];
print "Session $conn_idx $url DONE ", (time - $t0), "\n";
create_conn($cv, $ref_handle, $conn_idx, $cnt);
$cv->end;
};
}
my $cv = AE::cv;
my @conn;
for my $conn_idx (0 .. MAX_CONN-1) {
print "Session $conn_idx has started.\n";
create_conn($cv, \$conn[$conn_idx], $conn_idx, 0);
}
$cv->recv;
# Posted on POE mailing list by Gabor Szabo
use strict ;
use warnings ;
use POE;
use LWP::UserAgent::POE;
my @urls = qw(
http://perl.org/
http://jobs.perl.org/
http://www.perl.org/
http://blogs.perl.org/
http://moose.perl.org/
http://dbi.perl.org/
http://debugger.perl.org/
);
for (1 .. 3) {
POE::Session->create(
inline_states => {
_start => \&handler_start,
get => \&handler_get,
count => \&handler_count,
_stop => \&handler_stop,
}
);
}
POE::Kernel->run();
exit;
sub handler_start {
my ($kernel, $heap, $session) = @_[KERNEL, HEAP, SESSION];
print "Session ", $session->ID, " has started.\n";
$heap->{cnt} = 0;
$kernel->yield('get');
}
sub handler_get {
my ($kernel, $heap, $session) = @_[KERNEL, HEAP, SESSION];
my $url = shift @urls;
if ($url) {
$heap->{url} = $url;
$kernel->yield('count');
print "Session ", $session->ID, " $url\n";
my $t0 = time;
my $ua = LWP::UserAgent::POE->new(timeout => 10);
my $response = $ua->get($url) ;
print "Session ", $session->ID, " $url DONE ", (time - $t0) , "\n";
delete $heap->{url};
$kernel->yield('get');
} else {
print "We ran out of urls " . $session->ID, "\n";
}
}
sub handler_count {
my ($kernel, $heap, $session) = @_[KERNEL, HEAP, SESSION];
$heap->{cnt}++;
print "Session " . $session->ID . " count $heap->{cnt}\n";
if ($heap->{url}) {
$kernel->delay( count => 1 );
}
}
sub handler_stop {
print "Session ", $_[SESSION]->ID, " has stopped.\n";
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment