crawler-ex.pl

sub run {
	my $url = shift;
	my $cb = shift;

	my @queue;
	push @queue, $start_url;
	my %seen;

	my $cv = AE::cv; $cv->begin;


	my $work;$work = sub {
		my $url = shift @queue;
		return unless $url;
		$cv->begin;
		http_request
			HEAD => $url,
			sub {
				...
				my $next = sub {
					$work->() while @queue and $cv->{_ae_counter} < 100;
				};

				if (not $ok) {
					$next->()
				} else {
					$cv->begin;
					my_get(
						$url,
						\%seen,
						...,
						sub {
							$next->()
							$cv->end;
						}
					);
				}
				$cv->end;

			}

	}; $work->() for 1..100;

	$cv->end;
	$cv->cb( $cb );

}

sub my_get {
	my ($url, $seen, ..., $cb ) = @_;
	http_request
		GET => $url,
		sub {
			...
			push @queue, grep { !exists $seen{$_}} @extracted_urls

			$cb->();
		}

}