gistfile1.txt

#!/usr/bin/env perl6

use DOM::Tiny;
use HTTP::UserAgent;

my $channel = Channel.new;

sub MAIN(Str :$seed = "http://perl6.org", :$depth = 2) {
    $channel.send: $seed => $depth;

    for $channel.list.race(:1degree, :1batch) {
        crawl .key, .value
    }
}

sub crawl($url, $depth where *.so) {
    say $depth, ‘ - ’, $url;
    my $ua = HTTP::UserAgent.new;
    react {
        whenever $ua.get($url) -> $response {
            my $dom = DOM::Tiny.parse: ~$response;

            process $dom;

            return if $depth ≤ 1;
            for $dom.find('a[href]') -> $e {
                if $e<href> ~~ /http/ {
                    $channel.send: $e<href> => $depth - 1;
                } else {
                    $channel.send: "$url$e<href>" => $depth - 1;
                }
            }
        }
    }
}

sub process($dom) {
    # Do other page processing here
}