gistfile1.txt

#!/usr/bin/env perl6

use v6;

use DOM::Tiny;
use HTTP::UserAgent;

my $semaphore;

sub MAIN(Str :$seed = "http://perl6.org",
         Str :$file = "test.links",
         Int :$depth = 2,
         Int :$workers = 4,
        ) {

    $semaphore = Semaphore.new: $workers;
    crawl($seed, $file, $depth);
    sleep ∞;
}

sub crawl($url, $file, $depth) {
    return if $depth ≤ 0;

    $semaphore.acquire;
    my $acquired = True;
    LEAVE $semaphore.release if $acquired;

    say $url;

    my $ua = HTTP::UserAgent.new;
    my @links;
    my $response = $ua.get($url);
    my $dom = DOM::Tiny.parse(~$response);
    for $dom.find('a[href]') -> $e {
        #say "Getting $e<href>";
        if $e<href> ~~ /http/ {
            @links.push($e<href>);
        }
        else {
            @links.push("$url$e<href>");
        }
    }
    my $fh = open $file, :a;
    $fh.say(@links.unique.join("\n"));
    $fh.close;
    for @links -> $link {
        start crawl($link, $file, $depth - 1);
    }
}