perl-fetcher

fetcher.pl

#!/usr/bin/env perl
use warnings;
use LWP::Simple;
use utf8;
use strict;
use LWP::UserAgent ();
my $s;
my $response;
# Start your script below this line;
open(URL, '<url.list.txt') or die $!;
open(RAW, '>>raw.pages.txt') or die $!;
 
my $ua = LWP::UserAgent->new;
while (<URL>){
    print $_;
    $response = $ua->get($_);;
    if ($response->is_success) {
        print $response->decoded_content;  # or whatever
    }
    else {
        die $response->status_line;
    }
    print "----\n";
    #print RAW "NEWITEM\t", get($_);
}
close RAW;
close URL;