A simple Perl-script to extract and execute wget commands from a log

gwfl.pl

#!/usr/bin/perl                                                                                                                         
# A simple Perl-script to extract and execute wget commands from a log                                                                  
# I wrote it originally for Kippo because it downloads only if wget
# goes to port 80 but it's maybe capable of some other                                                                  
# usecases. I call it gwfl for "get wget from log"                                                                                      
#                                                                                                                                       
# Don't forget to install missing packages from CPAN with "cpan" or                                                                     
# your package manager                                                                                                                  
use strict;
use warnings;
use Digest::SHA;
use File::Copy qw(move);
use LWP::Simple;
# Look for new lines all dt secs                                                                                                        
my $dt = 5;
my $pos = 0;
my $run = time + $dt;
# Change this string to a folder where all files should downloaded to                                                                   
# e.g. "/home/user/kippo/dl/"                                                                                                           
my $dlfolder = "";
# Change this string to a folder where the hash log should written to                                                                   
# e.g. "/home/user/gwfl/"                                                                                                               
my $logfolder = "";
# Kippo-Log or something else                                                                                                           
# e.g. "/home/user/kippo/log/kippo.log"                                                                                                 
my $honeylog = "";
# Folder where to save files by host                                                                                                                                                                                                                                             
# Comment it out if you don't need this feature                                                                                                                                                                                                                                  
# I want this because my vtd-daemon sends all files from $dlfolder                                                                                                                                                                                                               
# to Virustotal but I also want everything sorted by host/thread                                                                                                                                                                                                                 
my $hostfolder = "";
# Uncomment to print stdout to a file instead of the command line                                                                       
# open STDOUT, '>>', "gwfl_out";                                                                                                        
for(;;) {
    if(time >= $run) {
        open FILE, $honeylog or die "$!";
        # Works for the corner case of rotated logs                                                                                     
        $pos = 0 if -s $honeylog < $pos;
        seek(FILE, $pos, 0);
        my @lines;
        chomp(@lines = <FILE>);
        foreach (@lines) {
            # if(/(https?|ftp):\/\/(\.)?([^\s\/?\.#-]+\.?)+(\/[^\s]*)?$/) {                                                             
            if(/(https?|ftp):\/\/([^\s\/]+\.?)+(\/[^\s]*)?$/) {
                my $url = $&;
                my $host = $2;
                # Use Digest::SHA->new(1) if you want shorter filenames                                                                 
                my $sha = Digest::SHA->new(256);
                if(/wget.+\Q$url\E/) {
                    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime;
                    $year += 1900;
                    $mon += 1;
                    printf("%04d-%02d-%2d %02d:%02d:%02d - Download: $url\n", $year, $mon, $mday, $hour, $min, $sec);
                    my $status = getstore($url, "tmp");
                    if(is_success($status)) {
                        open TMP, "tmp" or die "$!";
                        while(<TMP>) {
                            $sha->add($_);
                        }
                        my $shasum = $sha->hexdigest;
                        close TMP;
                        print "Hash is $shasum\n";
                        move "tmp", "$shasum";
                        # Comment next two lines out if you don't want per host sorting                                                                                                                                                                                          
                        mkdir "$hostfolder"."$host";
                        copy "$shasum", "$hostfolder"."$host";
                        move "$shasum", "$dlfolder"."$shasum;
                        
                        open DB, ">>", $logfolder."gwfl.log" or die "$!";
                        printf(DB "$shasum %04d-%02d-%02d %02d:%02d:%02d $host\n", $year, $mon, $mday, $hour, $min, $sec);
                        close DB;
                    } else {
                        print "Could't download $url\n";
                    }
                }
            }
        }
        $pos = tell(FILE);
        $run = time + $dt;
        close FILE;
    }
}