Skip to content

Instantly share code, notes, and snippets.

@m0rb
Created October 13, 2021 16:56
Show Gist options
  • Save m0rb/f26d76c7eaf4398eb1142e46fbd52750 to your computer and use it in GitHub Desktop.
Save m0rb/f26d76c7eaf4398eb1142e46fbd52750 to your computer and use it in GitHub Desktop.
dumb perl download accelerator
#!/usr/bin/env perl
# dumb perl download accelerator
# 10/13/21 - morb_commat_misentropic_dot_commercial
use warnings;
use strict;
use Getopt::Long;
use File::Basename;
use LWP::UserAgent;
use Parallel::ForkManager;
my $fs = 8;
my $ch = 4;
GetOptions(
"c|C|chunks=i" => \$fs,
"p|P|parallel=i" => \$ch
);
my $pm = Parallel::ForkManager->new($ch);
my $ua = LWP::UserAgent->new( agent => 'Mozilla/5.0' );
my $u = shift;
my $h = $ua->head($u) or die;
my $cl = $h->{'_headers'}{'content-length'} or die;
my $fn = basename($u);
my @ls = buildlist( $cl, $fs );
my $l = $#ls - 1;
WORK:
for ( my $i=0;$i<$#ls;$i++) {
$pm->start and next WORK;
$i ? ( $a = ( $ls[$i] + 1 ) ) : ( $a = ( $ls[$i] ) );
my $ua = LWP::UserAgent->new( agent => 'Mozilla/5.0', show_progress => 1 );
$ua->add_handler(
request_prepare => sub {
my ( $rq, $ua, $h ) = @_;
$rq->header( "Range" => "bytes=$a-$ls[($i+1)]" );
}
);
$ua->get( $u, ':content_file' => $fn . $i );
$pm->finish;
}
$pm->wait_all_children;
`cat $fn\{0..$l\} > $fn && rm $fn\{0..$l\}`;
sub buildlist {
my @ls;
my ( $cl, $fs ) = @_;
for ( my $i=0;$i<=$cl;$i=$i+int($cl/$fs) ) {
push @ls, $i;
}
push @ls, $cl if ( $ls[ ($#ls) ] != $cl );
return @ls;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment