Skip to content

Instantly share code, notes, and snippets.

@pyh
Created January 14, 2013 15:24
Show Gist options
  • Save pyh/4530759 to your computer and use it in GitHub Desktop.
Save pyh/4530759 to your computer and use it in GitHub Desktop.
Very high-tech glazier uploader. For large files you would need these 2 patches: https://rt.cpan.org/Public/Bug/Display.html?id=81219 and https://rt.cpan.org/Public/Bug/Display.html?id=81864
#!/usr/bin/perl
use strict;
use warnings;
use Net::Amazon::Glacier;
use File::Find;
use Digest::SHA qw/sha256_hex/;
use File::Slurp;
use Net::Amazon::TreeHash;
use File::Sync qw(fsync);
our $GPG_KEY = 'key_id....';
our $backupdir = "/somedir";
our $toplevel = "tld-dir-for-glazier";
open(LOG, "<upload.journal");
my $uploaded_files = {};
while(<LOG>){
chomp;
my($hash,$amazon_id,$size,$path)=split(/\s/,$_,4);
$path=$size if !$path;
$uploaded_files->{$path}={hash=>$hash,amazon_id=>$amazon_id};
}
open(LOG, ">>upload.journal");
our $glacier = Net::Amazon::Glacier->new(
'us-east-1',
'something-read-the-docs',
'something',
);
use Data::Dumper;
print Dumper($glacier->list_vaults());
find({wanted=>\&wanted, no_chdir=>1}, $backupdir);
sub wanted {
return if /\.(pp2|db|thm|pp3|pto|mk|html|exr|swf|xml)$/i;
return if !-f $_;
if (-s $_ > 6280*1024*1024){
print "Too large $_\n";
return;
}
my $desc = $_;
$desc=~s/^\E$backupdir\Q/$toplevel/;
$desc=~s/ä/a/g;
$desc=~s/ö/o/g;
my $file = $_;
if($uploaded_files->{$desc}){
# print "$desc already uploaded\n";
return;
my $localhash = sha256_hex(read_file($file));
if ($localhash eq $uploaded_files->{$desc}->{hash}){
print "$desc already uploaded\n";
return;
}else{
die("$desc hash mishmatch\n");
}
}
print "Encrypting $file\n";
if ( encrypt($file) == 0 ){
my $content_sha = Digest::SHA->new("sha256");
$content_sha->addfile( $file );
my $localhash = $content_sha->hexdigest;
if( my $amazon_id = $glacier->upload_archive( "backups", "/tmp/$$.gpg", $desc ) ){
print LOG "$localhash $amazon_id ", -s "/tmp/$$.gpg", " $desc\n";
fsync(\*LOG);
print "Uploaded ", -s "/tmp/$$.gpg", " $localhash $amazon_id $desc\n";
}else{
die("Could not upload $desc");
}
}else{
die("Enc error: $?\n");
}
}
#encrypt();
sub encrypt {
unlink("/tmp/$$.gpg");
system("gpg", "-r", $GPG_KEY, "--output", "/tmp/$$.gpg", "--encrypt", $_[0]);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment