Last active
December 11, 2017 12:49
-
-
Save speaktech-account/f553392f44d862af4959682c04def3f7 to your computer and use it in GitHub Desktop.
Archives local files to S3bucket with verifying integrity. Powershell
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Archive-FileToS3 archives local files which is older than specified days in a path to S3bucket with verifying integrity of uploaded S3objects. | |
# The verification method is to check if an eTag of S3object matches an eTag value calculated from a local file. | |
# Get-S3ETagHash calculates an eTag for a local file that should match the S3 eTag of the uploaded file. | |
# Credit goes to Sean Bamforth (https://gist.github.com/seanbamforth/9388507) and chrisdarth(https://gist.github.com/chrisdarth/02d030b31727d70d2c63) | |
function Archive-FileToS3 { | |
[cmdletbinding()] | |
Param ( | |
[Parameter(Mandatory=$true)] | |
[ValidateScript({ Test-Path $_ -PathType Container })] | |
[string]$Path, | |
[Parameter(Mandatory=$true)] | |
[ValidateScript({ $( Get-S3Bucket -BucketName $_ ) })] | |
[string]$bucketname, | |
[Parameter(Mandatory=$true)] | |
[Int32] $Days, | |
[Int32]$ConcurrentServiceRequest = 10 | |
) | |
if ($Path[$Path.Length-1] -eq "\" ){ $Path = $Path.Substring(0,$Path.Length-1) } | |
foreach ($file in $(Get-ChildItem $Path -Recurse | Where-Object{$_.Attributes -ne "directory"})){ | |
if ($file.LastWriteTime -lt ((Get-Date).AddDays(-$Days))){ | |
Write-S3Object -BucketName $bucketname -Key $file.FullName.Substring($Path.Length + 1) -File $file.FullName -ConcurrentServiceRequest $ConcurrentServiceRequest | |
$s3object = Get-S3Object -BucketName $bucketname -Key $file.FullName.Substring($Path.Length + 1) | |
$etag = $s3object.etag.Replace("`"","") | |
$hash = $(Get-S3ETagHash($file.FullName)).Hash | |
if ($etag -eq $hash){ | |
Remove-Item $file.FullName -Force | |
}else{ | |
Remove-S3Object -BucketName $bucketname -Key $file.FullName.Substring($Path.Length + 1) -Force | |
} | |
} | |
} | |
} | |
function Get-S3ETagHash { | |
[cmdletbinding()] | |
Param ( | |
[Parameter(Mandatory=$true)] | |
[ValidateScript({ Test-Path $_ -PathType Leaf })] | |
[string]$Path, | |
[Int32]$ChunkSize = 5 | |
) | |
$filename = Get-Item $Path | |
$md5 = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider | |
$blocksize = (1024*1024*$ChunkSize) | |
$startblocks = (1024*1024*16) | |
$lines = 0 | |
[byte[]] $binHash = @() | |
$reader = [System.IO.File]::Open($filename,"OPEN","READ") | |
if ($filename.length -ge $startblocks) { | |
$buf = new-object byte[] $blocksize | |
while (($read_len = $reader.Read($buf,0,$buf.length)) -ne 0){ | |
$lines += 1 | |
$binHash += $md5.ComputeHash($buf,0,$read_len) | |
} | |
$binHash=$md5.ComputeHash( $binHash ) | |
} | |
else { | |
$lines = 1 | |
$binHash += $md5.ComputeHash($reader) | |
} | |
$reader.Close() | |
$hash = [System.BitConverter]::ToString( $binHash ) | |
$hash = $hash.Replace("-","").ToLower() | |
if ($lines -gt 1) { | |
$hash = $hash + "-$lines" | |
} | |
# Output pscustomobject, equal to Get-FileHash | |
[pscustomobject]@{ | |
Algorithm = "S3ETag" | |
Hash =$hash | |
Path = $filename.FullName | |
} | |
return | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment