Skip to content

Instantly share code, notes, and snippets.

@speaktech-account
Last active December 11, 2017 12:49
Show Gist options
  • Save speaktech-account/f553392f44d862af4959682c04def3f7 to your computer and use it in GitHub Desktop.
Save speaktech-account/f553392f44d862af4959682c04def3f7 to your computer and use it in GitHub Desktop.
Archives local files to S3bucket with verifying integrity. Powershell
# Archive-FileToS3 archives local files which is older than specified days in a path to S3bucket with verifying integrity of uploaded S3objects.
# The verification method is to check if an eTag of S3object matches an eTag value calculated from a local file.
# Get-S3ETagHash calculates an eTag for a local file that should match the S3 eTag of the uploaded file.
# Credit goes to Sean Bamforth (https://gist.github.com/seanbamforth/9388507) and chrisdarth(https://gist.github.com/chrisdarth/02d030b31727d70d2c63)
function Archive-FileToS3 {
[cmdletbinding()]
Param (
[Parameter(Mandatory=$true)]
[ValidateScript({ Test-Path $_ -PathType Container })]
[string]$Path,
[Parameter(Mandatory=$true)]
[ValidateScript({ $( Get-S3Bucket -BucketName $_ ) })]
[string]$bucketname,
[Parameter(Mandatory=$true)]
[Int32] $Days,
[Int32]$ConcurrentServiceRequest = 10
)
if ($Path[$Path.Length-1] -eq "\" ){ $Path = $Path.Substring(0,$Path.Length-1) }
foreach ($file in $(Get-ChildItem $Path -Recurse | Where-Object{$_.Attributes -ne "directory"})){
if ($file.LastWriteTime -lt ((Get-Date).AddDays(-$Days))){
Write-S3Object -BucketName $bucketname -Key $file.FullName.Substring($Path.Length + 1) -File $file.FullName -ConcurrentServiceRequest $ConcurrentServiceRequest
$s3object = Get-S3Object -BucketName $bucketname -Key $file.FullName.Substring($Path.Length + 1)
$etag = $s3object.etag.Replace("`"","")
$hash = $(Get-S3ETagHash($file.FullName)).Hash
if ($etag -eq $hash){
Remove-Item $file.FullName -Force
}else{
Remove-S3Object -BucketName $bucketname -Key $file.FullName.Substring($Path.Length + 1) -Force
}
}
}
}
function Get-S3ETagHash {
[cmdletbinding()]
Param (
[Parameter(Mandatory=$true)]
[ValidateScript({ Test-Path $_ -PathType Leaf })]
[string]$Path,
[Int32]$ChunkSize = 5
)
$filename = Get-Item $Path
$md5 = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider
$blocksize = (1024*1024*$ChunkSize)
$startblocks = (1024*1024*16)
$lines = 0
[byte[]] $binHash = @()
$reader = [System.IO.File]::Open($filename,"OPEN","READ")
if ($filename.length -ge $startblocks) {
$buf = new-object byte[] $blocksize
while (($read_len = $reader.Read($buf,0,$buf.length)) -ne 0){
$lines += 1
$binHash += $md5.ComputeHash($buf,0,$read_len)
}
$binHash=$md5.ComputeHash( $binHash )
}
else {
$lines = 1
$binHash += $md5.ComputeHash($reader)
}
$reader.Close()
$hash = [System.BitConverter]::ToString( $binHash )
$hash = $hash.Replace("-","").ToLower()
if ($lines -gt 1) {
$hash = $hash + "-$lines"
}
# Output pscustomobject, equal to Get-FileHash
[pscustomobject]@{
Algorithm = "S3ETag"
Hash =$hash
Path = $filename.FullName
}
return
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment