Created
October 27, 2018 10:35
-
-
Save speaktech-account/e799c82df09bebf445c0d067a8d8d625 to your computer and use it in GitHub Desktop.
The story when I verified whether the data copied to S3 is really the same as the copy source ref: https://qiita.com/speaktech/items/fd03a856b31afbb2ec73
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Archive-FileToS3 archives local files which is older than specified days in a path to S3bucket with verifying integrity of uploaded S3objects. | |
# The verification method is to check if an eTag of S3object matches an eTag value calculated from a local file. | |
# Get-S3ETagHash calculates an eTag for a local file that should match the S3 eTag of the uploaded file. | |
# Credit goes to Sean Bamforth (https://gist.github.com/seanbamforth/9388507) and chrisdarth(https://gist.github.com/chrisdarth/02d030b31727d70d2c63) | |
function Archive-FileToS3 { | |
[cmdletbinding()] | |
Param ( | |
[Parameter(Mandatory=$true)] | |
[ValidateScript({ Test-Path $_ -PathType Container })] | |
[string]$Path, | |
[Parameter(Mandatory=$true)] | |
[ValidateScript({ $( Get-S3Bucket -BucketName $_ ) })] | |
[string]$Bucketname, | |
[Parameter(Mandatory=$true)] | |
[Int32]$Days, | |
[Int32]$ConcurrentServiceRequest = 10 | |
) | |
if ($Path[$Path.Length-1] -eq "\" ){ $Path = $Path.Substring(0,$Path.Length-1) } | |
foreach ($file in $(Get-ChildItem $Path -Recurse | Where-Object{$_.Attributes -ne "directory"})){ | |
if ($file.LastWriteTime -lt ((Get-Date).AddDays(-$Days))){ | |
Write-S3Object -BucketName $Bucketname -Key $file.FullName.Substring($Path.Length + 1) -File $file.FullName -ConcurrentServiceRequest $ConcurrentServiceRequest | |
$s3object = Get-S3Object -BucketName $Bucketname -Key $file.FullName.Substring($Path.Length + 1) | |
$etag = $s3object.etag.Replace("`"","") | |
$hash = $(Get-S3ETagHash($file.FullName)).Hash | |
if ($etag -eq $hash){ | |
Remove-Item $file.FullName -Force | |
}else{ | |
Remove-S3Object -BucketName $Bucketname -Key $file.FullName.Substring($Path.Length + 1) -Force | |
} | |
} | |
} | |
} | |
function Get-S3ETagHash { | |
[cmdletbinding()] | |
Param ( | |
[Parameter(Mandatory=$true)] | |
[ValidateScript({ Test-Path $_ -PathType Leaf })] | |
[string]$Path, | |
[Int32]$ChunkSize = 5 | |
) | |
$filename = Get-Item $Path | |
$md5 = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider | |
$blocksize = (1024*1024*$ChunkSize) | |
$startblocks = (1024*1024*16) | |
$lines = 0 | |
[byte[]] $binHash = @() | |
$reader = [System.IO.File]::Open($filename,"OPEN","READ") | |
if ($filename.length -ge $startblocks) { | |
$buf = new-object byte[] $blocksize | |
while (($read_len = $reader.Read($buf,0,$buf.length)) -ne 0){ | |
$lines += 1 | |
$binHash += $md5.ComputeHash($buf,0,$read_len) | |
} | |
$binHash=$md5.ComputeHash( $binHash ) | |
} | |
else { | |
$lines = 1 | |
$binHash += $md5.ComputeHash($reader) | |
} | |
$reader.Close() | |
$hash = [System.BitConverter]::ToString( $binHash ) | |
$hash = $hash.Replace("-","").ToLower() | |
if ($lines -gt 1) { | |
$hash = $hash + "-$lines" | |
} | |
# Output pscustomobject, equal to Get-FileHash | |
[pscustomobject]@{ | |
Algorithm = "S3ETag" | |
Hash =$hash | |
Path = $filename.FullName | |
} | |
return | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# 例1)プロファイルの登録 | |
PS C:\> Set-AWSCredential -AccessKey アクセスキー -SecretKey シークレットキー -StoreAs プロファイル名 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# 例2)スクリプトのインポートと関数の利用方法 | |
PS D:\Tmp> Import-Module .\Archive-FileToS3.ps1 | |
PS D:\Tmp> Get-S3ETagHash -Path D:\Tmp\16MB.txt | |
Algorithm Hash Path | |
--------- ---- ---- | |
S3ETag eafa449afe224ad0b7f8f5bab4145d13-4 D:\Tmp\16MB.txt | |
PS D:\Tmp> Archive-FileToS3 -Path アーカイブ対象フォルダ -Bucketname バケット名 -Days 期間 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment