Created
August 23, 2012 15:19
-
-
Save neoeinstein/3437680 to your computer and use it in GitHub Desktop.
Powershell script to upload an archive to AWS Glacier with multipart retry and resume
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Aborts a multipart upload to an Amazon Web Services Glacier vault. | |
.DESCRIPTION | |
Abort-AwsGlacierUpload aborts a multipart upload to an AWS Glacier vault. After an upload is successfully aborted no further parts may be uploaded for that upload request. | |
.PARAMETER VaultName | |
Name of the Glacier vault in which the upload was initiated. | |
.PARAMETER UploadId | |
Specifies the upload id to be aborted. | |
.PARAMETER FailIfUploadNotFound | |
Normally, if an upload id is not found in the specified vault, the failure is swallowed silently. When this switch is true, the failure is propogated. | |
.PARAMETER LogFile | |
Path to a log file. | |
.PARAMETER Client | |
The Amazon Glacier client to use when creating the upload request. Encapsulates access credentials and the region endpoint. | |
.INPUTS | |
Amazon.Glacier.AmazonGlacier. The client may be passed in as a named parameter or through the pipeline. | |
.OUTPUTS | |
None. | |
.LINK | |
http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-abort-upload.html | |
.LINK | |
http://docs.amazonwebservices.com/sdkfornet/latest/apidocs/?topic=html/T_Amazon_Glacier_Model_AbortMultipartUploadRequest.htm | |
.LINK | |
New-AwsGlacierClient | |
#> | |
param( | |
[Parameter(Position=0, Mandatory=$true)] | |
[string]$VaultName=$(throw "VaultName Required"), | |
[Parameter(Position=1, Mandatory=$true)] | |
[string]$UploadId=$(throw "Upload Id to abort Required"), | |
[switch]$FailIfUploadNotFound, | |
[Parameter(Mandatory=$false)] | |
[string]$LogFile, | |
[Parameter(ValueFromPipeline=$true, Mandatory=$true)] | |
#[Amazon.Glacier.AmazonGlacier] | |
$Client) | |
Add-Type -Path "C:\Program Files (x86)\AWS SDK for .NET\bin\AWSSDK.dll" | |
Function Log | |
{ | |
param([Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=0)]$message) | |
if ($LogFile) | |
{ | |
$message | Log | |
} | |
} | |
$abortMpuRequest = New-Object Amazon.Glacier.Model.AbortMultipartUploadRequest | |
$abortMpuRequest.UploadId = $UploadId | |
$abortMpuRequest.VaultName = $VaultName | |
"Abort Multipart Upload Request" | Log | |
$abortMpuRequest | Format-List | Out-String | Log | |
Try | |
{ | |
[Void] $Client.AbortMultipartUpload($abortMpuRequest) | |
} | |
Catch [Amazon.Glacier.Model.ResourceNotFoundException] | |
{ | |
#This is often safe to ignore | |
if ($FailIfUploadNotFound) | |
{ | |
$Error[0].Exception.ToString() | Log | |
throw | |
} | |
} | |
Catch [System.Exception] | |
{ | |
$Error[0].Exception.ToString() | Log | |
throw | |
} | |
"Abort Multipart Upload Response" | Log | |
$abortMpuResponse | Format-List | Out-String | Log | |
if ($abortMpuResponse.AbortMultipartUploadResult) | |
{ | |
"Abort Multipart Upload Result" | Log | |
$abortMpuResponse.AbortMultipartUploadResult | Format-List | Out-String | Log | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Completes a multipart upload request for an Amazon Web Services Glacier vault. | |
.DESCRIPTION | |
Complete-AwsGlacierMultipartUpload finishes uploading an archive to an AWS Glacier vault. After an upload is successfully completed no further parts may be uploaded for that upload request. | |
.PARAMETER VaultName | |
Name of the vault in which the archive will be uploaded. | |
.PARAMETER UploadId | |
Specifies the upload id under which the archive is to be uploaded. | |
.PARAMETER ArchiveSize | |
Size of the uploaded archive. | |
.PARAMETER Checksum | |
The root of the SHA256 tree hash of the uploaded archive. One of Checksum or PartChecksumList is required. | |
.PARAMETER PartChecksumList | |
A list of the SHA256 tree hashes for each part of the uploaded archive. One of Checksum or PartChecksumList is required. | |
.PARAMETER LogFile | |
Path to a log file. | |
.PARAMETER Client | |
The Amazon Glacier client to use when creating the upload request. Encapsulates access credentials and the region endpoint. | |
.INPUTS | |
Amazon.Glacier.AmazonGlacier. The client may be passed in as a named parameter or through the pipeline. | |
.OUTPUTS | |
System.String. The archive id for the successfully uploaded archive. | |
.LINK | |
http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-complete-upload.html | |
.LINK | |
http://docs.amazonwebservices.com/sdkfornet/latest/apidocs/?topic=html/T_Amazon_Glacier_Model_CompleteMultipartUploadRequest.htm | |
.LINK | |
New-AwsGlacierClient | |
.LINK | |
Initiate-AwsGlacierMultipartUpload | |
#> | |
param( | |
[Parameter(Position=0, Mandatory=$true)] | |
[string]$VaultName=$(throw "VaultName is required"), | |
[Parameter(Position=1, Mandatory=$true)] | |
[string]$UploadId=$(throw "UploadId is required"), | |
[Parameter(Position=2, Mandatory=$true)] | |
[long]$ArchiveSize=$(throw "ArchiveSize is required"), | |
[Parameter(Position=3, Mandatory=$false)] | |
[string]$Checksum, | |
[Parameter(ValueFromRemainingArguments=$true, Mandatory=$false)] | |
[array]$PartChecksumList, | |
[Parameter(Mandatory=$false)] | |
[string]$LogFile, | |
[Parameter(ValueFromPipeline=$true, Mandatory=$true)] | |
#[Amazon.Glacier.AmazonGlacier] | |
$Client) | |
Add-Type -Path "C:\Program Files (x86)\AWS SDK for .NET\bin\AWSSDK.dll" | |
Function Log | |
{ | |
param([Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=0)]$message) | |
if ($LogFile) | |
{ | |
$message | Out-File $LogFile "UTF8" -NoClobber -Append | |
} | |
} | |
if (!$Checksum) | |
{ | |
if (!$PartChecksumList) | |
{ | |
throw "Either a whole-archive checksum or a list of checksums from each part is required" | |
} | |
$Checksum = [Amazon.Glacier.TreeHashGenerator]::CalculateTreeHash([String[]]$PartChecksumList) | |
} | |
$completeMpuRequest = New-Object Amazon.Glacier.Model.CompleteMultipartUploadRequest | |
$completeMpuRequest.VaultName = $VaultName | |
$completeMpuRequest.UploadId = $UploadId | |
$completeMpuRequest.ArchiveSize = $ArchiveSize | |
$completeMpuRequest.Checksum = $Checksum | |
"Complete Multipart Upload Request" | Log | |
$completeMpuRequest | Format-List | Out-String | Log | |
Try | |
{ | |
$completeMpuResponse = $Client.CompleteMultipartUpload($completeMpuRequest) | |
} | |
Catch [System.Exception] | |
{ | |
$Error[0].Exception.ToString() | Log | |
throw | |
} | |
"Complete Multipart Upload Response" | Log | |
$completeMpuResponse | Format-List | Out-String | Log | |
if ($completeMpuResponse.CompleteMultipartUploadResult) | |
{ | |
"Complete Multipart Upload Result" | Log | |
$completeMpuResponse.CompleteMultipartUploadResult | Format-List | Out-String | Log | |
} | |
$completeMpuResponse.CompleteMultipartUploadResult.ArchiveId |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Uploads an archive (possibly in parts) to an Amazon Web Services Glacier vault. | |
.DESCRIPTION | |
Do-AwsGlacierUpload uploads a file to an AWS Glaicer vault as an archive. | |
.PARAMETER VaultName | |
Name of the vault in which the archive will be uploaded. | |
.PARAMETER ArchivePath | |
Path to the file that will be uploaded as an archive. | |
.PARAMETER ArchiveDescription | |
Description of the archive for use in inventory. | |
.PARAMETER PartSize | |
Size of individual chunks used to upload the archive. Defaults to the size that will provide between 5000 and 10000 parts between 1MB and 4GB. | |
.PARAMETER ResumeUploadId | |
Resumes an in-progress upload. Specifies the upload id to resume. | |
.PARAMETER ContinueFromPart | |
When resuming an upload, specifies the next part number that should be uploaded. Uploading a part that has already been uploaded will overwrite that part. | |
.PARAMETER LogFile | |
Path to a log file. | |
.PARAMETER Client | |
The Amazon Glacier client to use when creating the upload request. Encapsulates access credentials and the region endpoint. | |
.INPUTS | |
Amazon.Glacier.AmazonGlacier. The client may be passed in as a named parameter or through the pipeline. | |
.OUTPUTS | |
System.String. The archive id of the completed upload | |
.LINK | |
http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-upload-part.html | |
.LINK | |
New-AwsGlacierClient | |
#>[CmdletBinding(DefaultParameterSetName="Initial")] | |
param( | |
[Parameter(Position=0, Mandatory=$true)] | |
[string]$VaultName=$(throw "VaultName Required"), | |
[Parameter(Position=1, Mandatory=$true)] | |
[string]$ArchivePath=$(throw "ArchivePath is required"), | |
[Parameter(Position=2, ParameterSetName="Resume", Mandatory=$true)] | |
[Parameter(Position=2, ParameterSetName="Initial", Mandatory=$false)] | |
[string]$ArchiveDescription, | |
[Parameter(Mandatory=$false)] | |
[long]$PartSize=$(.\Get-AwsGlacierMultipartUploadPartSize $ArchivePath), | |
[Parameter(ParameterSetName="Resume", Mandatory=$true)] | |
[string]$ResumeUploadId, | |
[Parameter(ParameterSetName="Resume", Mandatory=$true)] | |
[int]$ContinueFromPart, | |
[Parameter(Mandatory=$false)] | |
[string]$LogFile, | |
[Parameter(ValueFromPipeline=$true, Mandatory=$true)] | |
#[Amazon.Glacier.AmazonGlacier] | |
$Client) | |
Add-Type -Path "C:\Program Files (x86)\AWS SDK for .NET\bin\AWSSDK.dll" | |
Function IsPowerOfTwo | |
{ | |
param([long]$value) | |
($value -ne 0) -and (($value -band ($value - 1)) -eq 0) | |
} | |
if ($PartSize -gt 4GB -or !(IsPowerOfTwo $($PartSize/1MB))) | |
{ | |
throw "Part size must be a power of 2 between 1 and $(4GB/1MB)MB" | |
} | |
if (!$ResumeUploadId) | |
{ | |
Write-Host "Initiating upload" | |
$uploadId = $Client | .\Initiate-AwsGlacierMultipartUpload -VaultName:$VaultName -PartSize:$PartSize -ArchiveDescription:$ArchiveDescription -LogFile:$LogFile | |
Write-Host "Received upload id: $uploadId" | |
} | |
else | |
{ | |
Write-Host "Continuing upload id: $ResumeUploadId" | |
Write-Host "from part $ContinueFromPart" | |
$uploadId = $ResumeUploadId | |
} | |
if (!$ContinueFromPart) | |
{ | |
$ContinueFromPart = 1 | |
} | |
$result = $Client | .\Execute-AwsGlacierMultipartUpload -VaultName:$VaultName -UploadId:$uploadId -ArchivePath:$ArchivePath -PartSize:$PartSize -ContinueFromPart:$ContinueFromPart -LogFile:$LogFile | |
if ($result.Cancelled) | |
{ | |
Write-Host "Cancelling upload" | |
Write-Host "Be sure to abort upload if you wish to abandon this session" | |
$message = "Cancelled upload to Glacier Vault '{1}'.{0}Total data transferred {2:N2} GB ({3:P2}). Last successful part: {4}" -f [System.Environment]::NewLine,$VaultName,($result.TransferredBytes/1GB),($result.TransferredBytes/$result.TotalBytes),$result.TransferredParts | |
} | |
elseif ($result.Success) | |
{ | |
Write-Host "Completing upload" | |
$archiveId = $Client | .\Complete-AwsGlacierMultipartUpload -VaultName:$VaultName -UploadId:$uploadId -ArchiveSize:$($result.ArchiveSize) -PartChecksumList:$($result.PartChecksumList) -LogFile:$LogFile | |
$message = "Completed upload as archive (ID: {1}){0}To Glacier Vault '{2}'. Total data transferred: {3:N2} GB in {4} parts" -f [System.Environment]::NewLine,$archiveId,$VaultName,($result.TransferredBytes/1GB),$result.TransferredParts | |
} | |
else | |
{ | |
Write-Host "Upload Failed" | |
$message = "Failed upload to Glacier Vault '{1}'.{0}Total data transferred {2:N2} GB ({3:P2}). Last successful part: {4}" -f [System.Environment]::NewLine,$VaultName,($result.TransferredBytes/1GB),($result.TransferredBytes/$result.TotalBytes),$result.TransferredParts | |
} | |
Write-Host $message | |
$archiveId |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Uploads an archive in parts to an Amazon Web Services Glacier vault. | |
.DESCRIPTION | |
Execute-AwsGlacierMultipartUpload splits a file into parts and uploads them to an AWS Glaicer vault, which then assembles those parts into an archive. | |
Individual part failures are automatically retried up to -RetriesPerPart times. | |
.PARAMETER VaultName | |
Name of the vault in which the archive will be uploaded. | |
.PARAMETER UploadId | |
Specifies the upload id under which the archive is to be uploaded. | |
.PARAMETER ArchivePath | |
Path to the file that will be uploaded as an archive. | |
.PARAMETER PartSize | |
Size of individual chunks used to upload the archive. Must be the same as was specified when initiating the multipart upload request. | |
.PARAMETER ContinueFromPart | |
When resuming an upload, specifies the next part number that should be uploaded. Uploading a part that has already been uploaded will overwrite that part. | |
.PARAMETER RetriesPerPart | |
Number of times to retry each part before stopping in the event of a failure during upload. Defaults to 3 | |
.PARAMETER LogFile | |
Path to a log file. | |
.PARAMETER Client | |
The Amazon Glacier client to use when creating the upload request. Encapsulates access credentials and the region endpoint. | |
.INPUTS | |
Amazon.Glacier.AmazonGlacier. The client may be passed in as a named parameter or through the pipeline. | |
.OUTPUTS | |
System.String[]. A list of the hashes for each part of the archive. | |
.LINK | |
http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-upload-part.html | |
.LINK | |
http://docs.amazonwebservices.com/sdkfornet/latest/apidocs/?topic=html/T_Amazon_Glacier_Model_UploadMultipartUploadRequest.htm | |
.LINK | |
New-AwsGlacierClient | |
.LINK | |
Initiate-AwsGlacierMultipartUpload | |
#> | |
param( | |
[Parameter(Position=0, ValueFromPipelineByPropertyName=$true, Mandatory=$true)] | |
[string]$VaultName=$(throw "VaultName is required"), | |
[Parameter(Position=1, ValueFromPipelineByPropertyName=$true, Mandatory=$true)] | |
[string]$UploadId=$(throw "UploadId is required"), | |
[Parameter(Position=2, ValueFromPipelineByPropertyName=$true, Mandatory=$true)] | |
[string]$ArchivePath=$(throw "ArchivePath is required"), | |
[Parameter(Position=3, ValueFromPipelineByPropertyName=$true, Mandatory=$true)] | |
[long]$PartSize=$(throw "PartSize is required"), | |
[Parameter(Mandatory=$false)] | |
[int]$ContinueFromPart=1, | |
[Parameter(Mandatory=$false)] | |
[int]$RetriesPerPart=3, | |
[Parameter(Mandatory=$false)] | |
[string]$LogFile, | |
[Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true, Mandatory=$true)] | |
#[Amazon.Glacier.AmazonGlacier] | |
$Client) | |
Add-Type -Path "C:\Program Files (x86)\AWS SDK for .NET\bin\AWSSDK.dll" | |
$Progress = $true | |
Function Log | |
{ | |
param([Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=0)]$message) | |
if ($LogFile) | |
{ | |
$message | Out-File $LogFile "UTF8" -NoClobber -Append | |
} | |
} | |
Function Write-UploadProgress | |
{ | |
param([switch]$CalculatingHash) | |
if ($Progress) | |
{ | |
$title = "Uploading {0} to {1}" -f $ArchivePath,$VaultName | |
$time = [System.DateTimeOffset]::Now | |
$percentDone = ($currentPosition + $e.TransferredBytes)/$archiveSize | |
$timeDiff = $time - $startTime | |
$secondsRemaining = -1 | |
$kBps = [double]::NaN | |
if ($currentPosition -gt $startingPosition) | |
{ | |
if ($timeDiff.TotalSeconds -ge 60) | |
{ | |
$secondsRemaining = [int]($timeDiff.TotalSeconds * (($archiveSize - ($currentPosition + $partTransferredBytes))/($currentPosition + $partTransferredBytes - $startingPosition))) | |
} | |
if ($timeDiff.TotalSeconds -gt 0) | |
{ | |
$kBps = ($currentPosition + $partTransferredBytes - $startingPosition)/1KB/$timeDiff.TotalSeconds | |
} | |
} | |
$status = "{0:N2} of {1:N2} GB transferred ({2:P2}) at {3:N0} kBps avg" -f (($currentPosition + $partTransferredBytes)/1GB),($archiveSize/1GB),$percentDone,$kBps | |
if ($attempt -le 1) | |
{ | |
$attemptMsg = "" | |
} | |
else | |
{ | |
$attemptMsg = "[attempt {0}]" -f $attempt | |
} | |
if (!$CalculatingHash) | |
{ | |
$task = "Uploading" | |
} | |
else | |
{ | |
$task = "Calculating SHA256 for" | |
} | |
$partTitle = "{3} part {0} of {1} {2}" -f $currentPart,$totalParts,$attemptMsg,$task | |
$partPercentDone = $partTransferredBytes/$partTotalBytes | |
$partTimeDiff = $time - $partStartTime | |
$partSecondsRemaining = -1 | |
$partKBps = [double]::NaN | |
if ($partTransferredBytes -gt 0) | |
{ | |
if ($timeDiff.TotalSeconds -ge 60) | |
{ | |
$partSecondsRemaining = [int]($partTimeDiff.TotalSeconds * (($partTotalBytes - $partTransferredBytes)/$partTransferredBytes)) | |
} | |
if ($partTimeDiff.TotalSeconds -gt 0) | |
{ | |
$partKBps = $partTransferredBytes/1KB/$partTimeDiff.TotalSeconds | |
} | |
} | |
$partStatus = "{0:N2} of {1:N0} MB transferred ({2:P2}) at {3:N0} kBps avg" -f ($partTransferredBytes/1MB),($partTotalBytes/1MB),$partPercentDone,$partKBps | |
Write-Progress $title -id 1 -status $status -PercentComplete $([int]($PercentDone * 100)) -SecondsRemaining $secondsRemaining | |
Write-Progress $partTitle -id 2 -status $partStatus -PercentComplete $([int]($partPercentDone * 100)) -ParentId 1 -SecondsRemaining $partSecondsRemaining | |
} | |
} | |
Function Trap-ControlC | |
{ | |
param([switch]$Throw) | |
if ([System.Console]::KeyAvailable) | |
{ | |
$key = [System.Console]::ReadKey($true) | |
if (($key.Modifiers -band [System.ConsoleModifiers]::Control) -and ($key.Key -eq "C")) | |
{ | |
$halt = $true | |
$retVal.Cancelled = $true | |
"Halting on Control+C" | Log | |
if ($Throw) | |
{ | |
throw (New-Object System.OperationCanceledException) | |
} | |
} | |
} | |
} | |
$retVal = @{} | |
$retVal.PartChecksumList = @() | |
$retVal.Client = $Client | |
$startingPosition = $currentPosition = [long]0 | |
if ($ContinueFromPart) | |
{ | |
$startingPosition = $PartSize * ($ContinueFromPart - 1) | |
} | |
$retVal.TransferredBytes = $currentPosition | |
$retVal.TransferredParts = $ContinueFromPart - 1 | |
$archiveSize = (Get-Item $ArchivePath | Measure-Object -Property Length -Sum).Sum | |
$retVal.TotalBytes = $archiveSize | |
$totalParts = [int][System.Math]::Ceiling($archiveSize / $PartSize) | |
$retVal.TotalParts = $totalParts | |
if ($ContinueFromPart -gt $totalParts) | |
{ | |
throw "Can't continue from a part that doesn't exist. Max part: $totalParts" | |
} | |
$fileStream = New-Object System.IO.FileStream($ArchivePath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read) | |
Try | |
{ | |
[System.Console]::TreatControlCAsInput = $true | |
$startTime = $partStartTime = [System.DateTimeOffset]::Now | |
"Upload of '$ArchivePath' in $totalParts parts to $VaultName started at $startTime" | Log | |
"Upload Id: $UploadId" | Log | |
while ($currentPosition -lt $archiveSize -and !$halt) | |
{ | |
$currentPart = [int]($currentPosition / $PartSize) + 1 | |
$attempt = 1 | |
$success = $false | |
"Part $currentPart of $totalParts, Attempt 1" | Log | |
$uploadPartStream = [Amazon.Glacier.GlacierUtils]::CreatePartStream($fileStream, $PartSize) | |
$partTransferredBytes = 0 | |
$partTotalBytes = $uploadPartStream.Length | |
. Write-UploadProgress -CalculatingHash | |
$checksum = [Amazon.Glacier.TreeHashGenerator]::CalculateTreeHash($uploadPartStream) | |
if ($currentPosition -ge $startingPosition) | |
{ | |
Do | |
{ | |
if ($attempt -gt 1) | |
{ | |
"Part $currentPart of $totalParts, Attempt $attempt" | Log | |
$uploadPartStream = [Amazon.Glacier.GlacierUtils]::CreatePartStream($fileStream, $PartSize) | |
$partTransferredBytes = 0 | |
$partTotalBytes = $uploadPartStream.Length | |
. Write-UploadProgress -CalculatingHash | |
$checksum = [Amazon.Glacier.TreeHashGenerator]::CalculateTreeHash($uploadPartStream) | |
} | |
"Checksum : $checksum" | Log | |
Try | |
{ | |
$uploadMpuRequest = New-Object Amazon.Glacier.Model.UploadMultipartPartRequest | |
$uploadMpuRequest.VaultName = $VaultName | |
$uploadMpuRequest.Body = $uploadPartStream | |
$uploadMpuRequest.Checksum = $checksum | |
$uploadMpuRequest.UploadId = $UploadId | |
$uploadMpuRequest.StreamTransferProgress += ` | |
[System.EventHandler[Amazon.Runtime.StreamTransferProgressArgs]] ` | |
{ | |
param($sender,[Amazon.Runtime.StreamTransferProgressArgs]$e) | |
if ($count -eq 0 -or $e.PercentDone -eq 100) | |
{ | |
$partTransferredBytes = $e.TransferredBytes | |
$partTotalBytes = $e.TotalBytes | |
. Write-UploadProgress | |
} | |
. Trap-ControlC -Throw | |
$count = ($count + 1) % 100 | |
} | |
[Amazon.Glacier.AmazonGlacierExtensions]::SetRange($uploadMpuRequest,$currentPosition, $currentPosition + $uploadPartStream.Length - 1) | |
"Upload Multipart Part Request" | Log | |
$uploadMpuRequest | Format-List | Out-String | Log | |
$partStartTime = [System.DateTimeOffset]::Now | |
$uploadMpuResponse = $Client.UploadMultipartPart($uploadMpuRequest) | |
$success = $true | |
"Upload Multipart Part Response" | Log | |
$uploadMpuResponse | Format-List | Out-String | Log | |
if ($uploadMpuResponse.UploadMultipartPartResult) | |
{ | |
"Upload Multipart Part Result" | Log | |
$uploadMpuResponse.UploadMultipartPartResult | Format-List | Out-String | Log | |
} | |
} | |
Catch [System.Exception] | |
{ | |
if (!$halt) | |
{ | |
$err = $Error[0].Exception | |
Write-Host "Error while sending part $currentPart, attempt $attempt" | |
Write-Host $($Error[0]) -ForegroundColor Red | |
$err.ToString() | Log | |
$attempt += 1 | |
} | |
} | |
} Until ($success -or $attempt -gt $retriesPerPart -or $halt) | |
} | |
else | |
{ | |
$currentPosition = $currentPosition + $uploadPartStream.Length | |
$retVal.PartChecksumList += $checksum | |
. Trap-ControlC | |
} | |
if ($success) | |
{ | |
"Part $currentPart of $totalParts sent" | Log | |
Write-Verbose "Successfully sent part $currentPart" | |
$currentPosition = $currentPosition + $uploadPartStream.Length | |
$retVal.TransferredBytes = $currentPosition | |
$retVal.TransferredParts = $currentPart | |
$retVal.PartChecksumList += $checksum | |
} | |
elseif ($attempt -gt $retriesPerPart) | |
{ | |
"Part $currentPart of $totalParts failed!" | Log | |
$halt = $true | |
} | |
} | |
$retVal.Success = !$halt | |
} | |
Finally | |
{ | |
[System.Console]::TreatControlCAsInput = $false | |
$fileStream.Close() | |
} | |
"Finished upload of '$ArchivePath' to $VaultName at $([System.DateTimeOffset]::Now)" | Log | |
$retVal | Format-List | Out-String | Log | |
$retVal |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Creates a new multipart upload request for an Amazon Web Services Glacier vault. | |
.DESCRIPTION | |
Initiate-AwsGlacierMultipartUpload creates a new upload request to put an archive to an existing vault in an AWS Glacier account. | |
.PARAMETER VaultName | |
Name of the vault in which the archive will be uploaded. | |
.PARAMETER PartSize | |
Size of individual chunks used to upload the archive. Must be in multiples of a power of 2 times 1MB (e.g., 1MB, 2MB, 4MB, 8MB.) up to 4GB. | |
.PARAMETER ArchiveDescription | |
Description of the archive for use in inventory. | |
.PARAMETER LogFile | |
Path to a log file. | |
.PARAMETER Client | |
The Amazon Glacier client to use when creating the upload request. Encapsulates access credentials and the region endpoint. | |
.INPUTS | |
Amazon.Glacier.AmazonGlacier. The client may be passed in as a named parameter or through the pipeline. | |
.OUTPUTS | |
System.String. The upload id for the multipart archive upload request. | |
.LINK | |
http://docs.amazonwebservices.com/amazonglacier/latest/dev/api-multipart-initiate-upload.html | |
.LINK | |
http://docs.amazonwebservices.com/sdkfornet/latest/apidocs/?topic=html/T_Amazon_Glacier_Model_InitiateMultipartUploadRequest.htm | |
.LINK | |
New-AwsGlacierClient | |
#> | |
param( | |
[Parameter(Position=0, Mandatory=$true)] | |
[string]$VaultName=$(throw "VaultName Required"), | |
[Parameter(Position=1, Mandatory=$true)] | |
[long]$PartSize=$(throw "PartSize Required"), | |
[Parameter(Position=2, Mandatory=$true)] | |
[string]$ArchiveDescription=$(throw "ArchiveDescription Required"), | |
[Parameter(Mandatory=$false)] | |
[string]$LogFile, | |
[Parameter(ValueFromPipeline=$true, Mandatory=$true)] | |
#[Amazon.Glacier.AmazonGlacier] | |
$Client) | |
Add-Type -Path "C:\Program Files (x86)\AWS SDK for .NET\bin\AWSSDK.dll" | |
Function Log | |
{ | |
param([Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=0)]$message) | |
if ($LogFile) | |
{ | |
$message | Out-File $LogFile "UTF8" -NoClobber -Append | |
} | |
} | |
$initiateMpuRequest = New-Object Amazon.Glacier.Model.InitiateMultipartUploadRequest | |
$initiateMpuRequest.VaultName = $VaultName | |
$initiateMpuRequest.PartSize = $PartSize | |
$initiateMpuRequest.ArchiveDescription = $ArchiveDescription | |
"Initiate Multipart Upload Request" | Log | |
$initiateMpuRequest | Format-List | Out-String | Log | |
Try | |
{ | |
$initiateMpuResponse = $Client.InitiateMultipartUpload($initiateMpuRequest) | |
} | |
Catch [System.Exception] | |
{ | |
$Error[0].Exception.ToString() | Log | |
throw | |
} | |
"Initiate Multipart Upload Response" | Log | |
$initiateMpuResponse | Format-List | Out-String | Log | |
if ($initiateMpuResponse.InitiateMultipartUploadResult) | |
{ | |
"Initiate Multipart Upload Result" | Log | |
$initiateMpuResponse.InitiateMultipartUploadResult | Format-List | Out-String | Log | |
} | |
$initiateMpuResponse.InitiateMultipartUploadResult.UploadId |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
.SYNOPSIS | |
Creates an AWS Glacier Client | |
.DESCRIPTION | |
New-AwsGlacierClient creates a new AWS Glacier Client. | |
.PARAMETER SystemName | |
System name of the region endpoint to use. Defaults to "us-east-1". | |
.PARAMETER AwsKeyFile | |
The path to a file which contains AWS access credentials. The Access Key should be on the first line of the file, and the second should be the Secret Access Key. | |
.PARAMETER AwsAccessKeyId | |
The AWS access key to use for upload. Required if AwsKeyFile is not specified. | |
.PARAMETER AwsSecretAccessKeyId | |
The AWS secret access key to use to upload. Required if AwsKeyFile is not specified. | |
.INPUTS | |
None. Abort-AwsGlacierUpload does not accept piped objects. | |
.OUTPUTS | |
Amazon.Glacier.AmazonGlacier. The Amazon Glacier Client. | |
.LINK | |
http://docs.amazonwebservices.com/amazonglacier/latest/dev/amazon-glacier-accessing.html | |
.LINK | |
http://docs.amazonwebservices.com/sdkfornet/latest/apidocs/?topic=html/N_Amazon_Glacier.htm | |
#> | |
param( | |
[Parameter(Mandatory=$false)] | |
[string]$SystemName="us-east-1", | |
[Parameter(ParameterSetName="UsingKeyFile", Mandatory=$true)] | |
[string]$AwsKeyFile, | |
[Parameter(ParameterSetName="NotUsingKeyFile", Mandatory=$true)] | |
[string]$AwsAccessKeyId=$(if (!$awsKeyFile) {throw "AWS Access Key is required"}), | |
[Parameter(ParameterSetName="NotUsingKeyFile", Mandatory=$true)] | |
[string]$AwsSecretAccessKeyId=$(if (!$awsKeyFile) {throw "AWS Secret Access Key is required"})) | |
Add-Type -Path "C:\Program Files (x86)\AWS SDK for .NET\bin\AWSSDK.dll" | |
if ($awsSecretAccessKeyId -and $awsAccessKeyId -and $awsKeyFile) | |
{ | |
throw "Provide either an AWS Access and Secret Access Key pair or specify a file where those credentials can be found, but not both." | |
} | |
elseif (!($awsAccessKeyId -and $awsSecretAccessKeyId) -and !$awsKeyFile) | |
{ | |
throw "AWS Access and Secret Access Keys required or specify a file where those credentials can be found." | |
} | |
elseif (!($awsAccessKeyId -and $awsSecretAccessKeyId)) | |
{ | |
$content = Get-Content $awsKeyFile | |
$awsAccessKeyId = $content[0] | |
$awsSecretAccessKeyId = $content[1] | |
} | |
$RegionEndpoint = [Amazon.RegionEndpoint]::GetBySystemName($SystemName) | |
New-Object Amazon.Glacier.AmazonGlacierClient($awsAccessKeyId, $awsSecretAccessKeyId, $RegionEndpoint) |
Thank you. I was just about to embark on scripting this myself.
Execute-MPU line 210 -> Dont check final part -> Final part size use global part size and Upload Failed with error - "transfered more then archive size"
I'm add this lines for resolve this issue:
if(($archiveSize - $currentPosition) -lt $PartSize){
"Part $currentPart is final part. Calculate part size." | Log
$PartSize = ($archiveSize - $currentPosition)
}
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
File missing: Missing Get-AwsGlacierMultipartUploadPartSize.ps1