Clone the contents of an S3 bucket locally

clone_s3_bucket.ps1

Import-Module AWSPowerShell

# So our AWS Scripts work well with Proxies.
$Browser = New-Object System.Net.WebClient
$Browser.Proxy.Credentials =[System.Net.CredentialCache]::DefaultNetworkCredentials

$BucketName = 'TestBucket'
$Region = 'ex-ampleregion-1'
$TargetDirectory = 'C:\BucketOutput\'
$AllObjects = Get-S3Object $BucketName -KeyPrefix '/' -Region $Region

ForEach ($Object in $AllObjects) {
    $ItemPathParts = $Object.Key.Split('/')
    $CurrentPath = $TargetDirectory + '\'

    For($i = 0; $i -lt ($ItemPathParts.Length - 1); $i++) {
        $CurrentPath += $ItemPathParts[$i] + '\'

        if(!(Test-Path $CurrentPath)) {
            New-Item -ItemType Directory -Path $CurrentPath
        }
    }

    # Let's not attempt to create empty directories.
    # A bit of a funny one because there are really no "directories" in S3.
    if($Object.Key.Substring($Object.Key.Length - 1, 1) -ne '/') {
        Write-Host "Source File: $($Object.Key)."
        Write-Host "Target File: $($TargetDirectory + $Object.Key.Replace('/', '\'))."
        Read-S3Object $BucketName -Key $Object.Key -File ($TargetDirectory + $Object.Key.Replace('/', '\')) -Region $Region | Out-Null
    }
}