Batch download

batchdl.ps1

####################################################################################################
# This function copies a folder (and optionally, its subfolders)
#
# When copying subfolders it calls itself recursively
#
# Requires WebClient object $webClient defined, e.g. $webClient = New-Object System.Net.WebClient
#
# Parameters:
#   $source      - The url of folder to copy, with trailing /, e.g. http://website/folder/structure/
#   $destination - The folder to copy $source to, with trailing \ e.g. D:\CopyOfStructure\
#   $recursive   - True if subfolders of $source are also to be copied or False to ignore subfolders
#   Return       - None
#
#
#   Debug
#   > Get-ExecutionPolicy
#   > Set-ExecutionPolicy RemoteSigned -Scope CurrentUser
####################################################################################################
Function Copy-Folder([string]$source, [string]$destination, [bool]$recursive) {
    if (!$(Test-Path($destination))) {
        New-Item $destination -type directory -Force
    }

    Write-Host "src: $source, dest: $destination, recur: $recursive"

    # Get the file list from the web page
    $webClient = New-Object System.Net.WebClient
    $webString = $webClient.DownloadString($source)
    # $lines = [Regex]::Split($webString, "<br>")
    $lines = [Regex]::Split($webString, "<a")
    # Parse each line, looking for files and folders
    foreach ($line in $lines) {
        if ($line.ToUpper().Contains("HREF")) {
            # File or Folder
            if (!$line.ToUpper().Contains("[TO PARENT DIRECTORY]")) {
                # Not Parent Folder entry
                $items =[Regex]::Split($line, """")
                $items = [Regex]::Split($items[2], "(>|<)")
                $item = $items[2]
              # if ($line.ToLower().Contains("&lt;dir&gt")) {
                if ($line.ToLower().Contains("/</a")) {
                
                        # Folder
                        if ($recursive) {
                            # Subfolder copy required
                            Copy-Folder "$source$item/" "$destination$item/" $recursive
                        } else {
                            # Subfolder copy not required
                        }
                } else {
                    # File
                    write-Host "src-item: $source$item dest-item: $destination$items"
                    $webClient.DownloadFile("$source$item", "$destination$item")
                }
            }
        }
    }
}




Copy-Folder -source "https://shibboleth.net/downloads/service-provider/2.5.0/" -destination "C:\Users\Mayura\Documents\Pavan\Test\" -recursive 1