Skip to content

Instantly share code, notes, and snippets.

@DCAG
Last active January 7, 2018 21:13
Show Gist options
  • Save DCAG/7360dafd34c2c82f22be6b3b78135227 to your computer and use it in GitHub Desktop.
Save DCAG/7360dafd34c2c82f22be6b3b78135227 to your computer and use it in GitHub Desktop.
Robust function to run parallel jobs
Start-SomeParrallelJobs
{
<#
.Synopsis
Robust function to run parallel jobs
.DESCRIPTION
Can be used as a template for running jobs concurrently.
Written by: Amir Granot.
Blog : http://granola.tech
GitHub : http://github.com/DCAG
.EXAMPLE
$result = Start-SomeParrallelJobs -Servers @(1..200) -ScriptBlock {
param([int]$toSend)
Start-Sleep 3
new-object psobject -Property @{
HelpFile = New-TimeSpan -Seconds $toSend
Yoohoo = "$toSend [$using:Server]"
}
} -ArgumentsHash @(5) -ConcurrentAmount 20 -ttw 1 -WriteProgress #-JobTimeoutSec 30 -Verbose
#>
[CmdletBinding()]
param(
[Object[]]$Servers,
[ScriptBlock]$ScriptBlock,
[Object[]]$ArgumentsHash,
[int]$ConcurrentAmount = 5,
# Time to Wait
[int]$ttw = 3,
[int]$JobTimeoutSec = 30,
[switch]$WriteProgress)
Begin
{
$Jobs = @()
}#Begin
Process
{
foreach($Server in $Servers)
{
if($WriteProgress){
$percent = $Jobs.Count/$Servers.Count*100
Write-Progress -Activity "Some Activity in Progress" -Status "$percent% Complete" -CurrentOperation "Working on $Server" -PercentComplete $percent
}
$Jobs += Start-Job -ScriptBlock $ScriptBlock -Name "$Server" -ArgumentList @ArgumentsHash
Write-Verbose "Job $Server was created."
if($Jobs | ? {$_.State -eq "Completed"} -OutVariable Completed)
{
#Continue with another procedure? Change as you like...
Write-Verbose "> $($Completed.Count) Completed Jobs"
$Completed | %{
New-Object psobject -Property ([ordered]@{
JobName = $_.Name
Content = Receive-Job $_
Status = "Completed"
})
}
}#if
while(($Jobs | ? {$_.State -eq "Running"} -OutVariable Running).Count -gt $ConcurrentAmount)
{
Start-Sleep -s $ttw
Write-Verbose "> $($Running.Count) Running Jobs"
$Running | ForEach-Object{
$TimeOutPassedInterval = (Get-Date).AddSeconds(-$JobTimeoutSec)
if($_.PSBeginTime -lt $TimeOutPassedInterval)
{
Stop-Job $_
New-Object psobject -Property ([ordered]@{
JobName = $_.Name
Content = Receive-Job $_
Status = "Stopped"
})
}
}
}#while
}#foreach server in servers
while($Jobs | ? {$_.State -eq "Running"} -OutVariable Running)
{
Start-Sleep -s $ttw
Write-Verbose "> $($Running.Count) Running Jobs"
$Running | ForEach-Object{
$TimeOutPassedInterval = (Get-Date).AddSeconds(-$JobTimeoutSec)
if($_.PSBeginTime -lt $TimeOutPassedInterval)
{
Stop-Job $_
New-Object psobject -Property ([ordered]@{
JobName = $_.Name
Content = Receive-Job $_
Status = "Stopped"
})
}
}
}
if($Jobs | ? {$_.State -eq "Completed"} -OutVariable Completed)
{
#Continue with another procedure? Change as you like...
Write-Verbose "> $($Completed.Count) Completed Jobs"
$Completed | %{
New-Object psobject -Property ([ordered]@{
JobName = $_.Name
Content = Receive-Job $_
Status = "Completed"
})
}
}
}#Process
End
{
$Jobs | ? {$_.State -eq "Failed"} | ForEach-Object {
New-Object psobject -Property ([ordered]@{
JobName = $_.Name
Content = Receive-Job $_
Status = "Failed"
})
}
write-verbose "Completed jobs = $($Jobs | ?{$_.State -eq "Completed"} | Measure-Object | Select-Object -ExpandProperty Count)"
write-verbose "Failed jobs = $($Jobs | ?{$_.State -eq "Failed"} | Measure-Object | Select-Object -ExpandProperty Count)"
write-verbose "Stopped jobs = $($Jobs | ?{$_.State -eq "Stopped"} | Measure-Object | Select-Object -ExpandProperty Count)"
$Jobs | Remove-Job
}#End
}
Measure-Command {
$result = Start-SomeParrallelJobs -Servers @(1..200) -ScriptBlock {
param([int]$toSend)
Start-Sleep 3
new-object psobject -Property @{
HelpFile = New-TimeSpan -Seconds $toSend
Yoohoo = "$toSend [$using:Server]"
}
} -ArgumentsHash @(5) -ConcurrentAmount 20 -ttw 1 -WriteProgress #-JobTimeoutSec 30 -Verbose
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment