batch create vm based on a predefined .csv file

example.csv

vm_batch_create.ps1

#Requires -Version 3

<#
.SYNOPSIS
    
    Script automates deployment of multiple vms loaded from pre-defined .csv file 

.DESCRIPTION

    Script reads input from .csv file 
	Script Generates log files in working directory.
    After rudimentary input sanitization (removing lines with empty fields) a separate background job (process) is started for each unique vcenter found in input. 
    The scriptblock that defines background job takes care of asynchronous creation of requested VMs (clone from template). 
    After VM is deployed scriptblock powers it on to start OS Customization process.
	After Customization, scriptblock starts hardware customization.
	After hardware Customization, starts VM guest script scriptblock. 
    Background job exits when all powered on VMs completed Customization (no matter successfully or not) or when pre-defined time-out elapses.

.PARAMETER csvfile

    Parameter indicating csv file which list vm to create.
   
.EXAMPLE

    vm_batch_create.ps1
    
    Script will interactively ask for csv file.
    
.EXAMPLE

    vm_batch_create.ps1 vmfile.csv
    
    Script will read vmfile.csv from working directory to create vms.

.EXAMPLE

    vm_batch_create.ps1 c:\temp\vm.csv
    
    Script will read c:\temp\vm.csv to create vms.

.NOTES
    Author: Xi ErDe
    Date:   Dec 18, 2017
#>

[CmdletBinding()]
Param(
   [Parameter(Mandatory=$False,Position=1)]
   [ValidateNotNullOrEmpty()]
   [string]$csvfile
)

Function Get-FileName($initialDirectory){
    [System.Reflection.Assembly]::LoadWithPartialName("System.windows.forms") | Out-Null
    
    $OpenFileDialog = New-Object System.Windows.Forms.OpenFileDialog
    $OpenFileDialog.initialDirectory = $initialDirectory
    $OpenFileDialog.filter = "CSV (*.csv)| *.csv"
    $OpenFileDialog.ShowDialog() | Out-Null
    return $OpenFileDialog.filename
}

Function Write-And-Log {
	[CmdletBinding()]
	Param(
	   [Parameter(Mandatory=$True,Position=1)]
	   [ValidateNotNullOrEmpty()]
	   [string]$LogFile,
		
	   [Parameter(Mandatory=$True,Position=2)]
	   [ValidateNotNullOrEmpty()]
	   [string]$line,

	   [Parameter(Mandatory=$False,Position=3)]
	   [int]$Severity=0,

	   [Parameter(Mandatory=$False,Position=4)]
	   [string]$type="terse"
	)

	$timestamp = (Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] "))
	$ui = (Get-Host).UI.RawUI

	switch ($Severity) {

			{$_ -gt 0} {$ui.ForegroundColor = "red"; $type ="full"; $LogEntry = $timestamp + ":Error: " + $line; break;}
			{$_ -eq 0} {$ui.ForegroundColor = "green"; $LogEntry = $timestamp + ":Info: " + $line; break;}
			{$_ -lt 0} {$ui.ForegroundColor = "yellow"; $LogEntry = $timestamp + ":Warning: " + $line; break;}

	}
	switch ($type) {
	   
			"terse"   {Write-Output $LogEntry; break;}
			"full"    {Write-Output $LogEntry; $LogEntry | Out-file $LogFile -Append; break;}
			"logonly" {$LogEntry | Out-file $LogFile -Append; break;}
		 
	}

	$ui.ForegroundColor = "white" 

}

$deployscriptblock = {
	param($vCS, $cred, $vmlist, $log, $progress)

	#simple helper object to track job progress, we will dump it to $vmname-progres.csv for the parent process to read every minute
	$job_progress = New-Object PSObject

	$job_progress | Add-Member -Name "PWROK" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "PWRFAIL" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "DPLFAIL" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "CUSTSTART" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "CUSTOK" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "CUSTFAIL" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "VMTSTART" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "VMTOK" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "VMTFAIL" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "HWSTART" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "HWOK" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "HWFAIL" -Value 0 -MemberType NoteProperty
	$job_progress | Add-Member -Name "HWPFAIL" -Value 0 -MemberType NoteProperty
	$job_progress | Export-Csv -Path $progress -NoTypeInformation

	# scriptblock is started as separate PS (not PowerCLI!), completely autonomous process, so we really need to load modules
	$Error.Clear()
	if ((Get-Module|where{$_.name -eq "VMware.VimAutomation.Core"}).name -eq "VMware.VimAutomation.Core"){
		(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: PowerCLI VimAutomation.Core Module is already enabled" | out-file $log -Append
	}
	else{
		if (Get-Module -ListAvailable|where{$_.name -eq "VMware.VimAutomation.Core"})
		{
			Get-Module -ListAvailable VMware* | Import-Module 
			if ($error.Count -eq 0) {
				(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: PowerCLI VimAutomation.Core Module was successfully enabled." | out-file $log -Append
			}
			else
			{
				(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: Could not enable PowerCLI VimAutomation.Core Module, exiting script" | out-file $log -Append
				Exit
			}
		}
		else
		{
			(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: Could not enable PowerCLI VimAutomation.Core Module, exiting script" | out-file $log -Append
			Exit
		}
	}

	# connect vcenter
	Connect-VIServer $vCS -Credential (Get-Credential $cred)
	if ($error.Count -eq 0){
		(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: vCenter $vCS successfully connected" | out-file $log -Append
	}
	else{
		(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: vCenter $vCS connection failure" | out-file $log -Append
		Exit
	}

	#hash table to store new-vm async tasks
	$newvm_tasks = @{}

	#this is needed as timestamp for searching the logs for customization events at the end of this scriptblock
	$start_events = get-date
	$started_vms = @()

	#array of customization status values and a timeout for customization in seconds (it is exactly 2hrs, feel free to reduce it)
	$Customization_In_Progress = @("CustomizationNotStarted", "CustomizationStarted", "WaitingVMtools", "VMtoolsStarted", "WaitingShutdown", "HardwareConfigStarted", "WaitingPowerOn")
	[int]$timeout_sec = 7200

	# deploy vm
	#hash table to store new-vm async tasks
	$vmname=$vmlist.name
	$newvm_tasks[(new-vm -name $vmname -template (get-template|where {$_.Name -eq $vmlist.template -and $_.FolderId -eq (Get-Folder -Type vm -Name $vmlist.'template folder'|select id).id}) -vmhost $vmlist.VMHost -oscustomizationspec $vmlist.oscust -datastore (get-datastore $vmlist.'disk1 datastore') -diskstorageformat $vmlist.'disk1 mode' -location (Get-Folder -Type vm -Name $vmlist.folder) -Notes $vmlist."server role" -RunAsync -ErrorAction SilentlyContinue).id] = $vmname
	#catch new-vm errors - if any
	if ($error.count) {
		$error[0].exception | out-file $log -Append
		$job_progress.DPLFAIL++
		$error.clear()
	}

	#track the progress of async tasks
	$running_tasks = $newvm_tasks.count
	while($running_tasks -gt 0){
		$Error.clear()
		get-task | %{
			if ($newvm_tasks.ContainsKey($_.id)){ #check if deployment of this VM has been initiated above
				if($_.State -eq "Success"){ #if deployment successful - power on!
					(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: $($newvm_tasks[$_.id]) deployed! Powering on" | out-file $log -Append
					$started_vms += (Get-VM -name $newvm_tasks[$_.id] | Start-VM -confirm:$false -ErrorAction SilentlyContinue)
					if ($error.count) { $job_progress.PWRFAIL++ }
					else {$job_progress.PWROK++}
					$newvm_tasks.Remove($_.id) #and remove task from hash table 
					$running_tasks--
				}
				elseif($_.State -eq "Error"){ #if deployment failed - only report it and remove task from hash table
					(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: $($newvm_tasks[$_.id]) NOT deployed! Skipping" | out-file $log -Append
					$newvm_tasks.Remove($_.id)
					$job_progress.PWRFAIL++
					$running_tasks--
				}
			}
		}
		#and write it down for parent process to display
		$job_progress | Export-Csv -Path $progress -NoTypeInformation
		Start-Sleep -Seconds 10	
	}

	Start-Sleep -Seconds 10
	#this is where real rock'n'roll starts, searching for customization events
	#there is a chance, not all vms power-on successfully
	$started_vms = $started_vms | where-object { $_.PowerState -eq "PoweredOn"}
	#but if they are
	if ($started_vms){
		#first - initialize helper objects to track customization, we assume customization has not started for any of successfully powered-on vms
		#exit #debug
		$vm_descriptors = New-Object System.Collections.ArrayList
		foreach ($vm in $started_vms){
				$obj = "" | select VM,CustomizationStatus,StartVMEvent 
				$obj.VM = $vm
				$obj.CustomizationStatus = "CustomizationNotStarted"
				$obj.StartVMEvent = Get-VIEvent -Entity $vm -Start $start_events | where { $_ -is "VMware.Vim.VmStartingEvent" } | Sort-object CreatedTime | Select -Last 1 
				[void]($vm_descriptors.Add($obj))
		}
	
		#timeout from here
		$start_timeout = get-date
		#now that's real mayhem - scriptblock inside scriptblock
		$continue = {
				#we check if there are any VMs left with customization in progress and if we didn't run out of time
				$vms_in_progress = $vm_descriptors | where-object { $Customization_In_Progress -contains $_.CustomizationStatus }
				$now = get-date
				$elapsed = $now - $start_timeout
				$no_timeout = ($elapsed.TotalSeconds -lt $timeout_sec)
				if (!($no_timeout) ){
					(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: Timeout waiting for customization! Manual cleanup required! Exiting..." | out-file $log -Append
				}
				return ( ($vms_in_progress -ne $null) -and ($no_timeout)) #return $true or $false to control "while loop" below
		}

		#hash table to store vmware tools upgrade async tasks
		$newvmt_tasks=@{}
		#loop searching for events
		while (& $continue){
				foreach ($vmItem in $vm_descriptors){
					$vmName = $vmItem.VM.name
					switch ($vmItem.CustomizationStatus) {

						#for every VM filter "Customization Started" events from the moment it was last powered-on
						"CustomizationNotStarted" {
							$vmEvents = Get-VIEvent -Entity $vmItem.VM -Start $vmItem.StartVMEvent.CreatedTime 
							$startEvent = $vmEvents | where { $_ -is "VMware.Vim.CustomizationStartedEvent"} 
							if ($startEvent) {
								$vmItem.CustomizationStatus = "CustomizationStarted" 
								$job_progress.CUSTSTART++
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: OS Customization for $vmName started at $($startEvent.CreatedTime)" | out-file $log -Append
							}

						break;}

						#pretty much same here, just searching for customization success / failure)
						"CustomizationStarted" {
							$vmEvents = Get-VIEvent -Entity $vmItem.VM -Start $vmItem.StartVMEvent.CreatedTime 
							$succeedEvent = $vmEvents | where { $_ -is "VMware.Vim.CustomizationSucceeded" } 
							$failedEvent = $vmEvents | where { $_ -is "VMware.Vim.CustomizationFailed"} 
							if ($succeedEvent) {
								$job_progress.CUSTOK++
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: OS Customization for $vmName completed at $($succeedEvent.CreatedTime)" | out-file $log -Append
								$vmItem.CustomizationStatus = "WaitingVMtools"
							}
							if ($failedEvent) {
								$job_progress.CUSTFAIL++
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: OS Customization for $vmName failed at $($failedEvent.CreatedTime)" | out-file $log -Append 
								$vmItem.CustomizationStatus = "WaitingVMtools"
							}

						break;} 
						
						"WaitingVMtools" {
							if((Get-VM $vmName).extensionData.Guest.ToolsStatus -eq "toolsOK" -or (Get-VM $vmName).extensionData.Guest.ToolsStatus -eq "toolsOld"){
								if ((get-vmguest $vmName).GuestFamily -eq 'windowsGuest' -and ((get-vm $vmName).ExtensionData.Guest.ToolsVersionStatus -eq 'guestToolsNeedUpgrade')){
									$newvmt_tasks[(Update-Tools -VM $vmName -RunAsync).id] = $vmName
									$job_progress.VMTSTART++
									(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM tools upgrade of VM $vmname has started. Checking for Completed Status......." | out-file $log -Append
									$vmItem.CustomizationStatus = "VMtoolsStarted"
								}
								else{
									(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmname shutdown in progress. waiting powered off......." | out-file $log -Append
									Shutdown-VMGuest -VM $vmName -Confirm:$false
									$vmItem.CustomizationStatus = "WaitingShutdown"
								}
							}
							else{
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: Waiting $vmName VM tools ready......." | out-file $log -Append
							}
						break;}

						"VMtoolsStarted" {
							get-task | %{
								if ($newvmt_tasks.ContainsKey($_.id)){
									if ($($newvmt_tasks[$_.id]) -eq $vmName){
										if($_.State -eq "Success"){
											(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $($newvmt_tasks[$_.id]) vm-tools upgraded!" | out-file $log -Append
											$newvmt_tasks.Remove($_.id)
											$job_progress.VMTOK++
											$vmItem.CustomizationStatus = "WaitingVMtools"
										}
										elseif($_.State -eq "Error"){
											(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: VM $($newvmt_tasks[$_.id]) vm-tools NOT upgrade! Skipping" | out-file $log -Append
											$newvmt_tasks.Remove($_.id)
											$job_progress.VMTFAIL++
											$vmItem.CustomizationStatus = "WaitingVMtools"
										}
									}
								}
							}
						break;}

						"WaitingShutdown" {
							if ((get-vm $vmName).PowerState -eq 'PoweredOff') {
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName has powered off. Hardware config is starting." | out-file $log -Append
								$job_progress.HWSTART++
								$vmItem.CustomizationStatus = "HardwareConfigStarted"
							}
						break;}

						"HardwareConfigStarted" {
		                    #hash table to store hardware customization tasks
		                    $newhw_tasks=@{}
							$vm = get-vm $vmName
							# resize cpu/mem
							$newhw_tasks[(set-vm -vm $vm -NumCpu $vmlist.cpu -MemoryGB $vmlist.mem -Confirm:$false).id] = $vmName
							(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName cpu/mem configed." | out-file $log -Append
							# enable ctk
							$vmview = $vm| get-view
							$vmConfigSpec = New-Object VMware.Vim.VirtualMachineConfigSpec
							$vmConfigSpec.changeTrackingEnabled = $true
							$vmview.reconfigVM($vmConfigSpec)
							(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName ctkenabled configed." | out-file $log -Append

							# extend disk1
							if ((Get-HardDisk $vmname)[0].CapacityGB -lt $vmlist.disk1){
								$newhw_tasks[((Get-HardDisk $vmname)[0] | Set-HardDisk -CapacityGB $vmlist."disk1" -Confirm:$false).id] = $vmName
                                (Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName harddisk1 extended." | out-file $log -Append
							}
                            else{
                                (Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName harddisk1 is already bigger or equal than requirement." | out-file $log -Append
                            }
							# add disk
							$disknumber=26
							for ($n=2; $n -le $disknumber; $n++)
							{
								if(($vmlist."disk$n" -ne "") -and (($vmlist |Get-Member -MemberType 'NoteProperty' | Select-Object -ExpandProperty 'Name') -contains "disk$n"))
								{
									$ds = get-datastore $vmlist."disk$n datastore"
									$newhw_tasks[(New-HardDisk -VM $vm -CapacityGB $vmlist."disk$n" -Datastore $ds -StorageFormat $vmlist."disk$n mode").id] = $vmName
                                    (Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName harddisk$n added." | out-file $log -Append
								}
								else
								{
									break
								}
							}
							# change nic1 portgroup and connect
							$nic1=Get-VirtualPortGroup -VMHost $vmlist.VMHost|where{$_.VLanId -eq $vmlist.nic1 -and (@("host") -notcontains $_.Port)}
							$newhw_tasks[(Get-NetworkAdapter $vm|where{$_.name -eq 'Network adapter 1'}|Set-NetworkAdapter -NetworkName $nic1 -StartConnected:$True -Confirm:$false).id] = $vmName
							(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName NIC1 setup completed." | out-file $log -Append
							# add nic
							$nicnumber=9
							for ($n=2; $n -le $nicnumber; $n++)
							{
								if($vmlist."nic$n" -ne "" -and (($vmlist |Get-Member -MemberType 'NoteProperty' | Select-Object -ExpandProperty 'Name') -contains "nic$n"))
								{
									$nic=Get-VirtualPortGroup -VMHost $vmlist.VMHost|where{$_.VLanId -eq $vmlist."nic$n" -and (@("host") -notcontains $_.Port)}
									$newhw_tasks[(New-NetworkAdapter -vm $vm -NetworkName $nic -Type Vmxnet3 -StartConnected).id] = $vmName
									(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName NIC$n added." | out-file $log -Append
								}
								else
								{
									break
								}
							}

							(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName hardware configed. Checking configuration errors..." | out-file $log -Append
							$hwok=0
							$hwfail=0
                            foreach ($hwtask in ($newhw_tasks.GetEnumerator()|where{$_.name -eq $vmName}).name){
                                if ((get-task -id $hwtask).state -eq "Success"){
									(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName hardware customization task $hwtask succeeded." | out-file $log -Append
								    $hwok++
                                }
                                elseif((get-task -id $hwtask).state -eq "Error"){
                                    (Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Error: VM $vmName hardware customization task $hwtask failed." | out-file $log -Append
                                    $hwfail++
                                }
                            }
							if ($hwok -ge 1 -and $hwfail -ge 1){
								$job_progress.HWPFAIL++
							}elseif ($hwok -ge 1){
								$job_progress.HWOK++
							}elseif ($hwfail -ge 1){
								$job_progress.HWFAIL++
							}
							Start-VM $vmName -confirm:$false -ErrorAction SilentlyContinue -RunAsync
							(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: Starting VM $vmName to check guest script." | out-file $log -Append
							$vmItem.CustomizationStatus = "WaitingPowerOn"

						break;}

						"WaitingPowerOn" {
							$win_setIP={
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: Windows guest $vmName started script win_setIP..." | out-file $log -Append
								$mac1=(get-vm $vmName |Get-NetworkAdapter)[0].MacAddress.ToUpper()
								if ($vmlist.ip1 -ne "")
								{
									$GuestSettings='netsh interface ipv4 set address (Get-WmiObject -Class Win32_NetworkAdapter | where{$_.macaddress -eq "'+$mac1+'"}).netconnectionid source=static address='+$vmlist.ip1+' mask='+$vmlist.mask1+' gateway='+$vmlist.gw+';'
								}
								$dnsnumber=9
								for ($n=1; $n -le $dnsnumber; $n++)
								{
									if($vmlist."dns$n" -ne "" -and (($vmlist |Get-Member -MemberType 'NoteProperty' | Select-Object -ExpandProperty 'Name') -contains "dns$n"))
									{
										$GuestSettings = $GuestSettings+'netsh interface ipv4 add dnsserver (Get-WmiObject -Class Win32_NetworkAdapter | where{$_.macaddress -eq "'+$mac1+'"}).netconnectionid address='+$vmlist."dns$n"+' index='+$n+';'
									}
									else
									{
										break
									}
								}
								$nicnumber=9
								for ($n=2; $n -le $nicnumber; $n++)
								{
									if(($vmlist |Get-Member -MemberType 'NoteProperty' | Select-Object -ExpandProperty 'Name') -contains "nic$n")
									{
										if ($vmlist."ip$n" -ne "")
										{
											$mac=(get-vm $vmName|Get-NetworkAdapter)[($n-1)].MacAddress.ToUpper()
											$GuestSettings=$GuestSettings+'netsh interface ipv4 set address (Get-WmiObject -Class Win32_NetworkAdapter | where{$_.macaddress -eq "'+$mac+'"}).netconnectionid source=static address='+$vmlist."ip$n"+' mask='+$vmlist."mask$n"+';'
										}
									}
									else
									{
										break
									}
								}
								Invoke-VMScript -ScriptText $GuestSettings -VM $vmName -GuestCredential (Get-Credential -Message "Please enter guest VM $vmname local admin user and password") -Verbose -RunAsync
							}
							$win_joinAD={
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: Windows guest $vmName started script win_joinAD..." | out-file $log -Append
								$GuestSettings = '$DCname = "dir.saicgmac.com";
									$DomainUser = "SAICGMAC\sec_v_xs574g_rw";
									$DomainPWord = ConvertTo-SecureString -String "*****" -AsPlainText -Force;
									$DomainCredential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $DomainUser, $DomainPWord;
									Start-Sleep -Seconds 20;
									Add-Computer -DomainName $DCname -Credential $DomainCredential;
									Start-Sleep -Seconds 20;
									Shutdown /r /t 0'
								# run VM script
								Invoke-VMScript -ScriptText $GuestSettings -VM $vmName -GuestCredential (Get-Credential -Message "Please enter guest VM $vmName local admin user and password") -Verbose -RunAsync
							}

							if((Get-VM $vmName).extensionData.Guest.ToolsStatus -eq "toolsOK" -or (Get-VM $vmName).extensionData.Guest.ToolsStatus -eq "toolsOld"){
								$scriptnumber=9
								for ($n=1; $n -le $scriptnumber; $n++)
								{
									if($vmlist."guest script$n" -ne "" -and (($vmlist |Get-Member -MemberType 'NoteProperty' | Select-Object -ExpandProperty 'Name') -contains "guest script$n"))
									{
										&(Get-Variable $vmlist."guest script$n").Value
										(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: Running VM $vmName guest script $n" | out-file $log -Append
									}
									else
									{
										$vmItem.CustomizationStatus = "AllDone"
										(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName guest script completed" | out-file $log -Append
										break
									}
								}
							}
							else{
								(Get-Date -Format ("[yyyy-MM-dd HH:mm:ss] ")) + ":Info: VM $vmName power on, Waiting VM tools ready......." | out-file $log -Append
							}

						break;}

					}
				}

			$job_progress | Export-Csv -Path $progress -NoTypeInformation
			Start-Sleep -Seconds 10
		}
	}
}

###############
# Main script #
###############

#variables
$ScriptRoot = Split-Path $MyInvocation.MyCommand.Path
$StartTime = Get-Date -Format "yyyyMMddHHmmss_"
if ($PSBoundParameters.ContainsKey('csvfile')){
	$vmlistfile = import-csv $csvfile
}
else{
	while ($csvfile -eq "") {$csvfile=Get-FileName}
	$vmlistfile = import-csv $csvfile
}
foreach ($vcenter in $($vmlistfile| sort-object vcenter -unique | select-object vcenter).vcenter){
	$vcentercreds += @{$vcenter=(Get-Credential -Message "Please enter vcenter $vcenter credential")}
}
$logdir = $ScriptRoot + "\Logs\"
#$transcriptfilename = $logdir + $StartTime + "vm_batch_create_Transcript.log"
$logfilename = $logdir + $StartTime + "vm_batch_create.log"

#initializing maaaany counters
[int]$total_vms = 0 
[int]$processed_vms = 0
[int]$total_errors = 0
[int]$total_dplfail = 0
[int]$total_pwrok = 0
[int]$total_pwrfail = 0
[int]$total_custstart = 0
[int]$total_custok = 0
[int]$total_custfail = 0
[int]$total_vmtstart = 0
[int]$total_vmtok = 0
[int]$total_vmtfail = 0
[int]$total_hwstart = 0
[int]$total_hwpfail = 0
[int]$total_hwfail = 0
[int]$total_hwok = 0

#test for log directory, create if needed
if ( -not (Test-Path $logdir)) {
			New-Item -type directory -path $logdir | out-null
}

#start PowerShell transcript
#Start-Transcript -Path $transcriptfilename

# Import VMware modules if not loaded
$Error.Clear()
if ((Get-Module|where{$_.name -eq "VMware.VimAutomation.Core"}).name -eq "VMware.VimAutomation.Core"){
	write-and-log $logfilename "PowerCLI VimAutomation.Core Module is already enabled" 0 "terse"
}
else{
	if (Get-Module -ListAvailable|where{$_.name -eq "VMware.VimAutomation.Core"})
	{
		Get-Module -ListAvailable VMware* | Import-Module 
		if ($error.Count -eq 0) {
			write-and-log $logfilename "PowerCLI VimAutomation.Core Module was successfully enabled." 0 "terse"
		}
		else
		{
			write-and-log $logfilename "Could not enable PowerCLI VimAutomation.Core Module, exiting script" 1 "terse"
			Exit
		}
	}
	else
	{
		write-and-log $logfilename "Could not enable PowerCLI VimAutomation.Core Module, exiting script" 1 "terse"
		Exit
	}
}


#assume everything is OK at this point
$Error.Clear()

$total_vms = $vmlistfile.count

#measuring execution time is really hip these days
$stop_watch = [Diagnostics.Stopwatch]::StartNew()

#fire background jobs for each VM
# foreach ($vmlist in $vmlistfile){
		# $vmname=$vmlist.name
		# $logfile = $logdir + $StartTime + $vmname + "-DeployJob.log"
		# $progressfile = $logdir + $vmname + "-progress.csv"
		# Write-And-Log $logfilename "Dispatching background deployment job for VM $vmname" 0 "full"
		# $jobs_tab += @{ $vmname = start-job -name $vmname -scriptblock $deployscriptblock -argumentlist $vmlist.vcenter, $vcentercreds[$vmlist.vcenter], $vmlist, $logfile, $progressfile }
# }
foreach ($vmlist in $vmlistfile){
		$vmname=$vmlist.name
		$logfile = $logdir + $StartTime + $vmname + "-DeployJob.log"
		$progressfile = $logdir + $vmname + "-progress.csv"
		Write-And-Log $logfilename "Dispatching background deployment job for VM $vmname" 0 "full"
		$jobs_tab += @{ $vmname = start-job -name $vmname -scriptblock $deployscriptblock -argumentlist $vmlist.vcenter, $vcentercreds[$vmlist.vcenter], $vmlist, $logfile, $progressfile }
}

#track the job progress + "ornaments"
do{
	#do not repeat too often
	Start-Sleep -Seconds 20
	Write-And-Log $logfilename "Pooling background deployment jobs" -1
	$running_jobs = 0
	$total_pwrok = 0
	$total_dplfail = 0
	$total_pwrfail = 0
	$total_custstart = 0
	$total_custok = 0
	$total_custfail = 0
	$total_vmtstart = 0
	$total_vmtok = 0
	$total_vmtfail = 0
	$total_hwstart = 0
	$total_hwok = 0
	$total_hwfail = 0
	$total_hwpfail = 0

	foreach ($vmlist in $vmlistfile){
		if ($($jobs_tab.Get_Item($vmlist.name)).state -eq "running") {
			$running_jobs++
		}
						
		$progressfile = $logdir +$vmlist.name + "-progress.csv"
		$jobs_progress = Import-Csv -Path $progressfile
		$total_pwrok += $jobs_progress.PWROK
		$total_dplfail += $jobs_progress.DPLFAIL
		$total_pwrfail += $jobs_progress.PWRFAIL
		$total_custstart += $jobs_progress.CUSTSTART
		$total_custok += $jobs_progress.CUSTOK
		$total_custfail += $jobs_progress.CUSTFAIL
		$total_vmtstart += $jobs_progress.VMTSTART
		$total_vmtok += $jobs_progress.VMTOK
		$total_vmtfail += $jobs_progress.VMTFAIL
		$total_hwstart += $jobs_progress.HWSTART
		$total_hwok += $jobs_progress.HWOK
		$total_hwfail += $jobs_progress.HWFAIL
		$total_hwpfail += $jobs_progress.HWFAIL
	}

	#display different progress bar depending on stage we are at (if any customization started, show customization progress, in this way we always show "worst case" progress)
	if ($total_hwstart){
		$processed_vms = $total_hwok + $total_hwfail + $total_hwpfail
		write-progress -Activity "$running_jobs background deployment jobs in progress" -Status "Percent complete $("{0:N2}" -f (($processed_vms / $total_hwstart) * 100))%" -PercentComplete (($processed_vms / $total_vms) * 100) -CurrentOperation "VM hardware config in progress"
	}elseif ($total_vmtstart){
		$processed_vms = $total_vmtok + $total_vmtfail
		write-progress -Activity "$running_jobs background deployment jobs in progress" -Status "Percent complete $("{0:N2}" -f (($processed_vms / $total_vmtstart) * 100))%" -PercentComplete (($processed_vms / $total_vmtstart) * 100) -CurrentOperation "VM-tools update in progress"
	}
	elseif ($total_custstart){
		$processed_vms = $total_custok + $total_custfail
		write-progress -Activity "$running_jobs background deployment jobs in progress" -Status "Percent complete $("{0:N2}" -f (($processed_vms / $total_pwrok) * 100))%" -PercentComplete (($processed_vms / $total_vms) * 100) -CurrentOperation "VM OS customization in progress"
	}
	else {
		$processed_vms = $total_pwrok + $total_pwrfail + $total_dplfail
		write-progress -Activity "$running_jobs background deployment jobs in progress" -Status "Percent complete $("{0:N2}" -f (($processed_vms / $total_vms) * 100))%" -PercentComplete (($processed_vms / $total_vms) * 100) -CurrentOperation "VM deployment in progress"
	}

	Write-And-Log $logfilename "Out of total $total_vms VM deploy requests there are $total_pwrok VMs successfully powered on, $($total_pwrfail + $total_dplfail) failed." $($total_pwrfail + $total_dplfail) "full"
	Write-And-Log $logfilename "Out of total $total_pwrok successfully powered on VMs OS Customization has started for $total_custstart VMs, succeeded for $total_custok VMs, failed for $total_custfail." $total_custfail "full"
	Write-And-Log $logfilename "Out of total $total_vmtstart windows VMs vm-tools update has started, succeeded for $total_vmtok VMs, failed for $total_vmtfail." $total_vmtfail "full"
	Write-And-Log $logfilename "Out of total $total_hwstart VMs haredware config has started, succeeded for $total_hwok VMs, failed for $total_hwfail, partially failed for $total_hwpfail." $( $total_hwpfail + $total_hwfail) "full"


#until we are out of active jobs
} until ($running_jobs -eq 0)

#time!
$stop_watch.Stop()
$elapsed_seconds = ($stop_watch.elapsedmilliseconds)/1000
$total_errors = $total_pwrfail + $total_custfail + $total_dplfail + $total_vmtfail + $total_hwfail

#farewell message before disconnect
Write-And-Log $logfilename "Out of total $total_vms VM deploy requests $total_pwrok VMs were successfully powered on, $($total_pwrfail + $total_dplfail) failed, $($total_vms - $total_pwrok - $total_pwrfail - $total_dplfail) duplicate VM names were detected (not deployed)." $($total_pwrfail + $total_dplfail) "full"
Write-And-Log $logfilename "Out of total $total_pwrok successfully powered on VMs OS Customization has been successful for $total_custok VMs, failed for $total_custfail." $total_custfail "full"
Write-And-Log $logfilename "Out of total $total_vmtstart windows VMs vm-tools update has started, succeeded for $total_vmtok VMs, failed for $total_vmtfail." $total_vmtfail "full"
Write-And-Log $logfilename "Out of total $total_custok successfully customized VMs hardware config has been successful for $total_hwok VMs, failed for $total_hwfail." $total_hwfail "full"
Write-And-Log $logfilename "$total_vms background deployment jobs completed in $("{0:N2}" -f $elapsed_seconds)s, $total_errors ERRORs reported, exiting." $total_errors "full"	



#Stop-Transcript