RapidClone

0.0.7

RapidClone.psm1

$global:RapidCloneVmwareSession = ""
$postVmScriptBlock = {
    $result = @{}
    $result.message = @()
    $result.messagesuccess = @()
    $result.state = "Success"
    $obj = $args[0]
    $vm = $obj.vm
    sleep $obj.delay
    try{
        $vcenter = Connect-VIServer -Server $obj.vcenter -Session $obj.session
    }catch{
        $result.message += ("Failed to connect to vcenter {0}" -f $obj.vcenter)
        $result.state = "Failed"
    }
    try{
        $resources = $obj.resources
        $mem = $obj.resources.memorymb
        $cpu = $obj.resources.numcpu
        $cores = $obj.resources.corespersocket
        if($obj.resourcechanged -and $mem -and $cpu -and $cores){
            $setvm = Set-VM -VM $vm -MemoryMB $mem -NumCpu $cpu -CoresPerSocket $cores -Confirm:$false
            while($tasksetvm.state -in "Running","Queued"){
                $tasksetvm = get-task -id $setvm.id -ErrorAction stop
            }
            $result.message += ("Reconfigured vm {0} [{1}MB][{2} cpus][{3} cores]" -f $vm,$mem,$cpu,$cores)
            $result.messagesuccess += ("Reconfigured vm {0} [{1}MB][{2} cpus][{3} cores]" -f $vm,$mem,$cpu,$cores)
        }
    }catch{
        $result.message += ("Failed to reconfigure vm {0} {1}" -f $vm,$_.Exception.Message)
        $result.state = "Failed"
    }
    try{
        if($obj.networkchanged){
            $nics = $obj.networkadapters
            $vmhost = (get-vm -Name $vm).VMHost
            $realnics = Get-NetworkAdapter -VM $vm

            # collect portgroups
            $portgroups = @{}
            $vdportgroups = @{}
            Get-VirtualPortGroup -Standard -VMHost $vmhost | %{$portgroups[$_.Name]=$_}
            Get-VDPortgroup | %{$vdportgroups[$_.Name]=$_}

            # nic count matches ?
            if($realnics.Count -eq $nics.keys.count){

                # loop nics
                foreach($k in $nics.keys){

                    $pgname = $nics[$k]
                    $nic = $realnics | ?{$_.Name -eq $k}

                    # if the nic exists
                    if($nic){

                        # different ?
                        if($nic.NetworkName -ne $nics[$k]){

                            # regular portgroup ?
                            if($portgroups.ContainsKey($pgname)){
                            
                                $setnic = Set-NetworkAdapter -NetworkAdapter $nic -Portgroup $portgroups[$pgname] -Confirm:$false
                                while($setnic.state -in "Running","Queued"){
                                    $setnic = get-task -Id $setnic.id -ErrorAction stop
                                }
                                $result.message += ("Changed portgroup on $k to {0}" -f $pgname)  
                                $result.messagesuccess += ("Changed portgroup on $k to {0}" -f $pgname)  

                            # distributed switch portgroup ?
                            }elseif($vdportgroups.ContainsKey($pgname)){
                                $setnic = Set-NetworkAdapter -NetworkAdapter $nic -Portgroup $vdportgroups[$pgname] -Confirm:$false
                                while($setnic.state -in "Running","Queued"){
                                    $setnic = get-task -Id $setnic.id -ErrorAction stop
                                }
                                $result.message += ("Changed vds portgroup on $k to {0}" -f $pgname) 
                                $result.messagesuccess += ("Changed vds portgroup on $k to {0}" -f $pgname)   
                            }else{
                                $result.message += "Unknown portgroup $pgname on $k"
                                $result.state = "Failed"
                            }
                       
                        }else{
                            $result.message += ("Network on $k is already {0}" -f $pgname) 
                        }
                    }else{
                        $result.message += "Nic $nic not found on $vm"
                        $result.state = "Failed"
                    }
                }
            }else{
               $result.message += ("Nic count mismatch on {0} [{1}]<->[{2}]" -f $vm,$realnics.Count,$nics.keys.count)
               $result.state = "Failed"            
            }
        }
    }catch{
        $result.message += ("Failed to reconfigure networkadapters on vm {0} {1}" -f $vm,$_.Exception.Message)
        $result.state = "Failed"
    }
    try{
        if($obj.startvm){
            $startvm = Start-VM -RunAsync -VM $vm
            while($taskstartvm.state -in "Running","Queued"){
                $taskstartvm = get-task -Id $startvm.id -ErrorAction stop
            }
            $result.message += ("Started vm {0}" -f $vm,$taskstartvm.state)
            $result.messagesuccess += ("Started vm {0}" -f $vm,$taskstartvm.state)
        }
    }catch{
       $result.message += ("Failed to start vm {0} {1}" -f $vm,$_.Exception.Message)
       $result.state = "Failed"
    }
    return $result
}
function Compare-Hashtable {
<#
.SYNOPSIS
Compare two Hashtable and returns an array of differences.
.DESCRIPTION
The Compare-Hashtable function computes differences between two Hashtables. Results are returned as
an array of objects with the properties: "key" (the name of the key that caused a difference),
"side" (one of "<=", "!=" or "=>"), "lvalue" an "rvalue" (resp. the left and right value
associated with the key).
.PARAMETER left
The left hand side Hashtable to compare.
.PARAMETER right
The right hand side Hashtable to compare.
.EXAMPLE
Returns a difference for ("3 <="), c (3 "!=" 4) and e ("=>" 5).
Compare-Hashtable @{ a = 1; b = 2; c = 3 } @{ b = 2; c = 4; e = 5}
.EXAMPLE
Returns a difference for a ("3 <="), c (3 "!=" 4), e ("=>" 5) and g (6 "<=").
$left = @{ a = 1; b = 2; c = 3; f = $Null; g = 6 }
$right = @{ b = 2; c = 4; e = 5; f = $Null; g = $Null }
Compare-Hashtable $left $right
#>    
[CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [Hashtable]$Left,

        [Parameter(Mandatory = $true)]
        [Hashtable]$Right        
    )
    
    function New-Result($Key, $LValue, $Side, $RValue) {
        New-Object -Type PSObject -Property @{
                    key    = $Key
                    lvalue = $LValue
                    rvalue = $RValue
                    side   = $Side
            }
    }
    [Object[]]$Results = $Left.Keys | % {
        if ($Left.ContainsKey($_) -and !$Right.ContainsKey($_)) {
            New-Result $_ $Left[$_] "<=" $Null
        } else {
            if ($Left[$_] -is [hashtable] -and $Right[$_] -is [hashtable] ) {
                Compare-Hashtable $Left[$_] $Right[$_]
            }
            else {
                $LValue, $RValue = $Left[$_], $Right[$_]
                if ($LValue -ne $RValue) {
                    New-Result $_ $LValue "!=" $RValue
                }
            }
        }
    }
    $Results += $Right.Keys | % {
        if (!$Left.ContainsKey($_) -and $Right.ContainsKey($_)) {
            New-Result $_ $Null "=>" $Right[$_]
        } 
    }
    if ($Results -ne $null) { $Results }
}
function WriteWithColor($text,$state,$color){
# -----------------------------
# fancy output with color and state
# -----------------------------
    Write-Host "[" -NoNewline
    Write-Host $state -NoNewline -ForegroundColor $color
    Write-Host ("] {0}" -f $text)
}
function ProcessVmTasks($tasks,$description,[switch]$loop,[switch]$startvm,[switch]$reconfigurevm,$postvmjobs,[switch]$ignoreCustomAttributes,[switch]$nooutput,$outputhash){
# -----------------------------
# a vm task processor
# -----------------------------
    while(($tasks.Count -ne 0) -or (($startvm -or $reconfigurevm) -and ($postvmjobs.Count -ne 0))){

        # define a list of tasks to be remove in case of completed
        $todelete = @()

        # loop all vm tasks names
        foreach($t in $tasks.Keys){
            try{

                if(-not $tasks[$t]){
                    $todelete+=$t
                    continue
                }
                # get vm task
                $task = get-task -id $tasks[$t].id -ErrorAction stop

                # if success
                if($task.State -eq "Success"){

                    if(-not $nooutput){
                        $outputhash[$t].state = "Success"
                    }

                    # do we need to start or reconfigure the vm ?
                    if($startvm -or ($reconfigurevm -and $outputhash[$t].resourcechanged) -or ($reconfigurevm -and $outputhash[$t].networkchanged)){

                        # start the vm in the background using a start-job
                        # we add a random delay to not stress the vm environment
                        $delay = random -Minimum 0 -Maximum $RandomBootDelaySeconds
                        $postvmjobs[$t] = start-job -Name "Postscript $t" `
                                                    -ScriptBlock $postVmScriptBlock `
                                                    -ArgumentList `
                                                        @{delay = $delay;
                                                          vm = $t;
                                                          vcenter = $Vcenter;
                                                          session = $global:RapidCloneVmwareSession;
                                                          resources = $outputhash[$t]["resources"];
                                                          networkadapters = $outputhash[$t]["networkadapters"];
                                                          resourcechanged = ($reconfigurevm -and $outputhash[$t].resourcechanged);
                                                          networkchanged = ($reconfigurevm -and $outputhash[$t].networkchanged);
                                                          startvm = $startvm}    
                        
                        # set as cloned
                        WriteWithColor -text ("{0} {1} [processing with delay of $delay]" -f $description,$t) -state "Cloned" -color yellow

                    }else{
                    
                        # if not, we are done, set as success
                        WriteWithColor -text ("{0} {1}" -f $description,$t) -state "Success" -color green

                    }

                    # if creating clones, we might register tags
                    if(-not $ignoreCustomAttributes){
                        $vm = get-vm $t

                        foreach($k in $outputhash[$t].customAttributes.Keys){
                            try{
                                try{
                                    $ca = Get-CustomAttribute -Name $k -TargetType VirtualMachine
                                }catch{
                                    $ca = New-CustomAttribute -Name $k -TargetType VirtualMachine
                                }
                                $out = Set-Annotation -Entity $vm -CustomAttribute $ca -Value $outputhash[$t].customAttributes[$k]
                            }catch{
                                WriteWithColor -text "Failed to set customAttribute $k" -state "Error" -color red
                            }
                        }
                    }                            

                    # mark to remove from task list, cloning is done
                    $todelete+=$t

                }elseif($task.State -ne "Running" -and $task.State -ne "Queued"){

                    # something went wrong here, it's not success nor running
                    WriteWithColor -text $t -state $task.State -color red
                    if(-not $nooutput){
                        $outputhash[$t].state = $task.State
                    }
                    # mark to remove
                    $todelete+=$t
                }

            } catch{$todelete+=$t}
        }   

        # process remove tasks
        $todelete | %{$tasks.Remove($_)} 

        # checking start jobs status
        $todelete = @()

        # we loop all started jobs
        foreach($j in $postvmjobs.Keys){
            try{

                # get the job status
                $job = get-job -id $postvmjobs[$j].id -ErrorAction stop

                # if completed
                if($job.State -eq "Completed"){

                        # get the actual result
                        $result = $job | receive-job
                        
                        if($result.State -eq "Success"){
                            WriteWithColor -text ("{0} {1}" -f $description,$j) -state "Success" -color green
                            WriteWithColor ($result.messagesuccess -join " , ") -state "Info" -color cyan
                        }else{
                            WriteWithColor -text ("{0} {1}" -f $description,$j) -state $result.State -color red
                            WriteWithColor ($result.message -join " , ") -state "Error" -color red
                        }

                        # mark to delete from jobs
                        $todelete+=$j

                }elseif($job.State -ne "Running" -and $job.State -ne "Queued"){

                    # something went wrong, nor completed, nor running
                    WriteWithColor -text $j -state $job.State -color red
                    $todelete+=$j
                }
            } catch{
                WriteWithColor -text $_.Exception.Message -state "Warning" -color yellow
                $todelete+=$j
            }
        }   

        # remove jobs
        $todelete | %{$postvmjobs.Remove($_)} 

        # do we go in a loop or not ?
        if($loop){
            sleep 1
        }else{
            break
        }
    }
}
function CheckPlugin($hostname,$hostId){
# -----------------------------
# checks if the nas plugin is installed
# -----------------------------
    $list = @()
    if($hostId){
        $vmHost = Get-VMHost -Id $hostId
    }else{
        $vmHost = Get-VMHost -Name $hostname
    }
    if($vmHost){
        $vmHostName = $vmhost.Name
        $esxcli = $vmHost | Get-EsxCli -V2
        $list = $esxcli.software.vib.list.Invoke()
        $plugin = $list| ?{$_.Name -match "NetappNasPlugin"}
        if($plugin){
            WriteWithColor -text ("[{0}] {1} version {2}" -f $vmHostName,$plugin.Name,$plugin.Version) -state "Installed" -color green
        }else{
            Throw "[$vmHostName] The NetAppNasPlugin is not installed"
        }

    }
}
function GetResourcePoolHosts($resourcepool){
    try{
        $rp = Get-ResourcePool -Name $resourcepool
        $cluster = Get-Cluster -Id $rp.ExtensionData.Owner
        return $cluster.ExtensionData.Host
    }catch{
        Throw "ResourcePool [$resourcepool] is not in a vmware cluster"
    }
}
function getNewCloneNames($nametemplate,$count,$fillGaps){

    $nametemplateNrPlaceholder = ""
    if($nametemplate -match "(#+)"){
        $nametemplateNrPlaceholder = $Matches[1]
        WriteWithColor -text "Name template [$NameTemplate]" -state "Validated" -color green
    }elseif($count -eq 1){
        WriteWithColor -text "Name template [$NameTemplate] - single instance" -state "Validated" -color green
    }else{
        throw "No proper name template [$NameTemplate], it must contain 1 or more hashtags (#) as number placeholders to make the clones unique."
    }

    if(($count -gt 1) -or $nametemplateNrPlaceholder){
        $vmSearchPattern = $nametemplate -replace $nametemplateNrPlaceholder,"*"
        $vmNumberRegex =  $nametemplate -replace $nametemplateNrPlaceholder,"([0-9]+)"

        $currentVms = @(get-vm $vmSearchPattern) | %{$_.Name}
        $freeNumbers = @()
        $takenNumbers = @()

        foreach($vm in $currentVms){
            if($vm -match $vmNumberRegex){
                $takenNumbers+=[int]($Matches[1])
            }
        }
        $freeNumbers = @(1..10000) | ?{$_ -notin $takenNumbers}
        $lastNumber = 0
        if($takenNumbers.Count -gt 0){
            $lastNumber = ($takenNumbers | Sort-Object)[-1]
        }

        if($fillGaps){
            $newNumbers = $freeNumbers | Sort-Object | select -first $count
        }else{
            $newNumbers = ($lastNumber+1)..($lastNumber+$count)
        }

        return $newNumbers | %{ $nametemplate -replace $nametemplateNrPlaceholder,([string]($_)).PadLeft($nametemplateNrPlaceholder.Length,"0") }
    }else{
        return @(,$nametemplate)
    }
}
function vmExists($name,[switch]$passthru){
    $check = $null
    try{
        $check = get-vm -Name $name
    }catch{
    }
    if(-not $passthru){
        return [bool]$check
    }else{
        return $check
    }
}
function vmHostExists($name,[switch]$passthru){
    $check = $null
    try{
        $check = Get-VMHost -Name $name
    }catch{
    }
    if(-not $passthru){
        return [bool]$check
    }else{
        return $check
    }
}
function resourcePoolExists($name,[switch]$passthru){
    $check = $null
    try{
        $check = Get-ResourcePool -Name $name
    }catch{
    }
    if(-not $passthru){
        return [bool]$check
    }else{
        return $check
    }
}
function datastoreExists($name,[switch]$passthru){
    $check = $null
    try{
        $check = Get-Datastore -Name $name
    }catch{
    }
    if(-not $passthru){
        return [bool]$check
    }else{
        return $check
    }
}
function osCustomizationSpecExists($name,[switch]$passthru){
    $check = $null
    try{
        $check = Get-OSCustomizationSpec -Name $name
    }catch{
    }
    if(-not $passthru){
        return [bool]$check
    }else{
        return $check
    }
}
function isClone($vm){
    $source = ""
    try{
        $source = $vm.CustomFields["PnC.Source"]
    }catch{}

    return([bool]$source)
}
function parseVm($vm,[switch]$ignoreErrors,[switch]$silent,[switch]$noTemplateCheck){
# -----------------------------
# checks a vm and returns a hash with common info
# -----------------------------
    $return = @{}
    $parsedvm = $null
    if(-not $noTemplateCheck){
        try{
            # is template ?
            $parsedvm = get-template -Name $vm
            $return["vmhost"] = (Get-VMHost -id ($parsedvm.HostId)).Name
            $return["istemplate"]=$true
            if(-not $silent){
                WriteWithColor -text "template [$vm]" -state "Exists" -color green
            }
        }catch{}
    }
    if(-not $parsedvm){
        try{
            # is normal vm
            $parsedvm = get-vm -name $vm
            if(-not $silent){
                WriteWithColor -text "Source vm [$vm]" -state "Exists" -color green
            }
            $return["vmhost"] = $parsedvm.VMHost.Name
            $return["istemplate"] = $false
        }catch{
            if(-not $ignoreErrors){
                Throw "VM or Template [$vm] not found"
            }
        }
    }
 
    if($parsedvm){
        # if found
        $return["name"] = $parsedvm.Name
        
        $return["resources"] = @{}
        $return["resources"]["memoryMb"] = $parsedvm.MemoryMB
        $return["resources"]["numCpu"] = $parsedvm.NumCpu
        $return["resources"]["coresPerSocket"] = $parsedvm.CoresPerSocket
        $return["folderId"] = $parsedvm.FolderId
        $return["folder"] = $parsedvm.Folder
        $return["powerstate"] = $parsedvm.PowerState
        $return["customAttributes"] = @{}
        $return["networkadapters"] = @{}
        $nics = $parsedvm |Get-NetworkAdapter
        foreach($n in $nics){
            $return["networkadapters"][$n.name] = $n.NetworkName
        }
        $return["datastore"] = ""
        $return["resourcepool"] = ""
        $return["isclone"] = $false
        $return["uuid"] = $parsedvm.ExtensionData.Config.InstanceUuid
        try{
            $currentDatastores = @($parsedvm.ExtensionData.Storage.PerDatastoreUsage)
            if(($currentDatastores.Count -gt 1) -and (-not $ignoreErrors)){
                Throw "The source vm has more than 1 datastore, rapid cloning not possible"
            }
            $return["datastore"] = (Get-Datastore -id ($currentDatastores[0].Datastore)).Name
        }catch{
            Throw "No source datastore present"
        }
        if($parsedvm.ResourcePool.Name -ne "Resources"){
            $return["resourcepool"] = $parsedvm.ResourcePool.Name
        }
        $return["customAttributes"] = @{}
        foreach($f in $parsedvm.customFields.Keys){
            $return["customAttributes"][$f]=$parsedvm.customFields[$f]
        }
        if($return["customAttributes"]["PnC.Source"]){
            $return["isclone"] = $true
        }
    }

    return $return
}
function stopVms($vms,$maxqueue,[switch]$silent,[switch]$checkClone){
    [cmdletbinding(SupportsShouldProcess = $true)]
    $stoptasks = @{}
    
    $i=0
    $total = $vms.count
    $queuefull = $false
    while($i -lt $total){
        if($stoptasks.keys.Count -le $MaxQueue){
            $queuefull = $false
            $v = $vms[$i]

            try{
                $vm = parseVm -vm $v -silent -noTemplateCheck

                if(-not $checkClone -or $vm.isclone){
                    if($vm.powerstate -eq "PoweredOn"){
                        if ($PSCmdlet.ShouldProcess($v,"Stop vm")) {
                            $stoptasks["$v"] = Stop-VM -VM $v -RunAsync -Confirm:$false
                        }
                    }else{
                        WriteWithColor -text "Clone $v not running" -state $vm.powerstate -color green
                    }
                }else{
                    WriteWithColor -text "Aborting stopping $v - not a clone - no source found" -state "Ignoring" -color yellow
                }
            }catch{
                if(-not $silent){
                    WriteWithColor -text "Aborting stopping Clone $v" -state "Unknown vm" -color red
                }
            }
            $i++
            
        }else{
            if(-not $queuefull){
                WriteWithColor -text "Queue is full [$MaxQueue]" -state "Waiting" -color cyan
            }
            $queuefull = $true
            sleep 1
        }
        ProcessVmTasks -tasks $stoptasks -description "Stopping clone" -ignoreCustomAttributes -nooutput
    }
    ProcessVmTasks -tasks $stoptasks  -description "Stopping clone" -loop -ignoreCustomAttributes -nooutput
}
function removeVms($vms,$maxqueue,[switch]$silent,[switch]$checkClone){
    [cmdletbinding(SupportsShouldProcess = $true)]
    $removetasks = @{}
    $i=0
    $total = $vms.count
    $queuefull = $false
    while($i -lt $total){
        if($removetasks.keys.Count -le $MaxQueue){
            $queuefull = $false
            $v = $vms[$i]

            try{
                $vm = parseVm -vm $v -silent -noTemplateCheck
                if(-not $checkClone -or $vm.isclone){
                    if ($PSCmdlet.ShouldProcess($v,"Remove vm")) {
                        $removetasks["$v"] = Remove-VM -VM $v -DeletePermanently -RunAsync -Confirm:$false
                    }
                }else{
                    WriteWithColor -text "Aborting removing $v - not a clone - no source found" -state "Ignoring" -color yellow
                }
            }catch{
                if(-not $silent){
                    WriteWithColor -text "Aborting removing clone $v" -state "Unknown vm" -color red
                }
            }
            $i++
            
        }else{
            if(-not $queuefull){
                WriteWithColor -text "Queue is full [$MaxQueue]" -state "Waiting" -color cyan
            }
            $queuefull = $true
            sleep 1
        }
        ProcessVmTasks -tasks $removetasks -description "Removing clone" -ignoreCustomAttributes -nooutput
    }
    ProcessVmTasks -tasks $removetasks  -description "Removing clone" -loop -ignoreCustomAttributes -nooutput
}
function initTempTemplates($datastores,$sourcevmhash){
    [cmdletbinding(SupportsShouldProcess = $true)]
    # -----------------------------
    # creates datastore-local templates for rapid cloning
    # -----------------------------
    $ttemplates = @()
    $datastores = @($datastores)
    $cDatastore = $sourcevmhash.datastore
    $cHost = $sourcevmhash.vmhost
    $cIstemplate = $sourcevmhash.istemplate
    try{
        foreach($datastore in $datastores){

            if($datastore -eq $cDatastore){
                WriteWithColor -text "[$datastore] is the source datastore, rapid cloning" -state "Info" -color green
            }else{
                WriteWithColor -text "[$datastore] is not the source datastore, a slow copy is needed first" -state "Info" -color yellow
            }
            $copyName = "{0}__{1}__{2}" -f $datastore,$sourcevmhash.name,(get-date -Format "yyyyMMdd")
            $ttemplates+=$copyName

            if((vmExists -name $copyName) -and $ReuseExistingTemporaryTemplates){
                WriteWithColor -text "Re-using existing temporary template [$copyName]" -state "Success" -color green
            }else{
                removeVms -vms @($copyName) -maxqueue 50 -silent -whatif:$WhatIfPreference -confirm:$ConfirmPreference
                if($cIstemplate){
                    if ($PSCmdlet.ShouldProcess($cHost,"Create vm $copyname")) {
                        $task = New-Vm -VMHost $cHost -Datastore $datastore -Template $sourcevmhash.name -Name $copyName -RunAsync
                    }
                }else{
                    if ($PSCmdlet.ShouldProcess($cHost,"Create vm $copyname")) {
                        $task = New-Vm -VMHost $cHost -Datastore $datastore -VM $sourcevmhash.name -Name $copyName -RunAsync
                    }
                }
                if ($PSCmdlet.ShouldProcess($cHost,"Wait for temporary template $copyname to complete")) {
                    $out = $task | Wait-Task
                    WriteWithColor -text "Temporary template [$copyName]" -state "Success" -color green
                }
            }
        }
    }catch{
        WriteWithColor -text $_.Exception.Message -state "Error" -color red
        WriteWithColor -text $_.Exception.InnerException.Message -state "Error" -color red
        if($RemoveTemporaryTemplates){
            removeVms -vms @($ttemplates) -maxqueue 50 -silent -whatif:$WhatIfPreference -confirm:$ConfirmPreference
        }
        Throw "Something went wrong in preparing templates"
    }
    return $ttemplates
}
function cloneVm($name,$vmhost,$resourcepool,$oscustomizationspec,$sourcevm,$datastore,$istemplate){
    [cmdletbinding(SupportsShouldProcess = $true)]
    $expression = "New-VM -Name $name"

    if($istemplate){
        $expression += " -Template"
    }else{
        $expression += " -VM"
    }
    $expression += " $sourcevm -Datastore $datastore -RunAsync"

    if($resourcepool){
        $expression += " -ResourcePool $resourcepool"
    }else{
        $expression += " -VMHost $vmhost"
    }

    if($OSCustomizationSpec){
        $expression += " -OsCustomizationSpec $oscustomizationspec"
    }
    if ($PSCmdlet.ShouldProcess(("{0}{1}" -f $resourcepool,$vmhost),"Create vm $name")) {
        Write-Verbose "Invoking : $expression"
        return Invoke-Expression -Command $expression
    }else{
        return $null
    }
}
function connectVcenter($vcenter,$credentials,$session){
    if($session){
        $VcenterServer = Connect-VIServer $vcenter -Session $session
        WriteWithColor -text "Vcenter Connection [$vcenter] (with session)" -state "Connected" -color green
    }elseif($credentials){
        $VcenterServer = Connect-VIServer $vcenter -Credential $Credentials
        WriteWithColor -text "Vcenter Connection [$vcenter] (with credentials)" -state "Connected" -color green
    }else{
        $VcenterServer = Connect-VIServer $vcenter
        WriteWithColor -text "Vcenter Connection [$vcenter] (SSO)" -state "Connected" -color green
    }
    if($session){
        $global:RapidCloneVmwareSession = $session
    }else{
        $global:RapidCloneVmwareSession = $VcenterServer.SessionId
    }
}
function disconnectVcenter($Vcenter,$session){
    if(-not $Session){
        $out = Disconnect-VIServer -Server $vcenter -Confirm:$false
        WriteWithColor -text "Vcenter Connection [$Vcenter]" -state "Disconnected" -color green
    }
}

<#
.Synopsis
   Rapidly clones multiple vm's by leveraging VAAI and the NetApp Nas Plugin.
.DESCRIPTION
   Using NetApps Nas Plugin and the vmware VAAI technology,
   vmware can leverage NetApp file cloning technology
   and thus rapidly (in seconds) clone vm's.
   This cmdlet allows you to easily clone a source vm or a source template.
   When rapid cloning is not possible (for example template not on
   then same datastore), this cmdlet will optimize the cloning process
   by distributing the source template to each datastore and then
   use rapid cloning withing the same datastore.
.EXAMPLE
    New-RapidClone -Vcenter jabba.slash.local -Credentials $credentials -SourceVm centostemplate -NameTemplate "test_##" -Count 3 -StartVM -ResourcePool RP1 -NoNetappNasPluginCheck
 
    [Connected] Vcenter Connection [jabba.slash.local]
    [Exists] Source vm [centostemplate]
    [Exists] ResourcePool is provided [RP1]
    [Validated] Name template [test_##]
    [Info] Inheriting datastore from source [r2d2-ds01]
    [Info] [r2d2-ds01] is the source datastore, rapid cloning
    [Success] Temporary template [r2d2-ds01__centostemplate__20200518]
    [Cloned] Adding clone test_01 [starting with delay of 11]
    [Cloned] Adding clone test_02 [starting with delay of 3]
    [Cloned] Adding clone test_03 [starting with delay of 50]
    [Started] test_02
    [Started] test_01
    [Started] test_03
    [Success] Removing clone r2d2-ds01__centostemplate__20200518
    [Disconnected] Vcenter Connection [jabba.slash.local]
 
.OUTPUTS
   The output is a list objects with the clone information. Pipe it for example to convertto-csv
   and out-file to save the results in a CSV-file.
#>
function New-RapidClone{
    [cmdletbinding(DefaultParameterSetName='Using Credentials',
                   SupportsShouldProcess = $true)]
    param(

        # Vcenter server
        [Parameter(Mandatory=$true)]
        [string]$Vcenter,

        # PSCredentials, if non given, we assume the current user has direct access.
        [Parameter(Mandatory=$false,ParameterSetName="Using Credentials")]
        [PSCredential]$Credentials,

        # An existing vmware session (instead of credentials)
        [Parameter(Mandatory=$true,ParameterSetName="Using Session")]
        [String]$Session,

        # The source vm name or template name
        [Parameter(Mandatory=$true)]
        [string]$SourceVm,

        # The naming convention template. Add hashtags (#) where you want auto numbering.
        # You can omit hashtags if you only deploy 1 clone
        [Parameter(Mandatory=$true)]
        [string]$NameTemplate,

        # Number of clones to create
        [Parameter(Mandatory=$true)]
        [int]$Count,

        # Maximum vm tasks that can run at the same time
        [Parameter(Mandatory=$false)]
        [int]$MaxQueue = 50,

        # The networkname for the first network card of the vm
        [Parameter(Mandatory=$false)]
        [string]$NetworkName,

        # Indicates if the clones need to be booted.
        # Booting happens in the background and a random delay is added to not boot all clones at once
        [switch]$StartVM=$false,

        # This will not re-create temorary templates, but re-use existing ones if they exist
        # When running tests, they might gain time, as you slow copied templates can be re-used.
        # Use in combination with switch RemoveTemporaryTemplates, by setting it to $false.
        [switch]$ReuseExistingTemporaryTemplates,

        # Remove temporary template after execution
        # When running tests, you may turn these off to avoid re-copying templates
        # Use the ReuseExistingTemporaryTemplates flag to re-use them
        [switch]$RemoveTemporaryTemplates=$true,

        # The maximum random boot delay in seconds. If you create more clones, you could make this higher to spread the boot processes.
        [int]$RandomBootDelaySeconds=60,

        # The script will automatically search for the last number and continue from there.
        # This switch will start from 1 instead and start filling gaps.
        [switch]$FillGaps,

        # The script does a Netapp Nas Plugin check. With this flag you can omit this check
        # If you pass a resourcepool, we assume the resourcepool is on a vmware cluster and we will check every host in the cluster
        [switch]$NoNetappNasPluginCheck,
    
        # A list of datastore over which the script should spread the clones.
        # If non are given. The datastore where the source vm lives is used.
        [string[]]$Datastores,

        # The destination VMHost
        # You can also choose for a resource pool
        # If both are missing, the VMHost of the source VM is used.
        [string]$VMHost,

        # The destination ResourcePool
        # You can also choose for a single VMHost
        # If both are missing, the VMHost of the source VM is used.
        [string]$ResourcePool,

        # OSCustomizationSpec
        [string]$OSCustomizationSpec
    )
    
    Begin{

        $customAttributesHash = @{} 
        $ErrorActionPreference = "stop"

        # ----------------------------------
        # Connecting to vcenter
        # ----------------------------------
        try{
            connectVcenter -Vcenter $Vcenter -Credentials $Credentials -Session $Session
            
        }catch{
            Throw "Failed to connect to vcenter $Vcenter"
        }
        if (-not $PSBoundParameters.ContainsKey('Confirm')) {
            $ConfirmPreference = $PSCmdlet.SessionState.PSVariable.GetValue('ConfirmPreference')
        }
        if (-not $PSBoundParameters.ContainsKey('WhatIf')) {
            $WhatIfPreference = $PSCmdlet.SessionState.PSVariable.GetValue('WhatIfPreference')
        }

    }
    Process{
        try{

            $currentHost = ""
            $currentDatastore = ""
            $sourceVmIsTemplate=$false

            # ----------------------------------
            # Checking source vm
            # ----------------------------------
            $vmhash = parseVm -vm $SourceVm
            $currentHost = $vmhash.vmhost
            $currentDatastore = $vmhash.datastore
            $sourceVmIsTemplate = $vmhash.istemplate

            # ----------------------------------
            # check vm host
            # ----------------------------------
            if($VMHost){
                if(vmHostExists -name $VMHost){
                    WriteWithColor -text "VMHost is provided [$VMHost]" -state "Exists" -color green
                }else{
                    Throw "The provided VMHost [$VMHost] is not found."
                }
            }
                
            # ----------------------------------
            # check resource pool
            # ----------------------------------
            if($ResourcePool){
                if(resourcePoolExists -name $ResourcePool){
                    WriteWithColor -text "ResourcePool is provided [$ResourcePool]" -state "Exists" -color green
                }else{
                    Throw "The provided ResourcePool [$ResourcePool] is not found."
                }
            }
                
            # ----------------------------------
            # fall back to template resources
            # ----------------------------------
            if(-not $VMHost -and -not $ResourcePool){
                WriteWithColor -text "Inheriting destination from source [$currentHost]" -state "Info" -color cyan
                $VMHost = $currentHost
            }

            # ----------------------------------
            # check if Netapp Nas Plugin installed
            # ----------------------------------
            if(-not $NoNetappNasPluginCheck -and $VMHost){
                CheckPlugin -hostname $VMHost
            }
            if(-not $NoNetappNasPluginCheck -and $ResourcePool){
                $rpHosts = GetResourcePoolHosts -resourcepool $ResourcePool
                $rpHosts | %{CheckPlugin -hostid $_}
            }
            # ----------------------------------
            # check OSCustomizationSpec
            # ----------------------------------
            if($OSCustomizationSpec){
                if(osCustomizationSpecExists -name $OSCustomizationSpec){
                    WriteWithColor -text "OSCustomizationSpec is provided [$OSCustomizationSpec]" -state "Exists" -color green
                }else{
                    Throw "The provided OSCustomizationSpec [$OSCustomizationSpec] is not found."
                }
            }

            # ----------------------------------
            # Getting new free auto increment names
            # ----------------------------------
            $cloneNames = @(getNewCloneNames -nametemplate $NameTemplate -count $Count -fillGaps $FillGaps)

            # ----------------------------------
            # searching and checking datastores
            # ----------------------------------
            if($Datastores){
                foreach($datastore in $Datastores){
                    if(datastoreExists -name $datastore){
                        WriteWithColor -text "Datastore [$datastore]" -state "Exists" -color green
                    }else{
                        Throw "Datastore [$datastore] not found"
                    }
                }

            }else{
                WriteWithColor -text "Inheriting datastore from source [$currentDatastore]" -state "Info" -color cyan
                $Datastores = @($currentDatastore)
            }

            #####################################
            # AT THIS POINT EVERYTHING IS CHECKED
            # NOW THE REAL COPYING STARTS
            # FIRST WE WILL MAKE SURE THAT EVERY
            # DATASTORE HAS A COPY
            #####################################
            $tempTemplates = @()
            $tempTemplates+=initTempTemplates -datastores $Datastores -sourcevmhash $vmhash -whatif:$WhatIfPreference -confirm:$ConfirmPreference

            # ----------------------------------
            # Start cloning
            # ----------------------------------
            $clonetasks = @{}
            $postvmjobs = @{}
            $outputhash = @{}

            # Getting custom attributes
            $customAttributes = @{}
            $customAttributes["PnC.Source"] = $vmhash.uuid
            $customAttributes["PnC.NameTemplate"] = $NameTemplate
            $customAttributes["PnC.Deployed"] = get-date
            $customAttributes["PnC.CustSpec"] = $OSCustomizationSpec
          
            $i=0
            $total = $cloneNames.count
            $roundrobin=0
            $queuefull = $false
            $reconfigurevm = $false

            # create all clones
            while($i -lt $total){
                if($clonetasks.Keys.Count -le $MaxQueue){
                    $queuefull = $false
                    $clonename = $cloneNames[$i]
                    $i++
                    $pct = [int]($i/$total*100)
                    Write-Progress -Activity "Cloning" -Status "Adding $clonename" -PercentComplete $pct

                    # round robin get template & datastore
                    $thisdatastore = $Datastores[$roundrobin]
                    $thistemplate = $tempTemplates |?{([string]$_).StartsWith("{0}__" -f $thisdatastore)}
                    $roundrobin++
                    if($roundrobin -ge $Datastores.Count){
                        $roundrobin=0
                    }

                    # set outputhash
                    $outputhash[$clonename]=@{}
                    $outputhash[$clonename]["vm"] = $clonename
                    $outputhash[$clonename]["vmhost"] = $VMHost
                    $outputhash[$clonename]["datastore"] = $thisdatastore
                    $outputhash[$clonename]["parenttemplate"] = $SourceVm
                    $outputhash[$clonename]["nametemplate"] = $NameTemplate
                    $outputhash[$clonename]["oscustomizationspec"] = $OSCustomizationSpec
                    $outputhash[$clonename]["resourcepool"] = $ResourcePool
                    $outputhash[$clonename]["customattributes"] = $customAttributes
                    $outputhash[$clonename]["networkadapters"]=$vmhash.networkadapters
                    if($NetworkName){
                        $firstnic = @($vmhash.networkadapters.Keys) | sort-object | select -First 1
                        $outputhash[$clonename]["networkadapters"][$firstnic]=$NetworkName
                        $outputhash[$clonename].networkchanged = $true
                        $reconfigurevm = $true
                    }

                    # invoke the clone
                
                    $clonetasks["$clonename"] = cloneVm -name $clonename -vmhost $VMHost -resourcepool $ResourcePool -oscustomizationspec $OSCustomizationSpec -sourcevm $thistemplate -datastore $thisdatastore -whatif:$WhatIfPreference -confirm:$ConfirmPreference
                }else{
                    if(-not $queuefull){
                        WriteWithColor -text "Queue is full [$MaxQueue]" -state "Waiting" -color cyan
                    }
                    $queuefull = $true
                    sleep 1
                }

                # process completed tasks
                ProcessVmTasks -tasks $clonetasks -description "Adding clone" -postvmjobs $postvmjobs -startvm:$StartVM  -outputhash $outputhash -reconfigurevm:$reconfigurevm
            }
            # process final tasks
            Write-Progress -Activity "Adding clone task" -Status "Done" -Completed
            ProcessVmTasks -tasks $clonetasks -description "Adding clone" -postvmjobs $postvmjobs -startvm:$StartVM -loop  -outputhash $outputhash -reconfigurevm:$reconfigurevm

        }catch{
            WriteWithColor -text $_.Exception.Message -state "Error" -color red
            if($_.Exception.InnerException.Message){
                WriteWithColor -text $_.Exception.InnerException.Message -state "Error" -color red
            }
        }
    }
    End{
        try{

            # remove templates if required
            if($RemoveTemporaryTemplates){
                removeVms -vms @($tempTemplates) -maxqueue $MaxQueue -silent -whatif:$WhatIfPreference -confirm:$ConfirmPreference
            }

            # disconnect
            disconnectVcenter -Vcenter $Vcenter -session $Session
        }catch{}

        # send to output stream for further piping
        foreach($h in $outputhash.Keys){
            New-Object -TypeName PSObject -Property $outputhash[$h]
        }    
    }
}

<#
.Synopsis
   Rapidly re-clones multiple vm-clones by leveraging VAAI and the NetApp Nas Plugin.
.DESCRIPTION
   Using NetApps Nas Plugin and the vmware VAAI technology,
   vmware can leverage NetApp file cloning technology
   and thus rapidly (in seconds) clone vm's.
   This cmdlet allows you to easily re-clone previously clones vm's.
   Using tags on the vm's, we remember the original template
   and re-deploy it with the same name.
   When rapid cloning is not possible (for example template not on
   then same datastore), this cmdlet will optimize the cloning process
   by distributing the source template to each datastore and then
   use rapid cloning withing the same datastore.
.EXAMPLE
    Update-RapidClone -Vcenter jabba.slash.local -Credentials $credentials -StartVM -Names (get-vm test*).Name
 
    [Connected] Vcenter Connection [jabba.slash.local]
    [Success] Stopping clone test_01
    [Success] Stopping clone test_02
    [Success] Stopping clone test_03
    [Success] Removing clone test_01
    [Success] Removing clone test_02
    [Success] Removing clone test_03
    [Exists] Source vm [centostemplate]
    [Info] [r2d2-ds01] is the source datastore, rapid cloning
    [Success] Temporary template [r2d2-ds01__centostemplate__20200518]
    [Cloned] Adding clone test_01 [starting with delay of 12]
    [Cloned] Adding clone test_02 [starting with delay of 34]
    [Cloned] Adding clone test_03 [starting with delay of 15]
    [Started] test_01
    [Started] test_03
    [Started] test_02
    [Success] Removing clone r2d2-ds01__centostemplate__20200518
    [Disconnected] Vcenter Connection [jabba.slash.local]
.OUTPUTS
   The output is a list objects with the clone information. Pipe it for example to convertto-csv
   and out-file to save the results in a CSV-file.
#>
function Update-RapidClone{

    [cmdletbinding(DefaultParameterSetName='Using Credentials',
                   SupportsShouldProcess = $true)]
    param(

        # Vcenter server
        [Parameter(Mandatory=$true)]
        $Vcenter,

        # PSCredentials, if non given, we assume the current user has direct access.
        [Parameter(Mandatory=$false,ParameterSetName="Using Credentials")]
        [PSCredential]$Credentials,

        # An existing vmware session (instead of credentials)
        [Parameter(Mandatory=$true,ParameterSetName="Using Session")]
        [String]$Session,

        # Indicates if the clones need to be booted.
        # Booting happens in the background and a random delay is added to not boot all clones at once
        [switch]$StartVM=$false,

        # Indicates if the clones need to be reconfigured to match cpu & memory of the previous clone.
        # Reconfiguring happens in the background and a random delay is added to not reconfigure all clones at once.
        [switch]$ReconfigureVm=$true,

        # Maximum vm tasks can run at the same time
        [Parameter(Mandatory=$false)]
        [int]$MaxQueue = 50,

        # This will not re-create temorary templates, but re-use existing ones if they exist
        # When running tests, they might gain time, as you slow copied templates can be re-used.
        # Use in combination with switch RemoveTemporaryTemplates, by setting it to $false.
        [switch]$ReuseExistingTemporaryTemplates,

        # Remove temporary template after execution
        # When running tests, you may turn these off to avoid re-copying templates
        # Use the ReuseExistingTemporaryTemplates flag to re-use them
        [switch]$RemoveTemporaryTemplates=$true,

        # The maximum random boot delay in seconds. If you create more clones, you could make this higher to spread the boot processes.
        [int]$RandomBootDelaySeconds=60,

        # An array of vm names to be re-deployed
        [Parameter(Mandatory=$true)]
        [string[]]$Names
    )

    Begin{

        $ErrorActionPreference = "stop"

        # ----------------------------------
        # Connecting to vcenter
        # ----------------------------------
        try{
            ConnectVcenter -Vcenter $Vcenter -Credentials $Credentials -session $Session
        }catch{
            Throw "Failed to connect to vcenter $Vcenter"
        }
        if (-not $PSBoundParameters.ContainsKey('Confirm')) {
            $ConfirmPreference = $PSCmdlet.SessionState.PSVariable.GetValue('ConfirmPreference')
        }
        if (-not $PSBoundParameters.ContainsKey('WhatIf')) {
            $WhatIfPreference = $PSCmdlet.SessionState.PSVariable.GetValue('WhatIfPreference')
        }

    }
    Process{
        try{

            $postvmjobs = @{}
            $clonetasks = @{}
            $templateshash = @{}
            $vmlistbyuuid = @{}
            $vmhashlist = @{}

            # we analyze all passed vm's
            WriteWithColor -text "Analyzing passed vm's" -state "Info" -color cyan
            $Names | %{$vmhashlist[$_] = parseVm -vm $_ -ignoreErrors -silent -noTemplateCheck}

            $tempTemplates = @()
            $sourcevmhashes = @{}
            $outputhash = @{}

            # get a list of all vm names by uuid
            WriteWithColor -text "Making inventory of all vm's" -state "Info" -color cyan
            get-vm | %{$uuid = $_.ExtensionData.Config.InstanceUuid ; $vmlistbyuuid[$uuid]=$_.Name}
            get-template | %{$uuid = $_.ExtensionData.Config.InstanceUuid ; $vmlistbyuuid[$uuid]=$_.Name}

            # remove bad templates
            WriteWithColor -text "Checking valid source information" -state "Info" -color cyan

            $todelete = @()
            foreach($v in $vmhashlist.Keys){
                $vmhash = $vmhashlist[$v]
                if(-not $vmhash.name){
                    WriteWithColor -text ("{0} does not exist" -f $v) -state "Warning" -color yellow
                    $todelete+= $v                    
                }elseif($vmhash.isclone){
                    $parenttemplate = $vmlistbyuuid[$vmhash.customAttributes["PnC.Source"]]
                    if(-not $parenttemplate){
                        WriteWithColor ("{0} has a bad source guid" -f $v) -state "Error" -color red
                        $todelete+= $v
                    }
                }else{
                    WriteWithColor -text ("{0} is not a clone - no source found" -f $v) -state "Warning" -color yellow
                    $todelete+= $v
                }
            }
            $todelete | %{$vmhashlist.Remove($_)} 

            # find all template-datastore combinations
            if($vmhashlist.Keys.Count -gt 0){
                foreach($v in $vmhashlist.Keys){
                    $vmhash = $vmhashlist[$v]
                    $parenttemplate = $vmlistbyuuid[$vmhash.customAttributes["PnC.Source"]]
                    $datastore = $vmhash.datastore

                    if(-not $templateshash.ContainsKey($parenttemplate)){
                        WriteWithColor -text "Found template $parenttemplate" -state "Info" -color cyan
                        $templateshash[$parenttemplate] = @{}
                        $templateshash[$parenttemplate]["data"] = parseVm -vm $parenttemplate
                        $templateshash[$parenttemplate]["datastores"] = @()
                    }

                    if($datastore -notin $templateshash[$parenttemplate]["datastores"]){
                        WriteWithColor -text "Found Datastore $datastore" -state "Info" -color cyan
                        $templateshash[$parenttemplate]["datastores"] += $datastore
                    }

                }   

                # prep all template-datastore combinations for rapid cloning
                foreach($k in $templateshash.Keys){
                    $tempTemplates+=initTempTemplates -datastores $templateshash[$k].datastores -sourcevmhash $templateshash[$k].data -whatif:$WhatIfPreference -confirm:$ConfirmPreference
                }

                # remove the old clones
                stopVms -vms @($vmhashlist.Keys) -maxqueue $MaxQueue -checkClone -whatif:$WhatIfPreference -confirm:$ConfirmPreference
                removeVms -vms @($vmhashlist.Keys) -maxqueue $MaxQueue -checkClone -whatif:$WhatIfPreference -confirm:$ConfirmPreference

                # we loop the list of
                $i=0
                $total = $vmhashlist.Keys.count
                $queuefull = $false
                while($i -lt $total){
                    if($clonetasks.Keys.Count -le $MaxQueue){

                        try{
                            $queuefull = $false
                            $vmhash = $vmhashlist[@($vmhashlist.Keys)[$i]]
                            $vmname = $vmhash.name

                            # progress bar
                            $i++
                            $pct = [int]($i/$total*100)
                            Write-Progress -Activity "Cloning" -Status "Adding $vmname" -PercentComplete $pct

                            # collect information from vm
                            $vmhost = $vmhash.vmhost
                            $datastore = $vmhash.datastore
                            $parentuuid = $vmhash.customAttributes["PnC.Source"]
                            $parenttemplate = $vmlistbyuuid[$parentuuid]
                            $parentuuid = $vmhash.customAttributes["PnC.Source"]
                            $nametemplate = $vmhash.customAttributes["PnC.NameTemplate"]
                            $oscustomizationspec = $vmhash.customAttributes["PnC.CustSpec"]
                            $resourcepool = $vmhash.resourcepool
                            $nics = $vmhash.networkadapters
                            # set output data
                            $outputhash[$vmname] = @{}
                            $outputhash[$vmname]["vm"] = $vmname
                            $outputhash[$vmname]["vmhost"] = $vmhost
                            $outputhash[$vmname]["parentuuid"] = $parentuuid
                            $outputhash[$vmname]["datastore"] = $datastore
                            $outputhash[$vmname]["parenttemplate"] = $parenttemplate
                            $outputhash[$vmname]["nametemplate"] = $nametemplate
                            $outputhash[$vmname]["oscustomizationspec"] = $oscustomizationspec
                            $outputhash[$vmname]["resourcepool"] = $resourcepool
                            $outputhash[$vmname]["resources"] = $vmhash["resources"]
                            $outputhash[$vmname]["networkadapters"] = $vmhash["networkadapters"]
                            $outputhash[$vmname]["numCpu"] = $numCpu
                            $outputhash[$vmname]["coresPerSocket"] = $coresPerSocket
                            $outputhash[$vmname].resourcechanged = $false
                            if(Compare-Hashtable $templateshash[$parenttemplate]["data"]["resources"] $vmhash["resources"]){
                                WriteWithColor -text ("$vmname has different resources [{0}MB][{1}cpu][{2}cores]" -f $memoryMb,$numCpu,$coresPerSocket) -state "Info" -color cyan
                                $outputhash[$vmname].resourcechanged = $true
                            }
                            if(Compare-Hashtable $templateshash[$parenttemplate]["data"]["networkadapters"] $vmhash["networkadapters"]){
                                WriteWithColor -text "$vmname has different networkconfig" -state "Info" -color cyan
                                $outputhash[$vmname].networkchanged = $true
                            }

                            $outputhash[$vmname]["state"] = "failed"
                            # Getting custom attributes
                            $customAttributes = $vmhash.customAttributes
                            $customAttributes["PnC.Deployed"] = get-date
                            $outputhash[$vmname]["customattributes"] = $customAttributes

                            # quick check if the vm is compliant to this script, we expect certain custom attributes
                            # - PnC.Source => guid of source template
                            # - PnC.CustSpec => custom os specifications
                            # - PnC.NameTemplate => the name template (not important, but we keep it for future)
                            # - PnC.Deployed => date of last cloning
                            if(-not $parenttemplate){
                                WriteWithColor -text "$vmname is not tagged with ParentTemplate" -state "Warning" -color yellow
                                continue
                            }

                            # redeploy the clones
                            try{
                                $clonetasks[$vmname] = clonevm -name $vmname -vmhost $vmhost -resourcepool $resourcepool -oscustomizationspec $oscustomizationspec -sourcevm $parenttemplate -datastore $datastore -istemplate $templateshash[$parenttemplate].data.istemplate -whatif:$WhatIfPreference -confirm:$ConfirmPreference
                            }catch{
                                WriteWithColor -text ("Failed to create clone $vmname. {0}" -f $_.Exception.Message) -state "Error" -color red
                            }

                        }catch{
                            WriteWithColor -text ("Unexpected error during preparation of clone $vmname. {0}" -f $_.Exception.Message) -state "Error" -color red
                        }
 
                    }else{
                        if(-not $queuefull){
                            WriteWithColor -text "Queue is full [$MaxQueue]" -state "Waiting" -color cyan
                        }
                        $queuefull = $true
                        sleep 1
                    }
                    # check if tasks have completed
                    ProcessVmTasks -tasks $clonetasks -description "Adding clone" -postvmjobs $postvmjobs -startvm:$StartVM -reconfigurevm:$ReconfigureVm -outputhash $outputhash

                }
            }   

            # finalize tasks
            Write-Progress -Activity "Adding clone task" -Status "Done" -Completed
            ProcessVmTasks -tasks $clonetasks -description "Adding clone" -postvmjobs $postvmjobs -startvm:$StartVM -reconfigurevm:$ReconfigureVm -loop -outputhash $outputhash
                    
 
        }catch{
            WriteWithColor -text $_.Exception.Message -state "Error" -color red
            if($_.Exception.InnerException.Message){
                WriteWithColor -text $_.Exception.InnerException.Message -state "Error" -color red
            }
        }
    }
    End{
        try{

            # remove templates if required
            if($RemoveTemporaryTemplates){
                removeVms -vms @($tempTemplates) -maxqueue $MaxQueue -silent -whatif:$WhatIfPreference -confirm:$ConfirmPreference
            }

            # disconnect
            disconnectVcenter -Vcenter $Vcenter -session $Session
        }catch{}

        # send to output stream for further piping
        foreach($h in $outputhash.Keys){
            New-Object -TypeName PSObject -Property $outputhash[$h]
        }   
    }
}

<#
.Synopsis
   Rapidly removes multiple vm's.
.DESCRIPTION
   This cmdlet is dangerous. It rapidly stops and remove vm's in bulk.
.EXAMPLE
    Remove-RapidClone -Vcenter jabba.slash.local -Credentials $credentials -Names (get-vm test*).name -Force
    [Connected] Vcenter Connection [jabba.slash.local]
    [Success] Stopping clone test_01
    [Success] Stopping clone test_02
    [Success] Stopping clone test_03
    [Success] Removing clone test_01
    [Success] Removing clone test_02
    [Success] Removing clone test_03
    [Disconnected] Vcenter Connection [jabba.slash.local]
 
#>
function Remove-RapidClone{

    [cmdletbinding(DefaultParameterSetName='Using Credentials',
                   SupportsShouldProcess = $true,
                   ConfirmImpact='High')]
    param(

        # Vcenter server
        [Parameter(Mandatory=$true)]
        $Vcenter,

        # PSCredentials, if non given, we assume the current user has direct access.
        [Parameter(Mandatory=$false,ParameterSetName="Using Credentials")]
        [PSCredential]$Credentials,

        # An existing vmware session (instead of credentials)
        [Parameter(Mandatory=$true,ParameterSetName="Using Session")]
        [String]$Session,

        # An array of vm names to be removed
        [Parameter(Mandatory=$true)]
        [string[]]$Names,


        # Maximum vm tasks can run at the same time
        [Parameter(Mandatory=$false)]
        [int]$MaxQueue = 50


    )
    Begin{

        ###################################
        # START CODE
        ###################################
        $ErrorActionPreference = "stop"

        # ----------------------------------
        # Connecting to vcenter
        # ----------------------------------
        try{
            ConnectVcenter -Vcenter $Vcenter -Credentials $Credentials -session $Session
            WriteWithColor -text "Vcenter Connection [$Vcenter]" -state "Connected" -color green
        }catch{
            Throw "Failed to connect to vcenter $Vcenter"
        }
        if (-not $PSBoundParameters.ContainsKey('Confirm')) {
            $ConfirmPreference = $PSCmdlet.SessionState.PSVariable.GetValue('ConfirmPreference')
        }
        if (-not $PSBoundParameters.ContainsKey('WhatIf')) {
            $WhatIfPreference = $PSCmdlet.SessionState.PSVariable.GetValue('WhatIfPreference')
        }

    }
    Process{
        try{

            #####################################
            # REMOVE ACTION
            #####################################

            stopVms -vms $Names -maxqueue $MaxQueue -checkClone -whatif:$WhatIfPreference -confirm:$ConfirmPreference
            removeVms -vms $Names -maxqueue $MaxQueue -checkClone -whatif:$WhatIfPreference -confirm:$ConfirmPreference
  
        }catch{
            WriteWithColor -text $_.Exception.Message -state "Error" -color red
            if($_.Exception.InnerException.Message){
                WriteWithColor -text $_.Exception.InnerException.Message -state "Error" -color red
            }
        }
    }
    End{
        try{

            # disconnect
            disconnectVcenter -Vcenter $Vcenter -session $Session
        }catch{}
    }
}

Export-ModuleMember -Function New-RapidClone
Export-ModuleMember -Function Update-RapidClone
Export-ModuleMember -Function Remove-RapidClone