AutomatedLabFailover.psm1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
#region Install-LabFailoverCluster
function Install-LabFailoverCluster
{
    [CmdletBinding()]
    param ( )

    $failoverNodes = Get-LabVm -Role FailoverNode -ErrorAction SilentlyContinue
    $clusters = $failoverNodes | Group-Object { ($_.Roles | Where-Object -Property Name -eq 'FailoverNode').Properties['ClusterName'] }
    $useDiskWitness = $false
    Start-LabVM -Wait -ComputerName $failoverNodes

    Install-LabWindowsFeature -ComputerName $failoverNodes -FeatureName Failover-Clustering, RSAT-Clustering -IncludeAllSubFeature

    Write-ScreenInfo -Message 'Restart post FCI Install'
    Restart-LabVM $failoverNodes -Wait

    if (Get-LabWindowsFeature -ComputerName $failoverNodes -FeatureName Failover-Clustering, RSAT-Clustering | Where InstallState -ne Installed)
    {
        Install-LabWindowsFeature -ComputerName $failoverNodes -FeatureName Failover-Clustering, RSAT-Clustering -IncludeAllSubFeature
        Write-ScreenInfo -Message 'Restart post FCI Install'
        Restart-LabVM $failoverNodes -Wait
    }

    if (Get-LabVm -Role FailoverStorage)
    {
        Write-ScreenInfo -Message 'Waiting for failover storage server to complete installation'
        Install-LabFailoverStorage
        $useDiskWitness = $true
    }

    Write-ScreenInfo -Message 'Waiting for failover nodes to complete installation'

    foreach ($cluster in $clusters)
    {
        $firstNode = $cluster.Group | Select-Object -First 1
        $clusterDomains = $cluster.Group.DomainName | Sort-Object -Unique
        $clusterNodeNames = $cluster.Group | Select-Object -Skip 1 -ExpandProperty Name
        $clusterName = $cluster.Name
        $clusterIp = ($firstNode.Roles | Where-Object -Property Name -eq 'FailoverNode').Properties['ClusterIp'] -split '\s*(?:,|;?),\s*'

        if (-not $clusterIp)
        {
            $adapterVirtualNetwork = Get-LabVirtualNetworkDefinition -Name $firstNode.NetworkAdapters[0].VirtualSwitch
            $clusterIp = $adapterVirtualNetwork.NextIpAddress().AddressAsString
        }

        if (-not $clusterName)
        {
            $clusterName = 'ALCluster'
        }

        $ignoreNetwork = foreach ($network in (Get-Lab).VirtualNetworks)
        {
            $range = Get-NetworkRange -IPAddress $network.AddressSpace.Network.AddressAsString -SubnetMask $network.AddressSpace.Cidr
            $inRange = $clusterIp | Where-Object {$_ -in $range}
            
            if (-not $inRange)
            {
                '{0}/{1}' -f $network.AddressSpace.Network.AddressAsString, $network.AddressSpace.Cidr
            }
        }

        if ($useDiskWitness -and -not ($firstNode.OperatingSystem.Version -lt 6.2))
        {
            Invoke-LabCommand -ComputerName $firstNode -ActivityName 'Preparing cluster storage' -ScriptBlock {
                if (-not (Get-ClusterAvailableDisk -ErrorAction SilentlyContinue))
                {
                    $offlineDisk = Get-Disk | Where-Object -Property OperationalStatus -eq Offline | Select-Object -First 1
                    if ($offlineDisk)
                    {
                        $offlineDisk | Set-Disk -IsOffline $false
                        $offlineDisk | Set-Disk -IsReadOnly $false
                    }

                    if (-not ($offlineDisk | Get-Partition | Get-Volume))
                    {
                        $offlineDisk | New-Volume -FriendlyName quorum -FileSystem NTFS
                    }
                }
            }

            Invoke-LabCommand -ComputerName $clusterNodeNames -ActivityName 'Preparing cluster storage on remaining nodes' -ScriptBlock {
                Get-Disk | Where-Object -Property OperationalStatus -eq Offline | Set-Disk -IsOffline $false
            }
        }

        $storageNode = Get-LabVm -Role FailoverStorage -ErrorAction SilentlyContinue
        $role = $storageNode.Roles | Where-Object Name -eq FailoverStorage

        if((-not $useDiskWitness) -or ($storageNode.Disks.Count -gt 1))
        {
            Invoke-LabCommand -ComputerName $firstNode -ActivityName 'Preparing cluster storage' -ScriptBlock {
                $diskpartCmd = 'LIST DISK'

                $disks = $diskpartCmd | diskpart.exe

                foreach ($line in $disks)
                {
                    if ($line -match 'Disk (?<DiskNumber>\d) \s+(Offline)\s+(?<Size>\d+) GB\s+(?<Free>\d+) GB')
                    {
                        $nextDriveLetter = [char[]](67..90) |
                        Where-Object { (Get-WmiObject -Class Win32_LogicalDisk |
                        Select-Object -ExpandProperty DeviceID) -notcontains "$($_):"} |
                        Select-Object -First 1

                        $diskNumber = $Matches.DiskNumber

                        $diskpartCmd = "@
                            SELECT DISK $diskNumber
                            ATTRIBUTES DISK CLEAR READONLY
                            ONLINE DISK
                            CREATE PARTITION PRIMARY
                            ASSIGN LETTER=$nextDriveLetter
                            EXIT
                        @"

                        $diskpartCmd | diskpart.exe | Out-Null

                        Start-Sleep -Seconds 2

                        cmd.exe /c "echo y | format $($nextDriveLetter): /q /v:DataDisk$diskNumber"
                    }
                }
            }

            Invoke-LabCommand -ComputerName $clusterNodeNames -ActivityName 'Preparing cluster storage' -ScriptBlock {
                $diskpartCmd = 'LIST DISK'

                $disks = $diskpartCmd | diskpart.exe

                foreach ($line in $disks)
                {
                    if ($line -match 'Disk (?<DiskNumber>\d) \s+(Offline)\s+(?<Size>\d+) GB\s+(?<Free>\d+) GB')
                    {
                        $diskNumber = $Matches.DiskNumber

                        $diskpartCmd = "@
                            SELECT DISK $diskNumber
                            ATTRIBUTES DISK CLEAR READONLY
                            ONLINE DISK
                            EXIT
                        @"

                        $diskpartCmd | diskpart.exe | Out-Null
                    }
                }
            }
        }


        $clusterAccessPoint = if ($clusterDomains.Count -ne 1)
        {
            'DNS'
        }
        else
        {
            'ActiveDirectoryAndDns'
        }

        Remove-LabPSSession -ComputerName $failoverNodes
        Invoke-LabCommand -ComputerName $firstNode -ActivityName 'Enabling clustering on first node' -ScriptBlock {
            Import-Module FailoverClusters -ErrorAction Stop -WarningAction SilentlyContinue

            $clusterParameters = @{
                Name                      = $clusterName
                StaticAddress             = $clusterIp
                AdministrativeAccessPoint = $clusterAccessPoint
                ErrorAction               = 'SilentlyContinue'
                WarningAction             = 'SilentlyContinue'
            }

            if ($ignoreNetwork)
            {
                $clusterParameters.IgnoreNetwork = $ignoreNetwork
            }

            $clusterParameters = Sync-Parameter -Command (Get-Command New-Cluster) -Parameters $clusterParameters

            $null = New-Cluster @clusterParameters
        } -Variable (Get-Variable clusterName, clusterNodeNames, clusterIp, useDiskWitness, clusterAccessPoint, ignoreNetwork) -Function (Get-Command Sync-Parameter)

        Remove-LabPSSession -ComputerName $failoverNodes
        Invoke-LabCommand -ComputerName $firstNode -ActivityName 'Adding nodes' -ScriptBlock {
            Import-Module FailoverClusters -ErrorAction Stop -WarningAction SilentlyContinue

            if (-not (Get-Cluster -Name $clusterName -ErrorAction SilentlyContinue))
            {
                Write-Error "Cluster $clusterName was not deployed"
            }

            foreach ($node in $clusterNodeNames)
            {
                Add-ClusterNode -Name $node -Cluster $clusterName -ErrorAction SilentlyContinue
            }

            if (Compare-Object -ReferenceObject $clusterNodeNames -DifferenceObject (Get-ClusterNode -Cluster $clusterName).Name | Where-Object SideIndicator -eq '<=')
            {
                Write-Error -Message "Error deploying cluster $clusterName, not all nodes were added to the cluster"
            }

            if ($useDiskWitness)
            {
                $clusterDisk = Get-ClusterResource -Cluster $clusterName -ErrorAction SilentlyContinue | Where-object -Property ResourceType -eq 'Physical Disk' | Select -First 1

                if ($clusterDisk)
                {
                    Get-Cluster -Name $clusterName | Set-ClusterQuorum -DiskWitness $clusterDisk
                }
            }
        } -Variable (Get-Variable clusterName, clusterNodeNames, clusterIp, useDiskWitness, clusterAccessPoint, ignoreNetwork)
    }
}
#endregion

#region Install-LabFailoverStorage
function Install-LabFailoverStorage
{
    [CmdletBinding()]
    param
    ( )

    $storageNodes = Get-LabVM -Role FailoverStorage -ErrorAction SilentlyContinue
    $failoverNodes = Get-LabVM -Role FailoverNode -ErrorAction SilentlyContinue
    if ($storageNodes.Count -gt 1)
    {
        foreach ($failoverNode in $failoverNodes)
        {
            $role = $failoverNode.Roles | Where-Object Name -eq 'FailoverNode'
            if (-not $role.Properties.ContainsKey('StorageTarget'))
            {
                Write-Error "There are $($storageNodes.Count) VMs with the 'FailoverStorage' role and the storage target is not defined for '$failoverNode'. Please define the property 'StorageTarget' with the 'FailoverStorage' role." -ErrorAction Stop
            }
        }
    }
    Start-LabVM -ComputerName (Get-LabVM -Role FailoverStorage, FailoverNode) -Wait
    
    $clusters = @{}
    $storageMapping = @{}
    
    foreach ($failoverNode in $failoverNodes) {
    
        $role = $failoverNode.Roles | Where-Object Name -eq 'FailoverNode'
        $name = $role.Properties['ClusterName']
        $storageMapping."$($failoverNode.Name)" = if ($role.Properties.ContainsKey('StorageTarget'))
        {
            $role.Properties['StorageTarget']
        }
        else
        {
            $storageNodes.Name
        }

        if (-not $name)
        {
            $name = 'ALCluster'
        }
    
        if (-not $clusters.ContainsKey($name))
        {
            $clusters[$name] = @()
        }
        $clusters[$name] += $failoverNode.Name
    }
    
    foreach ($cluster in $clusters.Clone().GetEnumerator())
    {
        $machines = $cluster.Value
        $clusterName = $cluster.Key
        $initiatorIds = Invoke-LabCommand -ActivityName 'Retrieving IQNs' -ComputerName $machines -ScriptBlock {
            Set-Service -Name MSiSCSI -StartupType Automatic
            Start-Service -Name MSiSCSI
            "IQN:$((Get-WmiObject -Namespace root\wmi -Class MSiSCSIInitiator_MethodClass).iSCSINodeName)"
        } -PassThru -ErrorAction Stop
    
        $clusters[$clusterName] = $initiatorIds
    }
    
    Install-LabWindowsFeature -ComputerName $storageNodes -FeatureName FS-iSCSITarget-Server

    foreach ($storageNode in $storageNodes)
    {
        foreach ($disk in $storageNode.Disks)
        {
            Write-ScreenInfo "Working on $($disk.name)"
            #$lunDrive = $role.Properties['LunDrive'][0] # Select drive letter only
            $driveLetter = $disk.DriveLetter

            Invoke-LabCommand -ActivityName "Creating iSCSI target for $($disk.name) on '$storageNode'" -ComputerName $storageNode -ScriptBlock {
                # assign drive letter if not provided
                if (-not $driveLetter)
                {
                    # http://vcloud-lab.com/entries/windows-2016-server-r2/find-next-available-free-drive-letter-using-powershell-
                    #$driveLetter = (68..90 | % {$L = [char]$_; if ((gdr).Name -notContains $L) {$L}})[0]
                    $driveLetter = $env:SystemDrive[0]
                }

                $driveInfo = [System.IO.DriveInfo] [string] $driveLetter

                if (-not (Test-Path $driveInfo))
                {
                    $offlineDisk = Get-Disk | Where-Object -Property OperationalStatus -eq Offline | Select-Object -First 1
                    if ($offlineDisk)
                    {
                        $offlineDisk | Set-Disk -IsOffline $false
                        $offlineDisk | Set-Disk -IsReadOnly $false
                    }

                    if (-not ($offlineDisk | Get-Partition | Get-Volume))
                    {
                        $offlineDisk | New-Volume -FriendlyName $disk -FileSystem ReFS -DriveLetter $driveLetter
                    }
                }

                $folderPath = Join-Path -Path $driveInfo -ChildPath $disk.Name
                $folder = New-Item -ItemType Directory -Path $folderPath -ErrorAction SilentlyContinue
                $folder = Get-Item -Path $folderPath -ErrorAction Stop

                foreach ($clu in $clusters.GetEnumerator())
                {
                    if (-not (Get-IscsiServerTarget -TargetName $clu.Key -ErrorAction SilentlyContinue))
                    {
                        New-IscsiServerTarget -TargetName $clu.Key -InitiatorIds $clu.Value
                    }
                    $diskTarget = (Join-Path -Path $folder.FullName -ChildPath "$($disk.name).vhdx")
                    $diskSize = [uint64]$disk.DiskSize*1GB
                    if (-not (Get-IscsiVirtualDisk -Path $diskTarget -ErrorAction SilentlyContinue))
                    {
                        New-IscsiVirtualDisk -Path $diskTarget -Size $diskSize
                    }
                    Add-IscsiVirtualDiskTargetMapping -TargetName $clu.Key -Path $diskTarget
                }
            } -Variable (Get-Variable -Name clusters, disk, driveletter) -ErrorAction Stop

            Invoke-LabCommand -ActivityName "Connecting iSCSI target - storage node '$storageNode' - disk '$disk'" -ComputerName (Get-LabVM -Role FailoverNode) -ScriptBlock {
                $targetAddress = $storageMapping[$env:COMPUTERNAME]
                if (-not (Get-Command New-IscsiTargetPortal -ErrorAction SilentlyContinue))
                {
                    iscsicli.exe QAddTargetPortal $targetAddress
                    $target = ((iscsicli.exe ListTargets) -match 'iqn.+target')[0].Trim()
                    iscsicli.exe QLoginTarget $target
                }
                else
                {
                    New-IscsiTargetPortal -TargetPortalAddress $targetAddress
                    Get-IscsiTarget | Where-Object {-not $_.IsConnected} | Connect-IscsiTarget -IsPersistent $true
                }
            } -Variable (Get-Variable storageMapping) -ErrorAction Stop
        }
    }
}
#endregion