modules/SdnDiag.Common/SdnDiag.Common.psm1
|
# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. Import-Module $PSScriptRoot\..\SdnDiag.Utilities\SdnDiag.Utilities.psm1 # create local variable to store configuration data $configurationData = Import-PowerShellDataFile -Path "$PSScriptRoot\SdnDiag.Common.Config.psd1" New-Variable -Name 'SdnDiagnostics_Common' -Scope 'Script' -Force -Value @{ Cache = @{} Config = $configurationData } ##### FUNCTIONS AUTO-POPULATED BELOW THIS LINE DURING BUILD ##### function Add-SdnDiagTraceMapping { param ( [Parameter(Mandatory=$true)] [string]$MacAddress, [Parameter(Mandatory=$true)] [string]$InfraHost, [Parameter(Mandatory=$false)] [string]$PortId, [Parameter(Mandatory=$false)] [string]$PortName, [Parameter(Mandatory=$false)] [string]$NicName, [Parameter(Mandatory=$false)] [string]$VmName, [Parameter(Mandatory=$false)] [string]$VmInternalId, [Parameter(Mandatory=$false)] [string[]]$PrivateIpAddress ) $cacheName = 'TraceMapping' $mapping = @{ MacAddress = $MacAddress PortId = $PortId PortName = $PortName NicName = $NicName VmName = $VmName VmInternalId = $VmInternalId InfraHost = $InfraHost PrivateIpAddress = $PrivateIpAddress } if (!$Script:SdnDiagnostics_Common.Cache.ContainsKey($cacheName)) { $Script:SdnDiagnostics_Common.Cache.Add($cacheName, @{}) } if (!$Script:SdnDiagnostics_Common.Cache[$cacheName].ContainsKey($InfraHost.ToLower())) { $Script:SdnDiagnostics_Common.Cache[$cacheName].Add($InfraHost.ToLower(), @{}) } $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost.ToLower()][$MacAddress] += $mapping } function Copy-CertificateToFabric { [CmdletBinding()] param ( [Parameter(Mandatory = $true, ParameterSetName = 'NetworkControllerRest')] [Parameter(Mandatory = $true, ParameterSetName = 'NetworkControllerNode')] [Parameter(Mandatory = $true, ParameterSetName = 'LoadBalancerMuxNode')] [Parameter(Mandatory = $true, ParameterSetName = 'ServerNode')] [System.String]$CertFile, [Parameter(Mandatory = $false, ParameterSetName = 'NetworkControllerRest')] [Parameter(Mandatory = $false, ParameterSetName = 'NetworkControllerNode')] [Parameter(Mandatory = $false, ParameterSetName = 'LoadBalancerMuxNode')] [Parameter(Mandatory = $false, ParameterSetName = 'ServerNode')] [System.Security.SecureString]$CertPassword, [Parameter(Mandatory = $true, ParameterSetName = 'NetworkControllerRest')] [Parameter(Mandatory = $true, ParameterSetName = 'NetworkControllerNode')] [Parameter(Mandatory = $true, ParameterSetName = 'LoadBalancerMuxNode')] [Parameter(Mandatory = $true, ParameterSetName = 'ServerNode')] [System.Object]$FabricDetails, [Parameter(Mandatory = $true, ParameterSetName = 'NetworkControllerRest')] [Switch]$NetworkControllerRestCertificate, [Parameter(Mandatory = $false, ParameterSetName = 'NetworkControllerRest')] [System.Boolean]$InstallToSouthboundDevices = $false, [Parameter(Mandatory = $true, ParameterSetName = 'NetworkControllerNode')] [Switch]$NetworkControllerNodeCert, [Parameter(Mandatory = $true, ParameterSetName = 'LoadBalancerMuxNode')] [Switch]$LoadBalancerMuxNodeCert, [Parameter(Mandatory = $true, ParameterSetName = 'ServerNode')] [Switch]$ServerNodeCert, [Parameter(Mandatory = $false, ParameterSetName = 'NetworkControllerRest')] [Parameter(Mandatory = $false, ParameterSetName = 'NetworkControllerNode')] [Parameter(Mandatory = $false, ParameterSetName = 'LoadBalancerMuxNode')] [Parameter(Mandatory = $false, ParameterSetName = 'ServerNode')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty ) # if we are installing the rest certificate and need to seed certificate to southbound devices # then define the variables to know which nodes must be updated if ($PSCmdlet.ParameterSetName -ieq 'NetworkControllerRest' -and $InstallToSouthboundDevices) { $southBoundNodes = @() if ($null -ne $FabricDetails.LoadBalancerMux) { $southBoundNodes += $FabricDetails.LoadBalancerMux } if ($null -ne $FabricDetails.Server) { $southBoundNodes += $FabricDetails.Server } } $certFileInfo = Get-Item -Path $CertFile -ErrorAction Stop switch ($certFileInfo.Extension) { '.pfx' { if ($CertPassword) { $certData = (Get-PfxData -FilePath $certFileInfo.FullName -Password $CertPassword).EndEntityCertificates } else { $certData = Get-PfxCertificate -FilePath $certFileInfo.FullName } } '.cer' { $certData = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2 $certData.Import($certFileInfo) } default { throw New-Object System.NotSupportedException("Unsupported certificate extension") } } switch ($PSCmdlet.ParameterSetName) { 'LoadBalancerMuxNode' { foreach ($controller in $FabricDetails.NetworkController) { # if the certificate being passed is self-signed, we will need to copy the certificate to the other controller nodes # within the fabric and install under localmachine\root as appropriate if ($certData.Subject -ieq $certData.Issuer) { "Importing certificate [Subject: {0} Thumbprint:{1}] to {2}" -f ` $certData.Subject, $certData.Thumbprint, $controller | Trace-Output [System.String]$remoteFilePath = Join-Path -Path $certFileInfo.Directory.FullName -ChildPath $certFileInfo.Name $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1) if (-NOT (Test-Path -Path $param1 -PathType Container)) { New-Item -Path $param1 -ItemType Directory -Force } } -ArgumentList $certFileInfo.Directory.FullName Copy-FileToRemoteComputer -ComputerName $controller -Credential $Credential -Path $certFileInfo.FullName -Destination $remoteFilePath $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][SecureString]$param2, [Parameter(Position = 2)][String]$param3) Import-SdnCertificate -FilePath $param1 -CertPassword $param2 -CertStore $param3 } -ArgumentList @($remoteFilePath, $CertPassword, 'Cert:\LocalMachine\Root') -ErrorAction Stop } else { "No action required for {0}" -f $certData.Thumbprint | Trace-Output -Level:Verbose } } } 'NetworkControllerRest' { # copy the pfx certificate for the rest certificate to all network controllers within the cluster # and import to localmachine\my cert directory foreach ($controller in $FabricDetails.NetworkController) { "Processing {0}" -f $controller | Trace-Output -Level:Verbose "[REST CERT] Importing certificate [Subject: {0} Thumbprint:{1}] to {2}" -f ` $certData.Subject, $certData.Thumbprint, $controller | Trace-Output if (Test-ComputerNameIsLocal -ComputerName $controller) { $importCert = Import-SdnCertificate -FilePath $certFileInfo.FullName -CertPassword $CertPassword -CertStore 'Cert:\LocalMachine\My' # if the certificate was detected as self signed # we will then copy the .cer file returned from the previous command to all the southbound nodes to install if ($importCert.SelfSigned -and $InstallToSouthboundDevices) { Install-SdnDiagnostics -ComputerName $southBoundNodes -Credential $Credential -ErrorAction Stop "[REST CERT] Installing self-signed certificate to southbound devices" | Trace-Output Invoke-PSRemoteCommand -ComputerName $southBoundNodes -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1) if (-NOT (Test-Path -Path $param1 -PathType Container)) { $null = New-Item -Path $param1 -ItemType Directory -Force } } -ArgumentList $importCert.CerFileInfo.Directory.FullName foreach ($sbNode in $southBoundNodes) { "[REST CERT] Installing self-signed certificate to {0}" -f $sbNode | Trace-Output Copy-FileToRemoteComputer -ComputerName $sbNode -Credential $Credential -Path $importCert.CerFileInfo.FullName -Destination $importCert.CerFileInfo.FullName $null = Invoke-PSRemoteCommand -ComputerName $sbNode -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1,[Parameter(Position = 1)][String]$param2) Import-SdnCertificate -FilePath $param1 -CertStore $param2 } -ArgumentList @($importCert.CerFileInfo.FullName, 'Cert:\LocalMachine\Root') -ErrorAction Stop } } } else { [System.String]$remoteFilePath = Join-Path -Path $certFileInfo.Directory.FullName -ChildPath $certFileInfo.Name $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1) if (-NOT (Test-Path -Path $param1 -PathType Container)) { New-Item -Path $param1 -ItemType Directory -Force } } -ArgumentList $certFileInfo.Directory.FullName Copy-FileToRemoteComputer -ComputerName $controller -Credential $Credential -Path $certFileInfo.FullName -Destination $remoteFilePath $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][SecureString]$param2, [Parameter(Position = 2)][String]$param3) Import-SdnCertificate -FilePath $param1 -CertPassword $param2 -CertStore $param3 } -ArgumentList @($remoteFilePath, $CertPassword, 'Cert:\LocalMachine\My') } } } 'NetworkControllerNode' { foreach ($controller in $FabricDetails.NetworkController) { "Processing {0}" -f $controller | Trace-Output -Level:Verbose # if the certificate being passed is self-signed, we will need to copy the certificate to the other controller nodes # within the fabric and install under localmachine\root as appropriate if ($certData.Subject -ieq $certData.Issuer) { "Importing certificate [Subject: {0} Thumbprint:{1}] to {2}" -f ` $certData.Subject, $certData.Thumbprint, $controller | Trace-Output [System.String]$remoteFilePath = Join-Path -Path $certFileInfo.Directory.FullName -ChildPath $certFileInfo.Name $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1) if (-NOT (Test-Path -Path $param1 -PathType Container)) { New-Item -Path $param1 -ItemType Directory -Force } } -ArgumentList $certFileInfo.Directory.FullName Copy-FileToRemoteComputer -ComputerName $controller -Credential $Credential -Path $certFileInfo.FullName -Destination $remoteFilePath $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][SecureString]$param2, [Parameter(Position = 2)][String]$param3) Import-SdnCertificate -FilePath $param1 -CertPassword $param2 -CertStore $param3 } -ArgumentList @($remoteFilePath, $CertPassword, 'Cert:\LocalMachine\Root') -ErrorAction Stop } else { "No action required for {0}" -f $certData.Thumbprint | Trace-Output -Level:Verbose } } } # for ServerNodes, we must distribute the server certificate and install to the cert:\localmachine\root directory on each of the # network controller nodes 'ServerNode' { foreach ($controller in $FabricDetails.NetworkController) { # if the certificate being passed is self-signed, we will need to copy the certificate to the other controller nodes # within the fabric and install under localmachine\root as appropriate if ($certData.Subject -ieq $certData.Issuer) { "Importing certificate [Subject: {0} Thumbprint:{1}] to {2}" -f ` $certData.Subject, $certData.Thumbprint, $controller | Trace-Output [System.String]$remoteFilePath = Join-Path -Path $certFileInfo.Directory.FullName -ChildPath $certFileInfo.Name $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1) if (-NOT (Test-Path -Path $param1 -PathType Container)) { New-Item -Path $param1 -ItemType Directory -Force } } -ArgumentList $certFileInfo.Directory.FullName Copy-FileToRemoteComputer -ComputerName $controller -Credential $Credential -Path $certFileInfo.FullName -Destination $remoteFilePath $null = Invoke-PSRemoteCommand -ComputerName $controller -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$param1, [Parameter(Position = 1)][SecureString]$param2, [Parameter(Position = 2)][String]$param3) Import-SdnCertificate -FilePath $param1 -CertPassword $param2 -CertStore $param3 } -ArgumentList @($remoteFilePath, $CertPassword, 'Cert:\LocalMachine\Root') -ErrorAction Stop } else { "No action required for {0}" -f $certData.Thumbprint | Trace-Output -Level:Verbose } } } } } function Copy-UserProvidedCertificateToFabric { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.IO.DirectoryInfo]$CertPath, [Parameter(Mandatory = $true)] [System.Security.SecureString]$CertPassword, [Parameter(Mandatory = $true)] [System.Object]$FabricDetails, [Parameter(Mandatory = $false)] [System.Boolean]$RotateNodeCerts = $false, [Parameter(Mandatory = $false)] [System.Boolean]$NetworkControllerHealthy = $false, [Parameter(Mandatory = $false)] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty ) $certificateCache = @() $certificateConfig = @{ RestCert = $null NetworkController = @{} } "Scanning certificates within {0}" -f $CertPath.FullName | Trace-Output $pfxFiles = Get-ChildItem -Path $CertPath.FullName -Filter '*.pfx' if ($null -eq $pfxFiles) { throw New-Object System.NullReferenceException("Unable to locate .pfx files under the specified CertPath location") } foreach ($pfxFile in $pfxFiles) { "Retrieving PfxData for {0}" -f $pfxFile.FullName | Trace-Output $pfxData = Get-PfxData -FilePath $pfxFile.FullName -Password $CertPassword -ErrorAction Stop $object = [PSCustomObject]@{ FileInfo = $pfxFile PfxData = $pfxData SelfSigned = $false } $certificateCache += $object } "Retrieving Rest Certificate" | Trace-Output -Level:Verbose $currentRestCertificate = Get-SdnNetworkControllerRestCertificate # enumerate the current certificates within the cache to isolate the rest certificate foreach ($cert in $certificateCache) { if ($cert.pfxdata.EndEntityCertificates.Subject -ieq $currentRestCertificate.Subject) { "Matched {0} [Subject: {1}; Thumbprint: {2}] to NC Rest Certificate" -f ` $cert.pfxFile.FileInfo.FullName, $cert.pfxData.EndEntityCertificates.Subject, $cert.pfxData.EndEntityCertificates.Thumbprint | Trace-Output -Level:Verbose $cert | Add-Member -MemberType NoteProperty -Name 'CertificateType' -Value 'NetworkControllerRest' $restCertificate = $cert $certificateConfig.RestCert = $restCertificate.pfxData.EndEntityCertificates.Thumbprint } if ($cert.pfxdata.EndEntityCertificates.Subject -ieq $cert.pfxdata.EndEntityCertificates.Issuer) { $cert.SelfSigned = $true } } # enumerate the certificates for network controller nodes if ($RotateNodeCerts) { foreach ($node in $FabricDetails.NetworkController) { "Retrieving current node certificate for {0}" -f $node | Trace-Output $currentNodeCert = Invoke-PSRemoteCommand -ComputerName $node -Credential $Credential -ScriptBlock { Get-SdnNetworkControllerNodeCertificate } -ErrorAction Stop foreach ($cert in $certificateCache) { $updatedNodeCert = $null if ($cert.PfxData.EndEntityCertificates.Subject -ieq $currentNodeCert.Subject) { $updatedNodeCert = $cert "Matched {0} [Subject: {1}; Thumbprint: {2}] to {3}" -f ` $updatedNodeCert.FileInfo.Name, $cert.PfxData.EndEntityCertificates.Subject, $cert.PfxData.EndEntityCertificates.Thumbprint, $node | Trace-Output $cert | Add-Member -MemberType NoteProperty -Name 'CertificateType' -Value 'NetworkControllerNode' break } } $certificateConfig.NetworkController[$node] = @{ Cert = $updatedNodeCert } } } # install the rest certificate to the network controllers to this node first # then seed out to the rest of the fabric $null = Import-SdnCertificate -FilePath $restCertificate.FileInfo.FullName -CertPassword $CertPassword -CertStore 'Cert:\LocalMachine\My' Copy-CertificateToFabric -CertFile $restCertificate.FileInfo.FullName -CertPassword $CertPassword -FabricDetails $FabricDetails ` -NetworkControllerRestCertificate -InstallToSouthboundDevices:$NetworkControllerHealthy -Credential $Credential # install the nc node certificate to other network controller nodes if self-signed if ($RotateNodeCerts) { foreach ($controller in $FabricDetails.NetworkController) { "Processing {0} for node certificates" -f $controller | Trace-Output -Level:Verbose $nodeCertConfig = $certificateConfig.NetworkController[$controller] # if we have identified a network controller node certificate then proceed # with installing the cert locally (if matches current node) if ($null -ne $nodeCertConfig.Cert.FileInfo.FullName) { if (Test-ComputerNameIsLocal -ComputerName $controller) { $null = Import-SdnCertificate -FilePath $nodeCertConfig.Cert.FileInfo.FullName -CertPassword $CertPassword -CertStore 'Cert:\LocalMachine\My' } # pass the certificate to sub-function to be seeded across the fabric if necassary Copy-CertificateToFabric -CertFile $nodeCertConfig.Cert.FileInfo.FullName -CertPassword $CertPassword -FabricDetails $FabricDetails -NetworkControllerNodeCert -Credential $Credential } else { "Unable to locate self-signed certificate file for {0}. Node certificate may need to be manually installed to other Network Controllers manually." -f $controller | Trace-Output -Level:Error } } } return $certificateCache } function Export-RegistryKeyConfigDetails { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String[]]$Path, [Parameter(Mandatory = $true)] [System.IO.FileInfo]$OutputDirectory ) try { # create the OutputDirectory if does not already exist if(!(Test-Path -Path $OutputDirectory.FullName -PathType Container)){ $null = New-Item -Path $OutputDirectory.FullName -ItemType Directory -Force } foreach($regKeyPath in $Path){ "Enumerating the registry key paths for {0}" -f $regkeyPath | Trace-Output -Level:Verbose $regKeyDirectories = @() $regKeyDirectories += Get-ChildItem -Path $regKeyPath -ErrorAction SilentlyContinue $regKeyDirectories += Get-ChildItem -Path $regKeyPath -Recurse -ErrorAction SilentlyContinue $regKeyDirectories = $regKeyDirectories | Sort-Object -Unique [System.String]$filePath = "{0}\Registry_{1}.txt" -f $OutputDirectory.FullName, $($regKeyPath.Replace(':','').Replace('\','_')) foreach($obj in $RegKeyDirectories){ "Scanning {0}" -f $obj.PsPath | Trace-Output -Level:Verbose try { $properties = Get-ItemProperty -Path $obj.PSPath -ErrorAction Stop } catch { "Unable to return results from {0}`n`t{1}" -f $obj.PSPath, $_.Exception | Trace-Output -Level:Warning continue } $properties | Out-File -FilePath $filePath -Encoding utf8 -Append # if the registry key item is referencing a dll, then lets get the dll properties so we can see the version and file information if($properties.Path -like "*.dll" -or $properties.Path -like "*.exe"){ "Getting file properties for {0}" -f $properties.Path | Trace-Output -Level:Verbose [System.String]$fileName = "FileInfo_{0}" -f $($properties.Path.Replace(':','').Replace('\','_').Replace('.','_')) Get-Item -Path $properties.Path | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name $fileName -FileType txt -Format List } } } } catch { $_ | Trace-Exception $_ | Write-Error } } function Get-CommonConfigState { <# .SYNOPSIS Retrieves a common set of configuration details that is collected on any role, regardless of the role. #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.IO.FileInfo]$OutputDirectory ) $currentErrorActionPreference = $ErrorActionPreference $ProgressPreference = 'SilentlyContinue' $ErrorActionPreference = 'Ignore' try { [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath "Common" "Collect general configuration state details" | Trace-Output -Level:Verbose if (-NOT (Initialize-DataCollection -FilePath $OutputDirectory.FullName -MinimumMB 100)) { "Unable to initialize environment for data collection" | Trace-Output -Level:Error return } # Gather general configuration details from all nodes "Gathering system details" | Trace-Output -Level:Verbose Get-Service | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-Service' -FileType txt -Format List Get-Process | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-Process' -FileType txt -Format List Get-Volume | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-Volume' -FileType txt -Format Table Get-ComputerInfo | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-ComputerInfo' -FileType txt # gather network related configuration details "Gathering network details" | Trace-Output -Level:Verbose Get-NetTCPConnection | Select-Object LocalAddress, LocalPort, RemoteAddress, RemotePort, State, OwningProcess, @{n="ProcessName";e={(Get-Process -Id $_.OwningProcess -ErrorAction $ErrorActionPreference).ProcessName}} ` | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetTCPConnection' -FileType csv Get-NetIPInterface | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetIPInterface' -FileType txt -Format Table Get-NetNeighbor -IncludeAllCompartments | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetNeighbor' -FileType txt -Format Table Get-NetConnectionProfile | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetConnectionProfile' -FileType txt -Format Table Get-NetRoute -AddressFamily IPv4 -IncludeAllCompartments | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetRoute' -FileType txt -Format Table ipconfig /allcompartments /all | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'ipconfig_allcompartments' -FileType txt Get-NetAdapter | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapter' -FileType txt -Format Table Get-NetAdapterSriov | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapterSriov' -FileType txt -Format Table Get-NetAdapterSriovVf | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapterSriovVf' -FileType txt -Format Table Get-NetAdapterRsc | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapterRsc' -FileType txt -Format Table Get-NetAdapterHardwareInfo | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapterHardwareInfo' -FileType txt -Format Table netsh winhttp show proxy | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'netsh_winhttp_show_proxy' -FileType txt $netAdapter = Get-NetAdapter if ($netAdapter) { $netAdapterRootDir = New-Item -Path (Join-Path -Path $OutputDirectory.FullName -ChildPath 'NetAdapter') -ItemType Directory -Force $netAdapter | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapter' -FileType txt -Format List $netAdapter | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-NetAdapter' -FileType json $netAdapter | ForEach-Object { $prefix = $_.Name.ToString().Replace(' ','_').Trim() $_ | Get-NetAdapterAdvancedProperty | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterAdvancedProperty' -FileType json $_ | Get-NetAdapterBinding | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterBinding' -FileType json $_ | Get-NetAdapterChecksumOffload | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterChecksumOffload' -FileType json $_ | Get-NetAdapterHardwareInfo -ErrorAction $ErrorActionPreference | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterHardwareInfo' -FileType json $_ | Get-NetAdapterRsc -ErrorAction $ErrorActionPreference | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterRsc' -FileType json $_ | Get-NetAdapterSriov -ErrorAction $ErrorActionPreference | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterSriov' -FileType json $_ | Get-NetAdapterStatistics | Export-ObjectToFile -FilePath $netAdapterRootDir.FullName -Prefix $prefix -Name 'Get-NetAdapterStatistics' -FileType json } } # Gather DNS client settings Get-DnsClient | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClient' -FileType txt -Format List Get-DnsClientCache | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientCache' -FileType txt -Format List Get-DnsClientServerAddress | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientServerAddress' -FileType txt -Format List Get-DnsClientGlobalSetting | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientGlobalSetting' -FileType txt -Format List Get-DnsClientNrptGlobal | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientNrptGlobal' -FileType txt -Format List Get-DnsClientNrptPolicy | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientNrptPolicy' -FileType txt -Format List Get-DnsClientNrptRule | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientNrptRule' -FileType txt -Format List Get-DnsClientServerAddress | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-DnsClientServerAddress' -FileType txt -Format List # gather the certificates configured on the system $certificatePaths = @('Cert:\LocalMachine\My','Cert:\LocalMachine\Root') foreach ($path in $certificatePaths) { $fileName = $path.Replace(':','').Replace('\','_') Get-SdnCertificate -Path $path | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name "Get-SdnCertificate_$($fileName)" -FileType csv } } catch { $_ | Trace-Exception $_ | Write-Error } $ProgressPreference = 'Continue' $ErrorActionPreference = $currentErrorActionPreference } function Get-SdnRole { <# .SYNOPSIS Retrieve the SDN Role for a given computername #> [CmdletBinding()] param ( [Parameter(Mandatory = $true, ValueFromPipeline)] [System.String]$ComputerName, [Parameter(Mandatory = $true)] [System.Object]$EnvironmentInfo ) # get the NetBIOS and FQDN name of the computer $result = Get-ComputerNameFQDNandNetBIOS -ComputerName $ComputerName # enumerate the objects for each of the available SDN roles to find a match # once match is found, return the role name as string back to calling function foreach ($role in $EnvironmentInfo.Keys) { if ($role -ieq 'FabricNodes') { continue } foreach ($object in $EnvironmentInfo[$role]) { if ($object -ieq $result.ComputerNameNetBIOS -or $object -ieq $result.ComputerNameFQDN) { return $role.ToString() } } } # if we made it to here, we were unable to locate any specific SdnRole such as LoadBalancerMux, Gateway, etc. # so instead we will return Common as the role return ([string]"Common") } function Get-TraceProviders { <# .SYNOPSIS Get ETW Trace Providers based on Role .PARAMETER Role The SDN Roles .PARAMETER Providers Allowed values are Default,Optional And All to control what are the providers needed #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateSet('Common', 'Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')] [String]$Role, [Parameter(Mandatory = $false)] [ValidateSet("Default", "Optional", "All")] [string]$Providers = "Default", [Parameter(Mandatory = $false)] [Switch]$AsString ) $traceProvidersArray = @() try { $config = Get-SdnModuleConfiguration -Role $Role if ($null -eq $config.properties.EtwTraceProviders) { return $null } foreach ($key in $config.properties.EtwTraceProviders.Keys) { $traceProvider = $config.properties.EtwTraceProviders[$key] switch ($Providers) { "Default" { if ($traceProvider.isOptional -ne $true) { $traceProvidersArray += [PSCustomObject]@{ Name = $key Properties = $traceProvider } } } "Optional" { if ($traceProvider.isOptional -eq $true) { $traceProvidersArray += [PSCustomObject]@{ Name = $key Properties = $traceProvider } } } "All" { $traceProvidersArray += [PSCustomObject]@{ Name = $key Properties = $traceProvider } } } } # we want to be able to return string value back so it can then be passed to netsh trace command # enumerate the properties that have values to build a formatted string that netsh expects if ($PSBoundParameters.ContainsKey('AsString') -and $traceProvidersArray) { [string]$formattedString = $null foreach ($traceProvider in $traceProvidersArray) { foreach ($provider in $traceProvider.Properties.Providers) { $formattedString += "$(Format-NetshTraceProviderAsString -Provider $provider -Level $traceProvider.level -Keywords $traceProvider.keywords) " } } return $formattedString.Trim() } return $traceProvidersArray } catch { $_ | Trace-Exception $_ | Write-Error } } function New-SdnDiagNetworkMappedShare { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateScript({ if ($_.contains("\\") -and $_.contains("\")) { return $true } else { throw "The network share path must be in the format of \\server\share" } })] [System.String]$NetworkSharePath, [Parameter(Mandatory = $false)] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty ) try { "Creating new drive mapping to {0}" -f $NetworkSharePath | Trace-Output # create a new drive mapping to the network share path # if the credential is empty, we will not use a credential if ($Credential -eq [System.Management.Automation.PSCredential]::Empty) { $null = New-PSDrive -Name "SdnDiag_NetShare_Logs" -PSProvider FileSystem -Root $NetworkSharePath -ErrorAction Stop } else { $null = New-PSDrive -Name "SdnDiag_NetShare_Logs" -PSProvider FileSystem -Root $NetworkSharePath -Credential $Credential -ErrorAction Stop } "Successfully created network share mapping to {0}" -f $NetworkSharePath | Trace-Output return $true } catch { $_ | Trace-Exception return $false } } function Reset-SdnDiagTraceMapping { $Script:SdnDiagnostics_Common.Cache['TraceMapping'] = @{} } function Start-EtwTraceSession { <# .SYNOPSIS Start the ETW trace with TraceProviders included. .PARAMETER TraceName The trace name to identify the ETW trace session .PARAMETER TraceProviders The trace providers in string format that you want to trace on .PARAMETER TraceFile The trace file that will be written. .PARAMETER MaxTraceSize Optional. Specifies the maximum size in MB for saved trace files. If unspecified, the default is 1024. #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [string]$TraceName, [Parameter(Mandatory = $true)] [string[]]$TraceProviders, [Parameter(Mandatory = $true)] [ValidateScript( { if ($_ -notmatch "(\.etl)") { throw "The file specified in the TraceFile argument must be etl extension" } return $true })] [System.IO.FileInfo]$TraceFile, [Parameter(Mandatory = $false)] [int]$MaxTraceSize = 1024 ) try { $logmanCmd = "logman create trace $TraceName -ow -o $TraceFile -nb 16 16 -bs 1024 -mode Circular -f bincirc -max $MaxTraceSize -ets" $result = Invoke-Expression -Command $logmanCmd # Session create failure error need to be reported to user to be aware, this means we have one trace session missing. # Provider add failure might be ignored and exposed via verbose trace/log file only to debug. if ("$result".Contains("Error")) { "Create session {0} failed with error {1}" -f $TraceName, "$result" | Trace-Output -Level:Warning } else { "Created session {0} with result {1}" -f $TraceName, "$result" | Trace-Output -Level:Verbose } foreach ($provider in $TraceProviders) { $logmanCmd = 'logman update trace $TraceName -p "$provider" 0xffffffffffffffff 0xff -ets' $result = Invoke-Expression -Command $logmanCmd "Added provider {0} with result {1}" -f $provider, "$result" | Trace-Output -Level:Verbose } } catch { $_ | Trace-Exception $_ | Write-Error } } function Start-NetshTrace { <# .SYNOPSIS Enables netsh tracing. Supports pre-configured trace providers or custom provider strings. .PARAMETER TraceProviderString The trace providers in string format that you want to trace on. .PARAMETER OutputDirectory Specifies a specific path and folder in which to save the files. .PARAMETER MaxTraceSize Optional. Specifies the maximum size in MB for saved trace files. If unspecified, the default is 1024. .PARAMETER Capture Optional. Specifies whether packet capture is enabled in addition to trace events. If unspecified, the default is No. .PARAMETER Overwrite Optional. Specifies whether this instance of the trace conversion command overwrites files that were rendered from previous trace conversions. If unspecified, the default is Yes. .PARAMETER Correlation Optional. Specifies whether related events will be correlated and grouped together. If unspecified, the default is disabled. .PARAMETER Report Optional. Specifies whether a complementing report will be generated in addition to the trace file report. If unspecified, the default is disabled. .EXAMPLE PS> Start-NetshTrace -OutputDirectory "C:\Temp\CSS_SDN" -Capture Yes .EXAMPLE PS> Start-NetshTrace -OutputDirectory "C:\Temp\CSS_SDN" -TraceProviderString 'provider="{EB171376-3B90-4169-BD76-2FB821C4F6FB}" level=0xff' -Capture No .EXAMPLE PS> Start-NetshTrace -OutputDirectory "C:\Temp\CSS_SDN" -TraceProviderString 'provider="{EB171376-3B90-4169-BD76-2FB821C4F6FB}" level=0xff' -Capture Yes .EXAMPLE PS> Start-NetshTrace -OutputDirectory "C:\Temp\CSS_SDN" -Capture Yes -MaxTraceSize 2048 -Report Disabled .EXAMPLE #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.IO.FileInfo]$OutputDirectory, [Parameter(Mandatory = $false)] [System.String]$TraceProviderString, [Parameter(Mandatory = $false)] [int]$MaxTraceSize = 1024, [Parameter(Mandatory = $false)] [ValidateSet('Yes', 'No')] [System.String]$Capture = 'No', [Parameter(Mandatory = $false)] [ValidateSet('Yes', 'No')] [System.String]$Overwrite = 'Yes', [Parameter(Mandatory = $false)] [ValidateSet('Yes', 'No', 'Disabled')] [System.String]$Report = 'Disabled', [Parameter(Mandatory = $false)] [ValidateSet('Yes', 'No', 'Disabled')] [System.String]$Correlation = 'Disabled' ) try { # ensure that we at least are attempting to configure NDIS tracing or ETW provider tracing, else the netsh # command will return a generic exception that is not useful to the operator if ($Capture -ieq 'No' -and !$TraceProviderString) { throw New-Object System.Exception("You must at least specify Capture or TraceProviderString parameter") } # ensure that the directory exists and specify the trace file name if (!(Test-Path -Path $OutputDirectory.FullName -PathType Container)) { $null = New-Item -Path $OutputDirectory.FullName -ItemType Directory -Force } $traceFile = "{0}\{1}_{2}_netshTrace.etl" -f $OutputDirectory.FullName, $env:COMPUTERNAME, (Get-FormattedDateTimeUTC) # enable the network trace if ($TraceProviderString) { $cmd = "netsh trace start capture={0} {1} tracefile={2} maxsize={3} overwrite={4} report={5} correlation={6}" ` -f $Capture, $TraceProviderString, $traceFile, $MaxTraceSize, $Overwrite, $Report, $Correlation } else { $cmd = "netsh trace start capture={0} tracefile={1} maxsize={2} overwrite={3} report={4}, correlation={5}" ` -f $Capture, $traceFile, $MaxTraceSize, $Overwrite, $Report, $Correlation } "Starting netsh trace" | Trace-Output "Netsh trace cmd:`n`t{0}" -f $cmd | Trace-Output -Level:Verbose $expression = Invoke-Expression -Command $cmd if ($expression -ilike "*Running*") { $object = New-Object -TypeName PSCustomObject -Property ( [Ordered]@{ Status = 'Running' FileName = $traceFile } ) } elseif ($expression -ilike "*A tracing session is already in progress*") { "A tracing session is already in progress" | Trace-Output -Level:Warning $object = New-Object -TypeName PSCustomObject -Property ( [Ordered]@{ Status = 'Running' } ) } else { # typically, the first line returned in scenarios where there was an error thrown will contain the error details $msg = $expression[0] throw New-Object System.Exception($msg) } return $object } catch { $_ | Trace-Exception $_ | Write-Error } } function Stop-EtwTraceSession { <# .SYNOPSIS Stop ETW Trace Session .PARAMETER TraceName The trace name to identify the ETW trace session #> [CmdletBinding()] param ( [Parameter(Mandatory = $false)] [string[]]$ComputerName, [Parameter(Mandatory = $false)] [string]$TraceName = $null ) try { $logmanCmd = "logman stop $TraceName -ets" $result = Invoke-Expression -Command $logmanCmd if ("$result".Contains("Error")) { "Stop session {0} failed with error {1}" -f $TraceName, "$result" | Trace-Output -Level:Warning } else { "Stop session {0} with result {1}" -f $TraceName, "$result" | Trace-Output -Level:Verbose } } catch { $_ | Trace-Exception $_ | Write-Error } } function Stop-NetshTrace { <# .SYNOPSIS Disables netsh tracing. #> try { "Stopping trace" | Trace-Output $expression = Invoke-Expression -Command "netsh trace stop" if ($expression -ilike "*Tracing session was successfully stopped.*") { "Tracing was successfully stopped" | Trace-Output -Level:Verbose $object = New-Object -TypeName PSCustomObject -Property ( [Ordered]@{ Status = 'Stopped' } ) } elseif ($expression -ilike "*There is no trace session currently in progress.*") { "There is no trace session currently in progress" | Trace-Output -Level:Warning $object = New-Object -TypeName PSCustomObject -Property ( [Ordered]@{ Status = 'Not Running' } ) } else { # typically, the first line returned in scenarios where there was an error thrown will contain the error details $msg = $expression[0] throw New-Object System.Exception($msg) } return $object } catch { $_ | Trace-Exception $_ | Write-Error } } function Update-SdnDiagTraceMapping { param ( [Parameter(Mandatory=$true)] [string]$MacAddress, [Parameter(Mandatory=$true)] [string]$InfraHost, [Parameter(Mandatory=$false)] [string]$PortId, [Parameter(Mandatory=$false)] [string]$PortName, [Parameter(Mandatory=$false)] [string]$NicName, [Parameter(Mandatory=$false)] [string]$VmName, [Parameter(Mandatory=$false)] [string]$VmInternalId, [Parameter(Mandatory=$false)] [string[]]$PrivateIpAddress ) $cacheName = 'TraceMapping' if($Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]){ if($PortId){ $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]['PortId'] = $PortId } if($PortName){ $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]['PortName'] = $PortName } if($NicName){ $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]['NicName'] = $NicName } if($VmName){ $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]['VmName'] = $VmName } if($VmInternalId){ $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]['VmInternalId'] = $VmInternalId } if($PrivateIpAddress){ $Script:SdnDiagnostics_Common.Cache[$cacheName][$InfraHost][$MacAddress]['PrivateIpAddress'] = $PrivateIpAddress } } } function Convert-SdnEtwTraceToTxt { <# .SYNOPSIS Used to convert existing etw (.etl) provider traces into text readable format .PARAMETER FileName ETL trace file path and name to convert .PARAMETER Destination Output file name and directory. If omitted, will use the FileName path and base name. .PARAMETER Overwrite Overwrites existing files. If omitted, defaults to no. .PARAMETER Report Generates an HTML report. If omitted, defaults to no. .EXAMPLE PS> Convert-SdnEtwTraceToTxt -FileName "C:\Temp\CSS_SDN\Trace.etl" .EXAMPLE PS> Convert-SdnEtwTraceToTxt -FileName "C:\Temp\CSS_SDN\Trace.etl" -Destination "C:\Temp\CSS_SDN_NEW\trace.txt" .EXAMPLE PS> Convert-SdnEtwTraceToTxt -FileName "C:\Temp\CSS_SDN\Trace.etl" -Overwrite Yes .EXAMPLE PS> Convert-SdnEtwTraceToTxt -FileName "C:\Temp\CSS_SDN\Trace.etl" -Report Yes #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateScript( { if ($_ -notmatch "(\.etl)") { throw "The file specified in the FileName argument must be etl extension" } return $true })] [System.String]$FileName, [Parameter(Mandatory = $false)] [System.String]$Destination, [Parameter(Mandatory = $false)] [ValidateSet('No', 'Yes')] [System.String]$Overwrite = 'No', [Parameter(Mandatory = $false)] [ValidateSet('No', 'Yes')] [System.String]$Report = 'No' ) try { $fileInfo = Get-Item -Path $FileName -ErrorAction Stop if (-NOT $PSBoundParameters.ContainsKey('Destination')) { [System.String]$Destination = $fileInfo.DirectoryName } if (-NOT (Test-Path -Path $Destination -PathType Container)) { $null = New-Item -Path $Destination -ItemType Directory -Force } [System.String]$outputFile = "{0}.txt" -f (Join-Path -Path $Destination -ChildPath $fileInfo.BaseName) [System.String]$cmd = "netsh trace convert input={0} output={1} overwrite={2} report={3}" ` -f $fileInfo.FullName, $outputFile, $Overwrite, $Report "Netsh trace cmd:`n`t{0}" -f $cmd | Trace-Output -Level:Verbose $expression = Invoke-Expression -Command $cmd # output returned is string objects, so need to manually do some mapping to correlate the properties # that can be then returned as psobject to the call if ($expression[5] -ilike "*done*") { $object = New-Object -TypeName PSCustomObject -Property ( [Ordered]@{ Status = 'Success' FileName = $outputFile } ) } else { # typically, the first line returned in scenarios where there was an error thrown will contain the error details $msg = $expression[0] throw New-Object System.Exception($msg) } return $object } catch { $_ | Trace-Exception $_ | Write-Error } } function Enable-SdnVipTrace { <# .SYNOPSIS Enables network tracing on the SDN fabric infrastructure related to the specified VIP address. .PARAMETER VirtualIP Specify the Virtual IP address that you want to enable SDN fabric tracing for. .PARAMETER NcUri Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller. .PARAMETER Credential Specifies a user account that has permission to perform this action. The default is the current user. .PARAMETER NcRestCredential Specifies a user account that has permission to access the northbound NC API interface. The default is the current user. .PARAMETER OutputDirectory Optional. Specifies a specific path and folder in which to save the files. .PARAMETER MaxTraceSize Optional. Specifies the maximum size in MB for saved trace files. If unspecified, the default is 1536. #> [CmdletBinding(DefaultParameterSetName = 'Default')] param ( [Parameter(Mandatory = $true, ParameterSetName = 'Default')] [System.String]$VirtualIP, [Parameter(Mandatory = $true, ParameterSetName = 'Default')] [ValidateScript({ if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") { throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.") } return $true })] [Uri]$NcUri, [Parameter(Mandatory = $false, ParameterSetName = 'Default')] [System.String]$OutputDirectory = "$(Get-WorkingDirectory)\NetworkTraces", [Parameter(Mandatory = $false, ParameterSetName = 'Default')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $NcRestCredential = [System.Management.Automation.PSCredential]::Empty, [Parameter(Mandatory = $false, ParameterSetName = 'Default')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty, [Parameter(Mandatory = $false, ParameterSetName = 'Default')] [int]$MaxTraceSize = 1536 ) $networkTraceNodes = @() Reset-SdnDiagTraceMapping try { # lets try and locate the resources associated with the public VIP address # the SdnPublicIpPoolUsageSummary is useful for this scenario, as it has the logic to scan both publicIpAddresses and loadBalancers # to locate the VIP IP we are looking for $publicIpAddressUsage = Get-SdnPublicIPPoolUsageSummary -NcUri $NcUri -NcRestCredential $NcRestCredential $publicIpResource = $publicIpAddressUsage | Where-Object {$_.IPAddress -ieq $VirtualIP} if ($null -ieq $publicIpResource) { throw "Unable to locate resources associated to $VirtualIP" } # get the load balancer muxes, as we will need to enable tracing on them $loadBalancerMuxes = Get-SdnLoadBalancerMux -NcUri $NcUri -Credential $NcRestCredential -ManagementAddressOnly $networkTraceNodes += $loadBalancerMuxes # we want to query the servers within the SDN fabric so we can get a list of the vfp switch ports across the hyper-v hosts # as we will use this reference to locate where the resources are located within the fabric $servers = Get-SdnServer -NcUri $NcUri -Credential $NcRestCredential -ManagementAddressOnly $Script:SdnDiagnostics_Common.Cache['VfpSwitchPorts'] = Get-SdnVfpVmSwitchPort -ComputerName $servers -Credential $Credential # determine the network interfaces associated with the public IP address $associatedResource = Get-SdnResource -NcUri $NcUri -Credential $NcRestCredential -ResourceRef $publicIpResource.AssociatedResource switch -Wildcard ($associatedResource.resourceRef) { "/loadBalancers/*" { "{0} is associated with load balancer {1}" -f $VirtualIP, $associatedResource.resourceRef | Trace-Output # depending on the environments, the associatedResource may come back as the parent load balancer object # or may be the frontend IP configuration object so in either situation, we should just split the resourceRef string and query to get the # parent load balancer object to ensure consistency $parentResource = "{0}/{1}" -f $associatedResource.resourceRef.Split('/')[1], $associatedResource.resourceRef.Split('/')[2] $loadBalancer = Get-SdnResource -NcUri $NcUri -Credential $NcRestCredential -ResourceRef $parentResource $ipConfigurations = $loadBalancer.properties.backendAddressPools.properties.backendIPConfigurations.resourceRef } "/networkInterfaces/*" { "{0} is associated with network interface {1}" -f $VirtualIP, $associatedResource.resourceRef | Trace-Output $ipConfigurations = $associatedResource.resourceRef } # public IP address(es) should only ever be associated to load balancer or network interface resources # except in the case for the gateway pool, which we would not expect in this scenario at this time default { throw "Unable to determine associated resource type" } } $ipConfigurations | ForEach-Object { $ipConfig = Get-SdnResource -NcUri $NcUri -Credential $NcRestCredential -ResourceRef $_ -ErrorAction Stop if ($null -ieq $ipConfig) { throw "Unable to locate resource for $($_)" } "Located associated resource {0} with DIP address {1}" -f $ipConfig.resourceRef, $ipconfig.properties.privateIPAddress | Trace-Output # we need the mac address of the network interface to locate the vfp switch port # since the ipConfiguration is a subobject of the network interface, we need to split the resourceRef to get the network interface resource # since we know the resourceRefs are defined as /networkInterfaces/{guid}/ipConfigurations/{guid}, we can split on the '/' and get the 3rd element $netInterface = Get-SdnResource -NcUri $NcUri -Credential $NcRestCredential -ResourceRef "/networkInterfaces/$($_.Split('/')[2])" $macAddress = Format-MacAddress -MacAddress $netInterface.properties.privateMacAddress -Dashes $vfpPort = $Script:SdnDiagnostics_Common.Cache['VfpSwitchPorts'] | Where-Object {$_.MacAddress -ieq $macAddress} if ($null -ieq $vfpPort) { throw "Unable to locate vfp switch port for $macAddress" } "Located vfp switch port {0} on {1}" -f $vfpPort.PortName, $vfpPort.PSComputerName | Trace-Output # once we have the information we need, we can update our internal cache mapping Add-SdnDiagTraceMapping ` -MacAddress $vfpPort.MacAddress ` -InfraHost $vfpPort.PSComputerName ` -PortId $vfpPort.PortId ` -PortName $vfpPort.Portname ` -NicName $vfpPort.NICname ` -VmName $vfpPort.VMname ` -VmInternalId $vfpPort.VMID ` -PrivateIpAddress $ipConfig.properties.privateIPAddress } # once we have identified all the nodes we will enable tracing on # add the server(s) to the list of nodes we will enable tracing on # as this will be used to disable tracing once we are done $networkTraceNodes += $Script:SdnDiagnostics_Common.Cache['TraceMapping'].Keys $networkTraceNodes = $networkTraceNodes | Select-Object -Unique # ensure that we have SdnDiagnostics installed to the nodes that we need to enable tracing for Install-SdnDiagnostics -ComputerName $networkTraceNodes -Credential $Credential "Network traces will be enabled on:`r`n`t - LoadBalancerMux: {0}`r`n`t - Server: {1}`r`n" ` -f ($loadBalancerMuxes -join ', '), ($Script:SdnDiagnostics_Common.Cache['TraceMapping'].Keys -join ', ') | Trace-Output # enable tracing on the infastructure $traceInfo = @() $traceInfo += Start-SdnNetshTrace -ComputerName $loadBalancerMuxes -Role 'LoadBalancerMux' -Credential $Credential -OutputDirectory $OutputDirectory -MaxTraceSize $MaxTraceSize $traceInfo += Start-SdnNetshTrace -ComputerName $Script:SdnDiagnostics_Common.Cache['TraceMapping'].Keys -Role 'Server' -Credential $Credential -OutputDirectory $OutputDirectory -MaxTraceSize $MaxTraceSize "Tracing has been enabled on the SDN infrastructure nodes {0}" -f ($traceInfo.PSComputerName -join ', ') | Trace-Output # at this point, tracing should be enabled on the sdn fabric and we can wait for user input to disable # once we receive user input, we will disable tracing on the infrastructure node(s) $null = Get-UserInput -Message "`r`nPress any key to disable tracing..." $null = Stop-SdnNetshTrace -ComputerName $networkTraceNodes -Credential $Credential "Tracing has been disabled on the SDN infrastructure. Saving configuration details to {0}\{1}_TraceMapping.json" -f (Get-WorkingDirectory), $VirtualIP | Trace-Output $Script:SdnDiagnostics_Common.Cache['TraceMapping'] | Export-ObjectToFile -FilePath (Get-WorkingDirectory) -Prefix $VirtualIP -Name 'TraceMapping' -FileType json -Depth 3 $traceFileInfo = @() foreach ($obj in $traceInfo) { $traceFileInfo += [PSCustomObject]@{ ComputerName = $obj.PSComputerName FileName = $obj.FileName } } return $traceFileInfo } catch { $_ | Trace-Exception $_ | Write-Error } } function Get-SdnCertificate { <# .SYNOPSIS Returns a list of the certificates within the given certificate store. .PARAMETER Path Defines the path within the certificate store. Path is expected to start with cert:\. .EXAMPLE PS> Get-SdnCertificate -Path "Cert:\LocalMachine\My" #> [CmdletBinding(DefaultParameterSetName = 'Default')] param ( [Parameter(Mandatory = $true, ParameterSetName = 'Default')] [Parameter(Mandatory = $true, ParameterSetName = 'Subject')] [Parameter(Mandatory = $true, ParameterSetName = 'Thumbprint')] [ValidateScript({ if ($_ -notlike "cert:\*") { throw New-Object System.FormatException("Invalid path") } return $true })] [System.String]$Path, [Parameter(Mandatory = $false, ParameterSetName = 'Subject')] [ValidateNotNullorEmpty()] [System.String]$Subject, [Parameter(Mandatory = $false, ParameterSetName = 'Thumbprint')] [ValidateNotNullorEmpty()] [System.String]$Thumbprint ) try { $certificateList = Get-ChildItem -Path $Path -Recurse | Where-Object {$_.PSISContainer -eq $false} -ErrorAction Stop switch ($PSCmdlet.ParameterSetName) { 'Subject' { $filteredCert = $certificateList | Where-Object {$_.Subject -ieq $Subject} } 'Thumbprint' { $filteredCert = $certificateList | Where-Object {$_.Thumbprint -ieq $Thumbprint} } default { return $certificateList } } if ($null -eq $filteredCert) { "Unable to locate certificate using {0}" -f $PSCmdlet.ParameterSetName | Trace-Output -Level:Warning return $null } $filteredCert | ForEach-Object { if ($_.NotAfter -le (Get-Date)) { "Certificate [Thumbprint: {0} | Subject: {1}] is currently expired" -f $_.Thumbprint, $_.Subject | Trace-Output -Level:Warning } } return $filteredCert } catch { $_ | Trace-Exception $_ | Write-Error } } function Get-SdnDiagnosticLogFile { <# .SYNOPSIS Collect the default enabled logs from SdnDiagnostics folder. .PARAMETER OutputDirectory Specifies a specific path and folder in which to save the files. .PARAMETER FromDate Determines the start time of what logs to collect. If omitted, defaults to the last 4 hours. .PARAMETER ToDate Determines the end time of what logs to collect. Optional parameter that if ommitted, defaults to current time. .PARAMETER ConvertETW Optional parameter that allows you to specify if .etl trace should be converted. By default, set to $true .EXAMPLE PS> Get-SdnDiagnosticLogFile -LogDir "C:\Windows\Tracing\SdnDiagnostics" -OutputDirectory "C:\Temp\CSS_SDN" .EXAMPLE PS> Get-SdnDiagnosticLogFile -LogDir "C:\Windows\Tracing\SdnDiagnostics" -FromDate (Get-Date).AddHours(-1) .EXAMPLE PS> Get-SdnDiagnosticLogFile -LogDir "C:\Windows\Tracing\SdnDiagnostics" -FromDate '2023-08-11 10:00:00 AM' -ToDate '2023-08-11 11:30:00 AM' #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String[]]$LogDir, [Parameter(Mandatory = $true)] [System.IO.DirectoryInfo]$OutputDirectory, [Parameter(Mandatory = $false)] [DateTime]$FromDate = (Get-Date).AddHours(-4), [Parameter(Mandatory = $false)] [DateTime]$ToDate = (Get-Date), [Parameter(Mandatory = $false)] [bool]$ConvertETW = $true, [Parameter(Mandatory = $false)] [bool]$CleanUpFiles = $false, [Parameter(Mandatory = $false)] [string[]]$FolderNameFilter ) begin { $fromDateUTC = $FromDate.ToUniversalTime() $toDateUTC = $ToDate.ToUniversalTime() $commonConfig = Get-SdnModuleConfiguration -Role 'Common' } process { $LogDir | ForEach-Object { $folder = Get-Item -Path $_ -ErrorAction SilentlyContinue # if the folder is not found, then log a message and continue to the next folder if ($null -ieq $folder) { "Unable to locate {0}" -f $_ | Trace-Output -Level:Verbose return } $logFiles = @() $getItemParams = @{ Path = $folder.FullName Include = $commonConfig.LogFileTypes Recurse = $true ErrorAction = 'SilentlyContinue' } "Scanning for {0} in {1} between {2} and {3} UTC" -f ($commonConfig.LogFileTypes -join ', '), $folder.FullName, $fromDateUTC, $toDateUTC | Trace-Output -Level:Verbose if ($FolderNameFilter) { $FolderNameFilter | ForEach-Object { [string]$filter = $_ $unfilteredlogFiles = Get-ChildItem @getItemParams | Where-Object { $_.LastWriteTime.ToUniversalTime() -ge $fromDateUTC -and $_.LastWriteTime.ToUniversalTime() -le $toDateUTC } if ($unfilteredlogFiles) { "Filtering logs related to DirectoryName contains '{0}'" -f $filter | Trace-Output -Level:Verbose $logFiles += $unfilteredlogFiles | Where-Object { $_.DirectoryName -ilike "*$filter*" } } } } else { $logFiles += Get-ChildItem @getItemParams | Where-Object { $_.LastWriteTime.ToUniversalTime() -ge $fromDateUTC -and $_.LastWriteTime.ToUniversalTime() -le $toDateUTC } } if ($logFiles) { # enumerate the group of log files based on the directory # and then create a dynamic directory based on the folder name in an effort to preserve the original directory structure $logDirectory = $logFiles | Group-Object -Property Directory $logDirectory | ForEach-Object { $splitIndex = $_.Name.IndexOf($folder.Name) [System.IO.DirectoryInfo]$outputPath = Join-Path -Path $OutputDirectory.FullName -ChildPath $_.Name.Substring($splitIndex) # we want to call the initialize datacollection after we have identify the amount of disk space we will need to create a copy of the logs # once the disk space is identified, we will initialize the data collection and copy the files to the output directory $minimumDiskSpace = [float](Get-FolderSize -FileName $logFiles.FullName -Total).GB * 3.5 if (-NOT (Initialize-DataCollection -FilePath $outputPath.FullName -MinimumGB $minimumDiskSpace)) { "Unable to copy files from {0} to {1}" -f $_.Name, $outputPath.FullName | Trace-Output -Level:Error continue } else { "Copying {0} files to {1}" -f $_.Group.Count, $outputPath.FullName | Trace-Output $_.Group | Copy-Item -Destination $outputPath.FullName -Force -ErrorAction Continue } # convert the most recent etl trace file into human readable format without requirement of additional parsing tools if ($ConvertETW) { $convertFile = Get-ChildItem -Path $outputPath.FullName -Include '*.etl' -Recurse | Sort-Object -Property LastWriteTime | Select-Object -Last 1 if ($convertFile) { $null = Convert-SdnEtwTraceToTxt -FileName $convertFile.FullName -Overwrite 'Yes' } } try { # compress the files into a single zip file "Compressing results to {0}.zip" -f $outputPath.FullName | Trace-Output Compress-Archive -Path "$($outputPath.FullName)\*" -Destination "$($outputPath.FullName).zip" -CompressionLevel Optimal -Force # once we have copied the files to the new location we want to compress them to reduce disk space # if confirmed we have a .zip file, then remove the staging folder if (Test-Path -Path "$($outputPath.FullName).zip" -PathType Leaf) { Clear-SdnWorkingDirectory -Path $outputPath.FullName -Force -Recurse } # if we opted to clean up the files, then proceed to do so now if ($CleanUpFiles) { "Cleaning up files" | Trace-Output -Level:Verbose Clear-SdnWorkingDirectory -Path $logFiles.FullName -Force -Recurse } } catch { "Unable to compress files to {0}" -f "$($folder.FullName).zip" | Trace-Output -Level:Error } } } else { "No log files found under {0} between {1} and {2} UTC." -f $folder.FullName, $fromDateUTC, $toDateUTC | Trace-Output -Level:Verbose } } } } function Get-SdnEventLog { <# .SYNOPSIS Collect the Windows Event Logs for different SDN Roles. .PARAMETER Role The specific SDN role to collect windows event logs from. .PARAMETER OutputDirectory Specifies a specific path and folder in which to save the files. .PARAMETER FromDate Determines the start time of what logs to collect. If omitted, defaults to the last 1 day. .PARAMETER ToDate Determines the end time of what logs to collect. Optional parameter that if ommitted, defaults to current time. .EXAMPLE PS> Get-SdnEventLog -OutputDirectory "C:\Temp\CSS_SDN" .EXAMPLE PS> Get-SdnEventLog -OutputDirectory "C:\Temp\CSS_SDN" -FromDate (Get-Date).AddHours(-12) #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateSet('Common', 'Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')] [String[]]$Role, [Parameter(Mandatory = $true)] [System.IO.FileInfo]$OutputDirectory, [parameter(Mandatory = $false)] [DateTime]$FromDate = (Get-Date).AddDays(-1), [Parameter(Mandatory = $false)] [DateTime]$ToDate = (Get-Date) ) $fromDateUTC = $FromDate.ToUniversalTime() $toDateUTC = $ToDate.ToUniversalTime() [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath "EventLogs" $eventLogs = @() $eventLogProviders = @() "Collect event logs between {0} and {1} UTC" -f $fromDateUTC, $toDateUTC | Trace-Output if (-NOT (Initialize-DataCollection -FilePath $OutputDirectory.FullName -MinimumMB 200)) { "Unable to initialize environment for data collection" | Trace-Output -Level:Error return } try { $Role | ForEach-Object { $roleConfig = Get-SdnModuleConfiguration -Role $_ $eventLogProviders += $roleConfig.Properties.EventLogProviders } # check to see if the event log provider is valid # and that we have events to collect "Collect the following events: {0}" -f ($eventLogProviders -join ', ') | Trace-Output foreach ($provider in $eventLogProviders) { "Looking for event matching {0}" -f $provider | Trace-Output -Level:Verbose $eventLogsToAdd = Get-WinEvent -ListLog $provider -ErrorAction SilentlyContinue | Where-Object { $_.RecordCount } if ($eventLogsToAdd) { $eventLogs += $eventLogsToAdd } else { "No events found for {0}" -f $provider | Trace-Output } } # process each of the event logs identified # and export them to csv and evtx files foreach ($eventLog in $eventLogs) { $fileName = ("{0}\{1}" -f $OutputDirectory.FullName, $eventLog.LogName).Replace("/", "_") "Export event log {0} to {1}" -f $eventLog.LogName, $fileName | Trace-Output -Level:Verbose $events = Get-WinEvent -LogName $eventLog.LogName -ErrorAction SilentlyContinue ` | Where-Object { $_.TimeCreated.ToUniversalTime() -gt $fromDateUTC -AND $_.TimeCreated -lt $toDateUTC } if ($events) { $events | Select-Object TimeCreated, LevelDisplayName, Id, ProviderName, ProviderID, TaskDisplayName, OpCodeDisplayName, Message ` | Export-Csv -Path "$fileName.csv" -NoTypeInformation -Force } wevtutil epl $eventLog.LogName "$fileName.evtx" /ow:$true } } catch { $_ | Trace-Exception $_ | Write-Error } } function Import-SdnCertificate { <# .SYNOPSIS Imports certificates (CER) and private keys from a Personal Information Exchange (PFX) file to the destination store. .PARAMETER FilePath Specifies the full path to the PFX or CER file. .PARAMETER CertStore Specifies the path of the store to which certificates will be imported. If paramater is not specified, defaults to Cert:\LocalMachine\Root. .PARAMETER CertPassword Specifies the password for the imported PFX file in the form of a secure string. .EXAMPLE PS> Import-SdnCertificate -FilePath c:\certs\cert.pfx -CertStore Cert:\LocalMachine\Root .EXAMPLE PS> Import-SdnCertificate -FilePath c:\certs\cert.pfx -CertStore Cert:\LocalMachine\Root -Password $secureString #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String]$FilePath, [Parameter(Mandatory = $true)] [System.String]$CertStore, [Parameter(Mandatory = $false)] [System.Security.SecureString]$CertPassword ) $trustedRootStore = 'Cert:\LocalMachine\Root' $fileInfo = Get-Item -Path $FilePath $certObject = @{ SelfSigned = $false CertInfo = $null CerFileInfo = $null } switch ($fileInfo.Extension) { '.pfx' { if ($CertPassword) { $certData = (Get-PfxData -FilePath $fileInfo.FullName -Password $CertPassword).EndEntityCertificates } else { $certData = Get-PfxCertificate -FilePath $fileInfo.FullName } } '.cer' { $certData = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2 $certData.Import($fileInfo) } default { throw New-Object System.NotSupportedException("Unsupported certificate extension") } } $certExists = Get-ChildItem -Path $CertStore | Where-Object {$_.Thumbprint -ieq $certData.Thumbprint} if ($certExists) { "{0} already exists under {1}" -f $certExists.Thumbprint, $CertStore | Trace-Output -Level:Verbose $certObject.CertInfo = $certExists } else { "Importing {0} to {1}" -f $certData.Thumbprint, $CertStore | Trace-Output if ($certData.HasPrivateKey) { $importCert = Import-PfxCertificate -FilePath $fileInfo.FullName -CertStoreLocation $CertStore -Password $CertPassword -Exportable -ErrorAction Stop Set-SdnCertificateAcl -Path $CertStore -Thumbprint $importCert.Thumbprint } else { $importCert = Import-Certificate -FilePath $fileInfo.FullName -CertStoreLocation $CertStore -ErrorAction Stop } $certObject.CertInfo = $importCert } # determine if the certificates being used are self signed if ($certObject.CertInfo.Subject -ieq $certObject.CertInfo.Issuer) { "Detected the certificate subject and issuer are the same. Setting SelfSigned to true" | Trace-Output -Level:Verbose $certObject.SelfSigned = $true # check to see if we installed to root store with above operation # if it is not, then we want to check the root store to see if this certificate has already been installed # and finally if does not exist, then export the certificate from current store and import into trusted root store if ($CertStore -ine $trustedRootStore) { $selfSignedCerExists = Get-ChildItem -Path $trustedRootStore | Where-Object {$_.Thumbprint -ieq $certObject.CertInfo.Thumbprint} [System.String]$selfSignedCerPath = "{0}\{1}.cer" -f (Split-Path $fileInfo.FullName -Parent), ($certObject.CertInfo.Subject).Replace('=','_') $selfSignedCer = Export-Certificate -Cert $certObject.CertInfo -FilePath $selfSignedCerPath -ErrorAction Stop $certObject.CerFileInfo = $selfSignedCer if (-NOT ($selfSignedCerExists)) { # import the certificate to the trusted root store "Importing public key to {0}" -f $trustedRootStore | Trace-Output $null = Import-Certificate -FilePath $selfSignedCer.FullName -CertStoreLocation $trustedRootStore -ErrorAction Stop } else { "{0} already exists under {1}" -f $certObject.CertInfo.Thumbprint, $trustedRootStore | Trace-Output -Level:Verbose } } } return $certObject } function Invoke-SdnGetNetView { <# .SYNOPSIS Invokes Get-Netview function on the specified ComputerNames. .PARAMETER OutputDirectory Specifies a specific path and folder in which to save the files. .PARAMETER BackgroundThreads Maximum number of background tasks, from 0 - 16. Defaults to 5. .PARAMETER SkipAdminCheck If present, skip the check for admin privileges before execution. Note that without admin privileges, the scope and usefulness of the collected data is limited. .PARAMETER SkipLogs If present, skip the EVT and WER logs gather phases. .PARAMETER SkipNetshTrace If present, skip the Netsh Trace data gather phases. .PARAMETER SkipCounters If present, skip the Windows Performance Counters (WPM) data gather phases. .PARAMETER SkipVM If present, skip the Virtual Machine (VM) data gather phases. .EXAMPLE PS> Invoke-SdnGetNetView -OutputDirectory "C:\Temp\CSS_SDN" #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.IO.FileInfo]$OutputDirectory, [Parameter(Mandatory = $false)] [int]$BackgroundThreads = 5, [Parameter(Mandatory = $false)] [switch]$SkipAdminCheck, [Parameter(Mandatory = $false)] [switch]$SkipLogs, [Parameter(Mandatory = $false)] [switch]$SkipNetshTrace, [Parameter(Mandatory = $false)] [switch]$SkipCounters, [Parameter(Mandatory = $false)] [switch]$SkipVm ) try { Copy-Item -Path "$PSScriptRoot\..\..\..\externalPackages\Get-NetView" -Destination "$($env:ProgramFiles)\WindowsPowerShell\Modules" -Force -Recurse Import-Module -Name 'Get-NetView' -Force "Using Get-NetView version {0}" -f (Get-Module -Name 'Get-NetView' -ErrorAction SilentlyContinue).Version.ToString() | Trace-Output -Level:Verbose [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath "NetView" # validate the output directory exists, else create the appropriate path if (!(Test-Path -Path $OutputDirectory.FullName -PathType Container)) { $null = New-Item -Path $OutputDirectory.FullName -ItemType Directory -Force } # execute Get-NetView with specified parameters and redirect all streams to null to prevent unnecessary noise on the screen Get-NetView -OutputDirectory $OutputDirectory.FullName ` -BackgroundThreads $BackgroundThreads ` -SkipAdminCheck:$SkipAdminCheck.IsPresent ` -SkipLogs:$SkipLogs.IsPresent ` -SkipNetshTrace:$SkipNetshTrace.IsPresent ` -SkipCounters:$SkipCounters.IsPresent ` -SkipVm:$SkipVm.IsPresent *> $null # remove the uncompressed files and folders to free up ~ 1.5GB of space $compressedArchive = Get-ChildItem -Path $OutputDirectory.FullName -Filter "*.zip" if ($compressedArchive) { Get-ChildItem -Path $OutputDirectory.FullName -Exclude *.zip | Remove-Item -Recurse -Confirm:$false } return $compressedArchive.FullName } catch { $_ | Trace-Exception $_ | Write-Error } } function New-SdnCertificate { <# .SYNOPSIS Creates a new self-signed certificate for use with SDN fabric. .PARAMETER Subject Specifies the string that appears in the subject of the new certificate. This cmdlet prefixes CN= to any value that does not contain an equal sign. .PARAMETER CertStoreLocation Specifies the certificate store in which to store the new certificate. If paramater is not specified, defaults to Cert:\LocalMachine\My. .PARAMETER NotAfter Specifies the date and time, as a DateTime object, that the certificate expires. To obtain a DateTime object, use the Get-Date cmdlet. The default value for this parameter is one year after the certificate was created. .EXAMPLE PS> New-SdnCertificate -Subject rest.sdn.contoso -CertStoreLocation Cert:\LocalMachine\My #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [System.String]$Subject, [Parameter(Mandatory = $false)] [ValidateScript({ if ($_ -notlike "cert:\*") { throw New-Object System.FormatException("Invalid path") } return $true })] [System.String]$CertStoreLocation = 'Cert:\LocalMachine\My', [Parameter(Mandatory = $true)] [System.DateTime]$NotAfter ) try { "Generating certificate with subject {0} under {1}" -f $Subject, $CertStoreLocation | Trace-Output $selfSignedCert = New-SelfSignedCertificate -Type Custom -KeySpec KeyExchange -Subject $Subject ` -KeyExportPolicy Exportable -HashAlgorithm sha256 -KeyLength 2048 ` -CertStoreLocation $CertStoreLocation -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.1,1.3.6.1.5.5.7.3.2,1.3.6.1.4.1.311.95.1.1.1") ` -NotAfter $NotAfter if ($selfSignedCert) { "Successfully generated self signed certificate`n`tSubject: {0}`n`tThumbprint: {1}`n`tNotAfter: {2}" ` -f $selfSignedCert.Subject, $selfSignedCert.Thumbprint, $selfSignedCert.NotAfter | Trace-Output Set-SdnCertificateAcl -Path $CertStoreLocation -Thumbprint $selfSignedCert.Thumbprint } return $selfSignedCert } catch { $_ | Trace-Exception $_ | Write-Error } } function Repair-SdnDiagnosticsScheduledTask { <# .SYNOPSIS Repairs the SDN Diagnostics scheduled task. #> [CmdletBinding()] param() $taskName = "SDN Diagnostics Task" try { $isLoggingEnabled = Get-ItemPropertyValue -Path "HKLM:\Software\Microsoft\NetworkController\Sdn\Diagnostics\Parameters" -Name 'IsLoggingEnabled' if (-NOT $isLoggingEnabled ) { "Logging is currently disabled. Logging must be enabled before the scheduled task can be repaired." | Trace-Output -Level:Warning return $null } $scheduledTask = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop if ($scheduledTask) { # if the scheduled task is disabled, enable it and start it if ($scheduledTask.State -ieq "Disabled") { "Enabling scheduled task." | Trace-Output $scheduledTask | Enable-ScheduledTask -ErrorAction Stop "Starting scheduled task." | Trace-Output Get-ScheduledTask -TaskName $taskName | Start-ScheduledTask -ErrorAction Stop } else { "Scheduled task is already enabled." | Trace-Output } return (Get-ScheduledTask -TaskName $taskName) } else { "Scheduled task does not exist." | Trace-Output -Level:Warning } } catch { $_ | Trace-Exception $_ | Write-Error } } function Set-SdnCertificateAcl { <# .SYNOPSIS Configures NT AUTHORITY/NETWORK SERVICE to have appropriate permissions to the private key of the Network Controller certificates. .PARAMETER Path Specifies the certificate store in which to retrieve the certificate. .PARAMETER Subject Gets the thumbprint of a certificate with the specified store to ensure correct ACLs are defined. .PARAMETER Thumbprint Gets the thumbprint of a certificate with the specified store to ensure correct ACLs are defined. .EXAMPLE PS> Set-SdnCertificateAcl -Path CERT:\LocalMachine\My -Subject 'NCREST.Contoso.Local' #> [CmdletBinding()] param ( [Parameter(Mandatory = $true, ParameterSetName = 'Subject')] [Parameter(Mandatory = $true, ParameterSetName = 'Thumbprint')] [ValidateScript({ if ($_ -notlike "cert:\*") { throw New-Object System.FormatException("Invalid path") } return $true })] [System.String]$Path, [Parameter(Mandatory = $true, ParameterSetName = 'Subject')] [System.String]$Subject, [Parameter(Mandatory = $true, ParameterSetName = 'Thumbprint')] [System.String]$Thumbprint ) try { switch ($PSCmdlet.ParameterSetName) { 'Subject' { $certificate = Get-SdnCertificate -Path $Path -Subject $Subject } 'Thumbprint' { $certificate = Get-SdnCertificate -Path $Path -Thumbprint $Thumbprint } } if ($null -eq $certificate) { throw New-Object System.NullReferenceException("Unable to locate the certificate based on $($PSCmdlet.ParameterSetName)") } else { "Located certificate with Thumbprint: {0} and Subject: {1}" -f $certificate.Thumbprint, $certificate.Subject | Trace-Output -Level:Verbose } if ($certificate.Count -ge 2) { throw New-Object System.Exception("Multiple certificates found matching $($PSCmdlet.ParameterSetName)") } if ($certificate.HasPrivateKey) { $privateKeyCertFile = Get-Item -Path "$($env:ProgramData)\Microsoft\Crypto\RSA\MachineKeys\*" | Where-Object {$_.Name -ieq $($certificate.PrivateKey.CspKeyContainerInfo.UniqueKeyContainerName)} $privateKeyAcl = Get-Acl -Path $privateKeyCertFile.FullName if ($privateKeyAcl.Access.IdentityReference -inotcontains "NT AUTHORITY\NETWORK SERVICE") { $networkServicePermission = "NT AUTHORITY\NETWORK SERVICE", "Read", "Allow" "Configuring {0} on {1}" -f ($networkServicePermission -join ', ').ToString(), $privateKeyCertFile.FullName | Trace-Output $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($networkServicePermission) [void]$privateKeyAcl.AddAccessRule($accessRule) $null = Set-Acl -Path $privateKeyCertFile.FullName -AclObject $privateKeyAcl } else { "Permissions already defined for NT AUTHORITY\NETWORK SERVICE for {0}. No ACL changes required." -f $certificate.PrivateKey.CspKeyContainerInfo.UniqueKeyContainerName | Trace-Output -Level:Verbose } } } catch { $_ | Trace-Exception $_ | Write-Error } } function Start-SdnDataCollection { <# .SYNOPSIS Automated data collection script to pull the current configuration state in conjuction with diagnostic logs and other data points used for debugging. .PARAMETER NetworkController Specifies the name or IP address of the network controller node on which this cmdlet operates. The parameter is optional if running on network controller node. .PARAMETER NcUri Specifies the Uniform Resource Identifier (URI) of the network controller that all Representational State Transfer (REST) clients use to connect to that controller. .PARAMETER Role The specific SDN role(s) to collect configuration state and logs from. .PARAMETER ComputerName Type the NetBIOS name, an IP address, or a fully qualified domain name of one or more remote computers. .PARAMETER OutputDirectory Directory the results will be saved to. If ommitted, will default to the current working directory. .PARAMETER IncludeNetView If enabled, will execute Get-NetView on the Role(s) or ComputerName(s) defined. .PARAMETER IncludeLogs If enabled, will collect the diagnostic logs from the Role(s) or ComputerName(s) defined. Works in conjunction with the FromDate parameter. .PARAMETER FromDate Determines the start time of what logs to collect. If omitted, defaults to the last 4 hours. .PARAMETER ToDate Determines the end time of what logs to collect. Optional parameter that if ommitted, defaults to current time. .PARAMETER Credential Specifies a user account that has permission to perform this action. The default is the current user. .PARAMETER NcRestCredential Specifies a user account that has permission to access the northbound NC API interface. The default is the current user. .PARAMETER Limit Used in conjuction with the Role parameter to limit how many nodes per role operations are performed against. If ommitted, defaults to 16. .PARAMETER ConvertETW Optional parameter that allows you to specify if .etl trace should be converted. By default, set to $true .EXAMPLE PS> Start-SdnDataCollection -NetworkController 'Contoso-NC01' -Role Gateway,NetworkController,Server,LoadBalancerMux .EXAMPLE PS> Start-SdnDataCollection -NetworkController 'Contoso-NC01' -Role Gateway,NetworkController,Server,LoadBalancerMux -IncludeLogs .EXAMPLE PS> Start-SdnDataCollection -NetworkController 'Contoso-NC01' -Role Gateway,Server,LoadBalancerMux -IncludeLogs -FromDate (Get-Date).AddHours(-1) -Credential (Get-Credential) .EXAMPLE PS> Start-SdnDataCollection -NetworkController 'Contoso-NC01' -Role LoadBalancerMux -IncludeLogs -IncludeNetView -FromDate '2023-08-11 10:00:00 AM' -ToDate '2023-08-11 11:30:00 AM' #> [CmdletBinding(DefaultParameterSetName = 'Role')] param ( [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [System.String]$NetworkController = $(HostName), [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [ValidateScript({ if ($_.Scheme -ne "http" -and $_.Scheme -ne "https") { throw New-Object System.FormatException("Parameter is expected to be in http:// or https:// format.") } return $true })] [Uri]$NcUri, [Parameter(Mandatory = $true, ParameterSetName = 'Role')] [ValidateSet('Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')] [String[]]$Role, [Parameter(Mandatory = $true, ParameterSetName = 'Computer')] [System.String[]]$ComputerName, [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [System.IO.FileInfo]$OutputDirectory = (Get-WorkingDirectory), [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [Switch]$IncludeNetView, [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [Switch]$IncludeLogs, [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [DateTime]$FromDate = (Get-Date).AddHours(-4), [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [DateTime]$ToDate = (Get-Date), [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty, [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $NcRestCredential = [System.Management.Automation.PSCredential]::Empty, [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Int]$Limit = 16, [Parameter(Mandatory = $false, ParameterSetName = 'Role')] [Parameter(Mandatory = $false, ParameterSetName = 'Computer')] [bool]$ConvertETW = $true ) $dataCollectionNodes = [System.Collections.ArrayList]::new() # need an arrayList so we can remove objects from this list $filteredDataCollectionNodes = @() $stopWatch = [System.Diagnostics.Stopwatch]::StartNew() $dataCollectionObject = [PSCustomObject]@{ DurationInMinutes = $null TotalSize = $null OutputDirectory = $null Role = $null IncludeNetView = $IncludeNetView IncludeLogs = $IncludeLogs FromDate = $FromDate.ToString() FromDateUTC = $FromDate.ToUniversalTime().ToString() ToDate = $ToDate.ToString() ToDateUTC = $ToDate.ToUniversalTime().ToString() Result = $null } $collectLogSB = { param([string[]]$arg0,[String]$arg1,[DateTime]$arg2,[DateTime]$arg3,[Boolean]$arg4,[Boolean]$arg5,[string[]]$arg6) Get-SdnDiagnosticLogFile -LogDir $arg0 -OutputDirectory $arg1 -FromDate $arg2 -ToDate $arg3 -ConvertETW $arg4 -CleanUpFiles $arg5 -FolderNameFilter $arg6 } $collectConfigStateSB = { param([Parameter(Position = 0)][String]$Role, [Parameter(Position = 1)][String]$OutputDirectory) Get-SdnConfigState -Role $Role -OutputDirectory $OutputDirectory } try { if (-NOT ($PSBoundParameters.ContainsKey('NetworkController'))) { $config = Get-SdnModuleConfiguration -Role 'NetworkController' $confirmFeatures = Confirm-RequiredFeaturesInstalled -Name $config.windowsFeature if (-NOT ($confirmFeatures)) { "The current machine is not a NetworkController, run this on NetworkController or use -NetworkController parameter to specify one" | Trace-Output -Level:Warning return # don't throw exception, since this is a controlled scenario and we do not need stack exception tracing } } [System.String]$childPath = 'SdnDataCollection_{0}' -f (Get-FormattedDateTimeUTC) [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath $childPath [System.IO.FileInfo]$workingDirectory = (Get-WorkingDirectory) [System.IO.FileInfo]$tempDirectory = "$(Get-WorkingDirectory)\Temp" # setup the directory location where files will be saved to "Starting SDN Data Collection" | Trace-Output if ($IncludeLogs) { $minGB = 10 } else { $minGB = 5 } if (-NOT (Initialize-DataCollection -FilePath $OutputDirectory.FullName -MinimumGB $minGB)) { "Unable to initialize environment for data collection" | Trace-Output -Level:Error return } "Results will be saved to {0}" -f $OutputDirectory.FullName | Trace-Output # generate a mapping of the environment if ($NcUri) { $sdnFabricDetails = Get-SdnInfrastructureInfo -NetworkController $NetworkController -Credential $Credential -NcUri $NcUri.AbsoluteUri -NcRestCredential $NcRestCredential } else { $sdnFabricDetails = Get-SdnInfrastructureInfo -NetworkController $NetworkController -Credential $Credential -NcRestCredential $NcRestCredential } # determine if network controller is using default logging mechanism to local devices or network share [xml]$clusterManifest = Get-SdnServiceFabricClusterManifest -NetworkController $NetworkController -Credential $Credential $fileShareWinFabEtw = $clusterManifest.ClusterManifest.FabricSettings.Section | Where-Object {$_.Name -ieq 'FileShareWinFabEtw'} $connectionString = $fileShareWinFabEtw.Parameter | Where-Object {$_.Name -ieq 'StoreConnectionString'} if ($connectionString.value) { # typically the network share will be in a format of file://share/path $diagLogNetShare = ($connectionString.value).Split(':')[1].Replace('/', '\').Trim() $ncNodeFolders = @() } switch ($PSCmdlet.ParameterSetName) { 'Role' { foreach ($value in $Role) { foreach ($node in $sdnFabricDetails[$value.ToString()]) { $object = [PSCustomObject]@{ Role = $value Name = $node } "{0} with role {1} added for data collection" -f $object.Name, $object.Role | Trace-Output [void]$dataCollectionNodes.Add($object) } } } 'Computer' { foreach ($computer in $ComputerName) { $computerRole = Get-SdnRole -ComputerName $computer -EnvironmentInfo $sdnFabricDetails if ($computerRole) { $object = [PSCustomObject]@{ Role = $computerRole Name = $computer } "{0} with role {1} added for data collection" -f $object.Name, $object.Role | Trace-Output [void]$dataCollectionNodes.Add($object) } } } } if ($dataCollectionNodes.Count -eq 0) { throw New-Object System.NullReferenceException("No data nodes identified") } # once we have identified the nodes, we need to validate WinRM connectivity to the nodes # if we are running on PowerShell 7 or greater, we can leverage the -Parallel parameter # to speed up the process # if we are running on PowerShell 5.1, we will need to run the process in serial # if we have any nodes that fail the WinRM connectivity test, we will remove them from the data collection "Validating WinRM connectivity to {0}" -f ($dataCollectionNodes.Name -join ', ') | Trace-Output $Global:ProgressPreference = 'SilentlyContinue' $nodesToRemove = [System.Collections.ArrayList]::new() $tncScriptBlock = { $tncResult = Test-NetConnection -ComputerName $_.Name -Port 5985 -InformationLevel Quiet if (-NOT ($tncResult)) { [void]$nodesToRemove.Add($_) } } if ($PSVersionTable.PSVersion.Major -ge 7) { $dataCollectionNodes | Foreach-Object -ThrottleLimit 10 -Parallel $tncScriptBlock } else { $dataCollectionNodes | ForEach-Object $tncScriptBlock } if ($nodesToRemove.Count -gt 0) { $nodesToRemove | ForEach-Object { "Removing {0} from data collection due to WinRM connectivity issues" -f $_.Name | Trace-Output -Level:Warning [void]$dataCollectionNodes.Remove($_) } } $Global:ProgressPreference = 'Continue' $dataCollectionNodes = $dataCollectionNodes | Sort-Object -Property Name -Unique $groupedObjectsByRole = $dataCollectionNodes | Group-Object -Property Role # ensure SdnDiagnostics installed across the data nodes and versions are the same # depending on the state of the environment though, these may result in failure Install-SdnDiagnostics -ComputerName $NetworkController -ErrorAction Continue Install-SdnDiagnostics -ComputerName $dataCollectionNodes.Name -ErrorAction Continue # ensure that the NcUrl is populated before we start collecting data # in scenarios where certificate is not trusted or expired, we will not be able to collect data if (-NOT ([System.String]::IsNullOrEmpty($sdnFabricDetails.NcUrl))) { $slbStateInfo = Get-SdnSlbStateInformation -NcUri $sdnFabricDetails.NcUrl -Credential $NcRestCredential $slbStateInfo | ConvertTo-Json -Depth 100 | Out-File "$($OutputDirectory.FullName)\SlbState.Json" Invoke-SdnResourceDump -NcUri $sdnFabricDetails.NcUrl -OutputDirectory $OutputDirectory.FullName -Credential $NcRestCredential Get-SdnNetworkControllerState -NetworkController $NetworkController -OutputDirectory $OutputDirectory.FullName -Credential $Credential -NcRestCredential $NcRestCredential } Get-SdnNetworkControllerClusterInfo -NetworkController $NetworkController -OutputDirectory $OutputDirectory.FullName -Credential $Credential $debugInfraHealthResults = Get-SdnFabricInfrastructureResult if ($debugInfraHealthResults) { $debugInfraHealthResults | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnFabricInfrastructureResult_Summary' -FileType 'txt' -Format 'table' $debugInfraHealthResults | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnFabricInfrastructureResult' -FileType json -Depth 5 } # enumerate through each role and collect appropriate data foreach ($group in $groupedObjectsByRole | Sort-Object -Property Name) { if ($PSCmdlet.ParameterSetName -eq 'Role') { if ($group.Group.Name.Count -ge $Limit) { "Exceeded node limit for role {0}. Limiting nodes to the first {1} nodes" -f $group.Name, $Limit | Trace-Output -Level:Warning } $dataNodes = $group.Group.Name | Select-Object -First $Limit } else { $dataNodes = $group.Group.Name } "Performing cleanup of {0} directory across {1}" -f $tempDirectory.FullName, ($dataNodes -join ', ') | Trace-Output Clear-SdnWorkingDirectory -Path $tempDirectory.FullName -Recurse -ComputerName $dataNodes -Credential $Credential # add the data nodes to new variable, to ensure that we pick up the log files specifically from these nodes # to account for if filtering was applied $filteredDataCollectionNodes += $dataNodes "Collect configuration state details for {0} nodes: {1}" -f $group.Name, ($dataNodes -join ', ') | Trace-Output Invoke-PSRemoteCommand -ComputerName $dataNodes -Credential $Credential -ScriptBlock $collectConfigStateSB ` -ArgumentList @($group.Name, $tempDirectory.FullName) -AsJob -PassThru -Activity "Collect $($group.Name) Configuration State" # collect any adhoc data based on the role switch ($group.Name) { 'Server' { Get-SdnProviderAddress -ComputerName $dataNodes -Credential $Credential ` | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnProviderAddress' -FileType csv Get-SdnVfpVmSwitchPort -ComputerName $dataNodes -Credential $Credential ` | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnVfpVmSwitchPort' -FileType csv Get-SdnVMNetworkAdapter -ComputerName $dataNodes -Credential $Credential -AsJob -PassThru -Timeout 900 ` | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnVMNetworkAdapter' -FileType csv } } # check to see if any network traces were captured on the data nodes previously "Checking for any previous network traces and moving them into {0}" -f $tempDirectory.FullName | Trace-Output Invoke-PSRemoteCommand -ComputerName $dataNodes -Credential $Credential -ScriptBlock $collectLogSB ` -ArgumentList @("$($workingDirectory.FullName)\NetworkTraces", $tempDirectory.FullName, $FromDate, $ToDate, $ConvertETW, $true) -AsJob -PassThru -Activity 'Collect Network Traces' # collect the sdndiagnostics etl files if IncludeLogs was provided if ($IncludeLogs) { $commonConfig = Get-SdnModuleConfiguration -Role:Common # check to see if we are using local or network share for the logs if (!$diagLogNetShare) { [String]$diagLogDir = $commonConfig.DefaultLogDirectory "Collect diagnostics logs for {0} nodes: {1}" -f $group.Name, ($dataNodes -join ', ') | Trace-Output $outputDir = Join-Path -Path $tempDirectory.FullName -ChildPath 'SdnDiagnosticLogs' Invoke-PSRemoteCommand -ComputerName $dataNodes -Credential $Credential -ScriptBlock $collectLogSB ` -ArgumentList @($diagLogDir, $outputDir, $FromDate, $ToDate, $ConvertETW) -AsJob -PassThru -Activity 'Get Diagnostic Log Files' # collect the service fabric logs for the network controller if ($group.Name -ieq 'NetworkController') { $ncConfig = Get-SdnModuleConfiguration -Role:NetworkController [string[]]$sfLogDir = $ncConfig.Properties.CommonPaths.serviceFabricLogDirectory "Collect service fabric logs for {0} nodes: {1}" -f $group.Name, ($dataNodes -join ', ') | Trace-Output $outputDir = Join-Path -Path $tempDirectory.FullName -ChildPath 'ServiceFabricLogs' Invoke-PSRemoteCommand -ComputerName $dataNodes -Credential $Credential -ScriptBlock $collectLogSB ` -ArgumentList @($sfLogDir, $outputDir, $FromDate, $ToDate) -AsJob -PassThru -Activity 'Get Service Fabric Logs' } # if the role is a server, collect the audit logs if they are available if ($group.Name -ieq 'Server') { Get-SdnAuditLog -NcUri $sdnFabricDetails.NcUrl -NcRestCredential $NcRestCredential -OutputDirectory "$($OutputDirectory.FullName)\AuditLogs" ` -ComputerName $dataNodes -Credential $Credential } } # if the role is network controller and we are using a network share # need to update variable to include the network controller nodes # so we can add these supplmental folders to the collection if ($group.Name -ieq 'NetworkController') { $ncNodeFolders += $dataNodes } # collect the event logs specific to the role "Collect event logs for {0} nodes: {1}" -f $group.Name, ($dataNodes -join ', ') | Trace-Output # because we may have a 'Common' role that is being collected, we need to account for that # and ensure that we are collecting the appropriate event logs switch ( $group.Name ) { 'Common' { $roleArray = @(); $roleArray += $group.Name } default { $roleArray = @(); $roleArray += $group.Name; $roleArray += 'Common' } } Invoke-PSRemoteCommand -ComputerName $dataNodes -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$OutputDirectory, [Parameter(Position =1)][String[]]$Role, [Parameter(Position =2)][DateTime]$FromDate, [Parameter(Position = 3)][DateTime]$ToDate) Get-SdnEventLog -OutputDirectory $OutputDirectory -Role $Role -FromDate $FromDate -ToDate $ToDate } -ArgumentList @($tempDirectory.FullName, $roleArray, $FromDate, $ToDate) -AsJob -PassThru -Activity "Get $($group.Name) Event Logs" } } if ($diagLogNetShare -and $IncludeLogs) { $isNetShareMapped = New-SdnDiagNetworkMappedShare -NetworkSharePath $diagLogNetShare -Credential $Credential if ($isNetShareMapped) { $outputDir = Join-Path -Path $OutputDirectory.FullName -ChildPath 'NetShare_SdnDiagnosticLogs' # create an array of names that we will use to filter the logs # this ensures that we will only pick up the logs from the nodes that we are collecting from $filterArray = @() $dataCollectionNodes.Name | ForEach-Object { $filterArray += (Get-ComputerNameFQDNandNetBIOS -ComputerName $_).ComputerNameNetBIOS } $filterArray = $filterArray | Sort-Object -Unique # create an array of folders to collect the logs from leveraging the common configuration $logDir = @() $commonConfig.DefaultLogFolders | ForEach-Object { $logDir += Join-Path -Path $diagLogNetShare -ChildPath $_ } $ncNodeFolders | ForEach-Object { $ncNetBiosName = (Get-ComputerNameFQDNandNetBIOS -ComputerName $_).ComputerNameNetBIOS $logDir += Join-Path -Path $diagLogNetShare -ChildPath $ncNetBiosName } $logDir = $logDir | Sort-Object -Unique # create parameters for the Get-SdnDiagnosticLogFile function $netDiagLogShareParams = @{ LogDir = $logDir OutputDirectory = $outputDir FromDate = $FromDate ToDate = $ToDate FolderNameFilter = $filterArray } Get-SdnDiagnosticLogFile @netDiagLogShareParams } } if ($IncludeNetView) { "Collect Get-NetView logs for {0}" -f ($filteredDataCollectionNodes -join ', ') | Trace-Output $null = Invoke-PSRemoteCommand -ComputerName $filteredDataCollectionNodes -Credential $Credential -ScriptBlock { param([Parameter(Position = 0)][String]$OutputDirectory) Invoke-SdnGetNetView -OutputDirectory $OutputDirectory ` -SkipAdminCheck ` -SkipNetshTrace ` -SkipVM ` -SkipCounters } -ArgumentList @($tempDirectory.FullName) -AsJob -PassThru -Activity 'Invoke Get-NetView' } foreach ($node in $filteredDataCollectionNodes) { [System.IO.FileInfo]$formattedDirectoryName = Join-Path -Path $OutputDirectory.FullName -ChildPath $node.ToLower() Copy-FileFromRemoteComputer -Path $tempDirectory.FullName -Destination $formattedDirectoryName.FullName -ComputerName $node -Credential $Credential -Recurse -Force Copy-FileFromRemoteComputer -Path (Get-TraceOutputFile) -Destination $formattedDirectoryName.FullName -ComputerName $node -Credential $Credential -Force } $dataCollectionObject.TotalSize = (Get-FolderSize -Path $OutputDirectory.FullName -Total) $dataCollectionObject.OutputDirectory = $OutputDirectory.FullName $dataCollectionObject.Role = $groupedObjectsByRole.Name $dataCollectionObject.Result = 'Success' } catch { $_ | Trace-Exception $_ | Write-Error $dataCollectionObject.Result = 'Failed' } finally { $stopWatch.Stop() $dataCollectionObject.DurationInMinutes = $stopWatch.Elapsed.TotalMinutes try { "Performing post operations and cleanup of {0} across the SDN fabric" -f $tempDirectory.FullName | Trace-Output # check for any failed PS remoting jobs and copy them to data collection if (Test-Path -Path "$(Get-WorkingDirectory)\PSRemoteJob_Failures") { Copy-Item -Path "$(Get-WorkingDirectory)\PSRemoteJob_Failures" -Destination $formattedDirectoryName.FullName -Recurse } if ($filteredDataCollectionNodes) { Clear-SdnWorkingDirectory -Path $tempDirectory.FullName -Recurse -ComputerName $filteredDataCollectionNodes -Credential $Credential } # remove any completed or failed jobs Remove-SdnDiagnosticJob -State @('Completed', 'Failed') } catch { $_ | Trace-Exception Write-Error -Message "An error occurred during cleanup of the SDN fabric." -Exception $_.Exception $dataCollectionObject.Result = 'Failed' } } $dataCollectionObject | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'SdnDataCollection_Summary' -FileType json -Depth 4 -ErrorAction Continue Copy-Item -Path (Get-TraceOutputFile) -Destination $OutputDirectory.FullName -Force -ErrorAction Continue # we will return the object to the caller regardless if the data collection was successful or not $msg = "Sdn Data Collection completed with status of {0}" -f $dataCollectionObject.Result switch ($dataCollectionObject.Result) { 'Success' { $msg | Trace-Output } 'Failed' { $msg | Trace-Output -Level:Error } } return $dataCollectionObject } function Start-SdnEtwTraceCapture { <# .SYNOPSIS Start ETW Trace capture based on Role .PARAMETER Role The SDN Roles .PARAMETER Providers Allowed values are Default,Optional And All to control what are the providers needed #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateSet('Common', 'Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')] [String]$Role, [Parameter(Mandatory = $false)] [System.String]$OutputDirectory = (Get-WorkingDirectory), [Parameter(Mandatory = $false)] [ValidateSet("Default", "Optional", "All")] [string]$Providers = "Default" ) # this is the default trace size that we will limit each etw trace session to $maxTraceSize = 1024 try { $traceProvidersArray = Get-TraceProviders -Role $Role -Providers $Providers # we want to calculate the max size on number of factors to ensure sufficient disk space is available $diskSpaceRequired = $maxTraceSize*($traceProvidersArray.Count)*1.5 if (-NOT (Initialize-DataCollection -Role $Role -FilePath $OutputDirectory -MinimumMB $diskSpaceRequired)) { "Unable to initialize environment for data collection" | Trace-Output -Level:Error return } foreach ($traceProviders in $traceProvidersArray) { "Starting trace session {0}" -f $traceProviders.name | Trace-Output -Level:Verbose Start-EtwTraceSession -TraceName $traceProviders.name -TraceProviders $traceProviders.properties.providers -TraceFile "$OutputDirectory\$($traceProviders.name).etl" -MaxTraceSize $maxTraceSize } } catch { $_ | Trace-Exception $_ | Write-Error } } function Start-SdnNetshTrace { <# .SYNOPSIS Enables netsh tracing based on pre-configured trace providers. .PARAMETER ComputerName Type the NetBIOS name, an IP address, or a fully qualified domain name of one or more remote computers. .PARAMETER Credential Specifies a user account that has permission to perform this action. The default is the current user. Type a user name, such as User01 or Domain01\User01, or enter a PSCredential object generated by the Get-Credential cmdlet. If you type a user name, you're prompted to enter the password. .PARAMETER Role The specific SDN role of the local or remote computer(s) that tracing is being enabled for. .PARAMETER OutputDirectory Specifies a specific path and folder in which to save the files. .PARAMETER MaxTraceSize Optional. Specifies the maximum size in MB for saved trace files. If unspecified, the default is 1024. .PARAMETER Capture Optional. Specifies whether packet capture is enabled in addition to trace events. If unspecified, the default is No. .PARAMETER Overwrite Optional. Specifies whether this instance of the trace conversion command overwrites files that were rendered from previous trace conversions. If unspecified, the default is Yes. .PARAMETER Correlation Optional. Specifies whether related events will be correlated and grouped together. If unspecified, the default is Disabled. .PARAMETER Report Optional. Specifies whether a complementing report will be generated in addition to the trace file report. If unspecified, the default is disabled. .EXAMPLE PS> Start-SdnNetshTrace -OutputDirectory "C:\Temp\CSS_SDN" -Capture Yes -Role Server .EXAMPLE PS> Start-SdnNetshTrace -ComputerName (Get-SdnInfrastructureInfo -NetworkController 'PREFIX-NC03').Server -Role Server -Credential (Get-Credential) #> [CmdletBinding(DefaultParameterSetName = 'Local')] param ( [Parameter(Mandatory = $true, ParameterSetName = 'Local')] [Parameter(Mandatory = $true, ParameterSetName = 'Remote')] [ValidateSet('Common', 'Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')] [String]$Role, [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [System.String]$OutputDirectory = "$(Get-WorkingDirectory)\NetworkTraces", [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [int]$MaxTraceSize = 1536, [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [ValidateSet('Yes', 'No')] [System.String]$Capture = 'Yes', [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [ValidateSet('Yes', 'No')] [System.String]$Overwrite = 'Yes', [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [ValidateSet('Yes', 'No', 'Disabled')] [System.String]$Correlation = 'Disabled', [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [ValidateSet('Yes', 'No', 'Disabled')] [System.String]$Report = 'Disabled', [Parameter(Mandatory = $false, ParameterSetName = 'Local')] [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [ValidateSet("Default", "Optional", "All")] [string]$Providers = "All", [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [System.String[]]$ComputerName, [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty ) $params = @{ OutputDirectory = $OutputDirectory MaxTraceSize = $MaxTraceSize Capture = $Capture Overwrite = $Overwrite Report = $Report Correlation = $Correlation } $scriptBlock = { param( [Parameter(Position = 0)][String]$Role, [Parameter(Position = 1)][String]$OutputDirectory, [Parameter(Position = 2)][int]$MaxTraceSize, [Parameter(Position = 3)][String]$Capture, [Parameter(Position = 4)][String]$Overwrite, [Parameter(Position = 5)][String]$Report, [Parameter(Position = 6)][String]$Correlation, [Parameter(Position = 7)][String]$Providers ) Start-SdnNetshTrace -Role $Role -OutputDirectory $OutputDirectory ` -MaxTraceSize $MaxTraceSize -Capture $Capture -Overwrite $Overwrite -Report $Report -Correlation $Correlation -Providers $Providers } try { if ($PSCmdlet.ParameterSetName -eq 'Remote') { Invoke-PSRemoteCommand -ComputerName $ComputerName -Credential $Credential -ScriptBlock $scriptBlock ` -ArgumentList @($Role, $params.OutputDirectory, $params.MaxTraceSize, $params.Capture, $params.Overwrite, $params.Report, $params.Correlation, $Providers) } else { $traceProviderString = Get-TraceProviders -Role $Role -Providers $Providers -AsString if ($traceProviderString) { $params.Add('TraceProviderString', $traceProviderString) "Trace providers configured: {0}" -f $traceProviderString | Trace-Output -Level:Verbose } elseif ($null -eq $traceProviderString) { "No default trace providers found for role {0}." -f $Role | Trace-Output if ($params.Capture -eq 'No') { $params.Capture = 'Yes' "Setting capture to {1}" -f $Role, $params.Capture | Trace-Output } } if (-NOT ( Initialize-DataCollection -Role $Role -FilePath $OutputDirectory -MinimumMB ($MaxTraceSize*1.5) )) { "Unable to initialize environment for data collection" | Trace-Output -Level:Error return } Start-NetshTrace @params } } catch { $_ | Trace-Exception $_ | Write-Error } } function Stop-SdnEtwTraceCapture { <# .SYNOPSIS Start ETW Trace capture based on Role .PARAMETER Role The SDN Roles .PARAMETER Providers Allowed values are Default,Optional And All to control what are the providers needed #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateSet('Common', 'Gateway', 'NetworkController', 'Server', 'LoadBalancerMux')] [String]$Role, [Parameter(Mandatory = $false)] [ValidateSet("Default", "Optional", "All")] [string]$Providers = "Default" ) try { $traceProvidersArray = Get-TraceProviders -Role $Role -Providers $Providers foreach ($traceProviders in $traceProvidersArray) { Stop-EtwTraceSession -TraceName $traceProviders.name } } catch { $_ | Trace-Exception $_ | Write-Error } } function Stop-SdnNetshTrace { <# .SYNOPSIS Disables netsh tracing. .PARAMETER ComputerName Type the NetBIOS name, an IP address, or a fully qualified domain name of one or more remote computers. .PARAMETER Credential Specifies a user account that has permission to perform this action. The default is the current user. Type a user name, such as User01 or Domain01\User01, or enter a PSCredential object generated by the Get-Credential cmdlet. If you type a user name, you're prompted to enter the password. #> [CmdletBinding(DefaultParameterSetName = 'Local')] param ( [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [System.String[]]$ComputerName, [Parameter(Mandatory = $false, ParameterSetName = 'Remote')] [System.Management.Automation.PSCredential] [System.Management.Automation.Credential()] $Credential = [System.Management.Automation.PSCredential]::Empty ) try { if ($PSCmdlet.ParameterSetName -eq 'Remote') { Invoke-PSRemoteCommand -ComputerName $ComputerName -Credential $Credential -ScriptBlock { Stop-SdnNetshTrace } } else { Stop-NetshTrace } } catch { $_ | Trace-Exception $_ | Write-Error } } # SIG # Begin signature block # MIInvwYJKoZIhvcNAQcCoIInsDCCJ6wCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAgF8snaGPzzIpU # RBr5IMk2YzCFD+5SP+BiQjLuON3n+KCCDXYwggX0MIID3KADAgECAhMzAAADrzBA # DkyjTQVBAAAAAAOvMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwOTAwWhcNMjQxMTE0MTkwOTAwWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDOS8s1ra6f0YGtg0OhEaQa/t3Q+q1MEHhWJhqQVuO5amYXQpy8MDPNoJYk+FWA # hePP5LxwcSge5aen+f5Q6WNPd6EDxGzotvVpNi5ve0H97S3F7C/axDfKxyNh21MG # 0W8Sb0vxi/vorcLHOL9i+t2D6yvvDzLlEefUCbQV/zGCBjXGlYJcUj6RAzXyeNAN # xSpKXAGd7Fh+ocGHPPphcD9LQTOJgG7Y7aYztHqBLJiQQ4eAgZNU4ac6+8LnEGAL # go1ydC5BJEuJQjYKbNTy959HrKSu7LO3Ws0w8jw6pYdC1IMpdTkk2puTgY2PDNzB # tLM4evG7FYer3WX+8t1UMYNTAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQURxxxNPIEPGSO8kqz+bgCAQWGXsEw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMTgyNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAISxFt/zR2frTFPB45Yd # mhZpB2nNJoOoi+qlgcTlnO4QwlYN1w/vYwbDy/oFJolD5r6FMJd0RGcgEM8q9TgQ # 2OC7gQEmhweVJ7yuKJlQBH7P7Pg5RiqgV3cSonJ+OM4kFHbP3gPLiyzssSQdRuPY # 1mIWoGg9i7Y4ZC8ST7WhpSyc0pns2XsUe1XsIjaUcGu7zd7gg97eCUiLRdVklPmp # XobH9CEAWakRUGNICYN2AgjhRTC4j3KJfqMkU04R6Toyh4/Toswm1uoDcGr5laYn # TfcX3u5WnJqJLhuPe8Uj9kGAOcyo0O1mNwDa+LhFEzB6CB32+wfJMumfr6degvLT # e8x55urQLeTjimBQgS49BSUkhFN7ois3cZyNpnrMca5AZaC7pLI72vuqSsSlLalG # OcZmPHZGYJqZ0BacN274OZ80Q8B11iNokns9Od348bMb5Z4fihxaBWebl8kWEi2O # PvQImOAeq3nt7UWJBzJYLAGEpfasaA3ZQgIcEXdD+uwo6ymMzDY6UamFOfYqYWXk # ntxDGu7ngD2ugKUuccYKJJRiiz+LAUcj90BVcSHRLQop9N8zoALr/1sJuwPrVAtx # HNEgSW+AKBqIxYWM4Ev32l6agSUAezLMbq5f3d8x9qzT031jMDT+sUAoCw0M5wVt # CUQcqINPuYjbS1WgJyZIiEkBMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGZ8wghmbAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAOvMEAOTKNNBUEAAAAAA68wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIPxNyGTQbFgNmtWOVQOCAaWp # XH2aCpbKnFb5hHrzK9FWMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAvtnTVLkYawUOzOAQIB/O16Ok4IrQsx8dWRuoTicenV7lCAv41iT0+TnJ # izagELa66YqFVTicOhfFsyxm8ZSbQl8yQKqHrdVP7bQjZ1fY5E9cPk5jSB6+ufZL # VJkKoF55PorJ0OXFljm3IeKAxC8/NZOoSMAz4bxz1QHrHLpUpuWOTQgFFSG9oaNx # vMIdMQd8tsIDdH4JeNUGiPbHI67cMyTQavLBC+jGR0v+4VoVT6EVy6oyzhbT+XFY # n2pTlAD1u5o0YHd/kVHBcyaeZ3p8KV4xBtdDgBw9niFGN+VBukvDHphvxqLtX2Ka # brzEqq0MxtdaEqj/XU5ihJpx3R2kP6GCFykwghclBgorBgEEAYI3AwMBMYIXFTCC # FxEGCSqGSIb3DQEHAqCCFwIwghb+AgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq # hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCAWfKFqCrhxACCdWSSxBu16VjAvy+ZlsyDaVcPcRoZadwIGZh/SW+di # GBMyMDI0MDQxOTIyNTIzMS4zNjRaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO # OjNCRDQtNEI4MC02OUMzMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT # ZXJ2aWNloIIReDCCBycwggUPoAMCAQICEzMAAAHlj2rA8z20C6MAAQAAAeUwDQYJ # KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjMx # MDEyMTkwNzM1WhcNMjUwMTEwMTkwNzM1WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl # cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjozQkQ0LTRC # ODAtNjlDMzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKl74Drau2O6LLrJO3HyTvO9 # aXai//eNyP5MLWZrmUGNOJMPwMI08V9zBfRPNcucreIYSyJHjkMIUGmuh0rPV5/2 # +UCLGrN1P77n9fq/mdzXMN1FzqaPHdKElKneJQ8R6cP4dru2Gymmt1rrGcNe800C # cD6d/Ndoommkd196VqOtjZFA1XWu+GsFBeWHiez/PllqcM/eWntkQMs0lK0zmCfH # +Bu7i1h+FDRR8F7WzUr/7M3jhVdPpAfq2zYCA8ZVLNgEizY+vFmgx+zDuuU/GChD # K7klDcCw+/gVoEuSOl5clQsydWQjJJX7Z2yV+1KC6G1JVqpP3dpKPAP/4udNqpR5 # HIeb8Ta1JfjRUzSv3qSje5y9RYT/AjWNYQ7gsezuDWM/8cZ11kco1JvUyOQ8x/JD # kMFqSRwj1v+mc6LKKlj//dWCG/Hw9ppdlWJX6psDesQuQR7FV7eCqV/lfajoLpPN # x/9zF1dv8yXBdzmWJPeCie2XaQnrAKDqlG3zXux9tNQmz2L96TdxnIO2OGmYxBAA # ZAWoKbmtYI+Ciz4CYyO0Fm5Z3T40a5d7KJuftF6CToccc/Up/jpFfQitLfjd71cS # +cLCeoQ+q0n0IALvV+acbENouSOrjv/QtY4FIjHlI5zdJzJnGskVJ5ozhji0YRsc # v1WwJFAuyyCMQvLdmPddAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU3/+fh7tNczEi # fEXlCQgFOXgMh6owHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD # VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j # cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG # CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw # MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD # CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBADP6whOFjD1ad8Gk # EJ9oLBuvfjndMyGQ9R4HgBKSlPt3pa0XVLcimrJlDnKGgFBiWwI6XOgw82hdolDi # MDBLLWRMTJHWVeUY1gU4XB8OOIxBc9/Q83zb1c0RWEupgC48I+b+2x2VNgGJUsQI # yPR2PiXQhT5PyerMgag9OSodQjFwpNdGirna2rpV23EUwFeO5+3oSX4JeCNZvgyU # OzKpyMvqVaubo+Glf/psfW5tIcMjZVt0elswfq0qJNQgoYipbaTvv7xmixUJGTbi # xYifTwAivPcKNdeisZmtts7OHbAM795ZvKLSEqXiRUjDYZyeHyAysMEALbIhdXgH # Eh60KoZyzlBXz3VxEirE7nhucNwM2tViOlwI7EkeU5hudctnXCG55JuMw/wb7c71 # RKimZA/KXlWpmBvkJkB0BZES8OCGDd+zY/T9BnTp8si36Tql84VfpYe9iHmy7Pqq # xqMF2Cn4q2a0mEMnpBruDGE/gR9c8SVJ2ntkARy5SfluuJ/MB61yRvT1mUx3lypp # O22ePjBjnwoEvVxbDjT1jhdMNdevOuDeJGzRLK9HNmTDC+TdZQlj+VMgIm8ZeEIR # NF0oaviF+QZcUZLWzWbYq6yDok8EZKFiRR5otBoGLvaYFpxBZUE8mnLKuDlYobjr # xh7lnwrxV/fMy0F9fSo2JxFmtLgtMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ # mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh # dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 # WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB # BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK # NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg # fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp # rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d # vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 # 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR # Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu # qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO # ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb # oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 # bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t # AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW # BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb # UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku # aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA # QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 # VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu # bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw # LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt # MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q # XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 # U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt # I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis # 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp # kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 # sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e # W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ # sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 # Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 # dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ # tB1VM1izoXBm8qGCAtQwggI9AgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh # bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjoz # QkQ0LTRCODAtNjlDMzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUA942iGuYFrsE4wzWDd85EpM6RiwqggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF # AOnM83kwIhgPMjAyNDA0MTkyMTQzMjFaGA8yMDI0MDQyMDIxNDMyMVowdDA6Bgor # BgEEAYRZCgQBMSwwKjAKAgUA6czzeQIBADAHAgEAAgIZqDAHAgEAAgIRQzAKAgUA # 6c5E+QIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIBAAID # B6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAAWIDGPzKzDY3qBULCsK # D7fqZANLyEBHcYXCsYWFJA3m2Rp0oD3xeExxETLdP3KJjRK9dxr6lngdWIL7ypX2 # qX560szxSSNEeRXCy2inn6rjJHHvdSKEKZOrs4Siu7okZ3n3BSbxv7he1xzAhwfy # 2YguIdkn0R4pbIzGtmVURM1vMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUt # U3RhbXAgUENBIDIwMTACEzMAAAHlj2rA8z20C6MAAQAAAeUwDQYJYIZIAWUDBAIB # BQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQx # IgQgxt92NtH6f5f3EcW+Ca/muUrWnISgCoHNFjPfFlHnFMswgfoGCyqGSIb3DQEJ # EAIvMYHqMIHnMIHkMIG9BCAVqdP//qjxGFhe2YboEXeb8I/pAof01CwhbxUH9U69 # 7TCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB5Y9q # wPM9tAujAAEAAAHlMCIEIGxHmOc3oYze9P1iaHpm3cRjpB0r1YFkqvLxjuvJs6wa # MA0GCSqGSIb3DQEBCwUABIICACfXmD7GNxVvDzuf+rAK6cEkb2Ybd4gFGn6GO4R7 # 7eC0+fPEg3fDCBlTbBYMwyQ0dMgNuxthl8ooNGXZ6AGBMusTSEMbwR8W++tfIAxO # 1bR0OS3AIRtKsKpgs+t2ZAycfi5Yv7fwKDisJgIH3uRbJ/ZXAFB+ubnKgJ5xBXdQ # JgzK/zMYZbe+zn3dC0tUEcaKY9D29iQM29U5DBXeK01w3YHs8iKB2oNdkPUyd6tO # NV/FNb1MZ9cO+tGw4h1dF5+OBlvBXPLl7AGKSlVggC0Doz+I+zsQO9hXt+Dbx1LJ # kMcg7V/a/A37tq2ODlPG36HImSR4ZtB1DBTUgQRAyv/fXM7JTqgLbI+//pCyOSG7 # Wrkm3kB5LqTQWH39StzRQ1GW5jMi3JfSTOsxq6cZwXR47eIjfscljxTT+JiTu73c # RwRcFHjEnc/a9aoIQPPCsqYdi/O6FU6AGHDBAKA2iG83XwSNUSHYLQbts11yS8Wv # qqAhLu3Fy8t9yH2LjmYXw3d33Y/b3XVYP9D+5RDsYmGbdScQfPeFY8Z6QKWPTKmP # dINEDJ2l6pjfbunjhpVJmz9TiQSA7HhN9scNdMWEZ12a4mi5ym8aS0SkHVaScbtt # 716ozPQa1ZREimn45g+dk58Qcb3Dnng452OGgJimKlAJ1u1UAW7sy8yB5Xp3X3G6 # 5kLm # SIG # End signature block |