AzSDK

2.8.1

Framework/Configurations/ContinuousAssurance/RunbookScanAgent.ps1

function ConvertStringToBoolean($strToConvert) {
    switch ($strToConvert) {
        "true" {return $true}
        "false" {return $false}
    }
    return $false #adding this to prevent error all path doesn't return value"
}

function SanitizeStorage(){
    $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG  -Name $StorageAccountName
    $currentContext = New-AzureStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $keys[0].Value -Protocol Https
    $Container = Get-AzureStorageContainer -Name "azsdk-controls-state" -Context $currentContext -ErrorAction SilentlyContinue
    if($null -eq $Container)
    {        
        New-AzureStorageContainer -Name "azsdk-controls-state" -Permission Off -Context $currentContext
    }
}

function RunAzSDKScan() {
    Set-AzSDKOMSSettings -OMSWorkspaceID $OMSWorkspaceId -OMSSharedKey $OMSWorkspaceSharedKey -Source "CC"

    #set values in azsdksettings.json
    $EnableAADAuthForOnlinePolicyStore = ConvertStringToBoolean($EnableAADAuthForOnlinePolicyStore)
    if ($EnableAADAuthForOnlinePolicyStore) {
        Set-AzSDKPolicySettings -OnlinePolicyStoreUrl $OnlinePolicyStoreUrl -EnableAADAuthForOnlinePolicyStore
    }
    else {
        Set-AzSDKPolicySettings -OnlinePolicyStoreUrl $OnlinePolicyStoreUrl
    }    
    Set-AzSDKPrivacyNoticeResponse -AcceptPrivacyNotice "yes" # skipping the EULA and privacy prompt

    PublishEvent -EventName "CA Scan Started" -Properties @{
        "ResourceGroupNames"       = $ResourceGroupNames; `
            "OnlinePolicyStoreUrl" = $OnlinePolicyStoreUrl; `
            "OMSWorkspaceId"       = $OMSWorkspaceId;
    }

    CheckForSubscriptionsSnapshotData
    #Check if the preview code is enabled

        #get the current storagecontext
        $existingStorage = Find-AzureRmResource -ResourceGroupNameEquals $StorageAccountRG -ResourceNameContains "azsdk" -ResourceType "Microsoft.Storage/storageAccounts"
        if(($existingStorage|Measure-Object).Count -gt 1)
        {
            $existingStorage = $existingStorage[0]
            Write-Output ("Multiple storage accounts found in resource group. Using Storage Account: $($existingStorage.ResourceName) for storing logs")
        }

        #Create output files in storage
        $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG -Name $existingStorage.ResourceName
        $centralStorageContext = New-AzureStorageContext -StorageAccountName $existingStorage.ResourceName -StorageAccountKey $keys[0].Value -Protocol Https
    if($Global:IsPreviewMode)
    {
        $Global:activeScanObjects | ForEach-Object {
            $activeScanObject = $_;
            if($activeScanObject.Status -ne "COM")
            {                
                $subId = $activeScanObject.SubscriptionId;
                Select-AzureRmSubscription -SubscriptionId $subId | Out-Null
                "Started scan for the subscription: $subId"
                PersistSubscriptionSnapshot -SubscriptionID $subId -Status "INP" -StorageContext $centralStorageContext 
                RunAzSDKScanForASub -SubscriptionID $subId -LoggingOption $activeScanObject.LoggingOption -StorageContext $centralStorageContext 
                PersistSubscriptionSnapshot -SubscriptionID $subId -Status "COM" -StorageContext $centralStorageContext 
                "Completed scan for the subscription: $subId"
            }
        }
    }
    else
    {
        RunAzSDKScanForASub -SubscriptionID $RunAsConnection.SubscriptionID -LoggingOption "CentralSub" -StorageContext $centralStorageContext 
    }
}

function RunAzSDKScanForASub
{
    param
    (
        $SubscriptionID,
        $LoggingOption,
        $StorageContext
    )
    $svtResultPath = [string]::Empty
    $subscriptionResultPath = [string]::Empty
    $parentFolderPath = [string]::Empty

    #------------------------------------Subscription scan----------------------------------------------------------------
    "Running command 'Get-AzSDKSubscriptionSecurityStatus'"
    $subScanTimer = [System.Diagnostics.Stopwatch]::StartNew();
    PublishEvent -EventName "CA Scan Subscription Started"
    $subscriptionResultPath = Get-AzSDKSubscriptionSecurityStatus -SubscriptionId $SubscriptionID -ExcludeTags "OwnerAccess"

    #---------------------------Check subscription scan status--------------------------------------------------------------
    if ([string]::IsNullOrWhiteSpace($subscriptionResultPath)) {
        PublishEvent -EventName "CA Scan Subscription Error" -Metrics @{"TimeTakenInMs" = $subScanTimer.ElapsedMilliseconds; "SuccessCount" = 0}
        "Subscription scan failed."
    }
    else {
        PublishEvent -EventName "CA Scan Subscription Completed" -Metrics @{"TimeTakenInMs" = $subScanTimer.ElapsedMilliseconds; "SuccessCount" = 1}
        "Subscription scan succeeded."
        $parentFolderPath = (Get-Item $subscriptionResultPath).parent.FullName
    }

    #-------------------------------------Resources Scan------------------------------------------------------------------
    "Running command 'Get-AzSDKAzureServicesSecurityStatus'"
    $serviceScanTimer = [System.Diagnostics.Stopwatch]::StartNew();
    PublishEvent -EventName "CA Scan Services Started"
    #if ($ResourceGroupNames.Trim() -eq "*") {
    # #run command on all resources
    # $svtResultPath = Get-AzSDKAzureServicesSecurityStatus -SubscriptionId $SubscriptionID -ExcludeTags "OwnerAccess"
    #}
    #else {
        #$svtResultPath = Get-AzSDKAzureServicesSecurityStatus -SubscriptionId $SubscriptionID -ResourceGroupNames $ResourceGroupNames -ExcludeTags "OwnerAccess"
        $svtResultPath = Get-AzSDKAzureServicesSecurityStatus -SubscriptionId $SubscriptionID -ResourceGroupNames "*" -ExcludeTags "OwnerAccess" -UseBaselineControls -UsePartialCommits
    #}
    #---------------------------Check resources scan status--------------------------------------------------------------
    if ([string]::IsNullOrWhiteSpace($svtResultPath)) {
        "Azure resources scan failed."
        PublishEvent -EventName "CA Scan Services Error" -Metrics @{"TimeTakenInMs" = $serviceScanTimer.ElapsedMilliseconds; "SuccessCount" = 0}
    }
    else {
        "Azure resources scan succeeded."
        $parentFolderPath = (Get-Item $svtResultPath).parent.FullName
        PublishEvent -EventName "CA Scan Services Completed" -Metrics @{"TimeTakenInMs" = $serviceScanTimer.ElapsedMilliseconds; "SuccessCount" = 1}
    }
    #----------------------------------------Export reports to storage---------------------------------------------------
    if (![string]::IsNullOrWhiteSpace($subscriptionResultPath) -or ![string]::IsNullOrWhiteSpace($svtResultPath)) {
        #Check if storage account exists
        if($Global:IsPreviewMode)
        {
            if($LoggingOption -ne "CentralSub")
            {
                $existingStorage = Find-AzureRmResource -ResourceGroupNameEquals $StorageAccountRG -ResourceNameContains "azsdk" -ResourceType "Microsoft.Storage/storageAccounts"
                if(($existingStorage|Measure-Object).Count -gt 1)
                {
                    $existingStorage = $existingStorage[0]
                    Write-Output ("Multiple storage accounts found in resource group. Using Storage Account: $($existingStorage.ResourceName) for storing logs")
                }

                #Create output files in storage
                $containerName = "azsdkexecutionlogs"
                $archiveFilePath = "$parentFolderPath\AutomationLogs_" + $(Get-Date -format "yyyyMMdd_HHmmss") + ".zip"
                $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG -Name $existingStorage.ResourceName
                $localStorageContext = New-AzureStorageContext -StorageAccountName $existingStorage.ResourceName -StorageAccountKey $keys[0].Value -Protocol Https
                try {
                    Get-AzureStorageContainer -Name $containerName -Context $localStorageContext -ErrorAction Stop | Out-Null
                }
                catch {
                    New-AzureStorageContainer -Name $containerName -Context $localStorageContext | Out-Null
                }

                PersistToStorageAccount -StorageContext $localStorageContext -SubscriptionResultPath $subscriptionResultPath -SvtResultPath $svtResultPath
                PurgeOlderScanReports -StorageContext $localStorageContext
            }
            else
            {
                PersistToStorageAccount -StorageContext $StorageContext -SubscriptionResultPath $subscriptionResultPath -SvtResultPath $svtResultPath
                PurgeOlderScanReports -StorageContext $StorageContext
            }
        }
        else
        {
            PersistToStorageAccount -StorageContext $StorageContext -SubscriptionResultPath $subscriptionResultPath -SvtResultPath $svtResultPath
            PurgeOlderScanReports -StorageContext $StorageContext
        }

        #clean-up of logs in automation sandbox
        if (![string]::IsNullOrWhiteSpace($svtResultPath)) {
            Remove-Item -Path $svtResultPath -Recurse -ErrorAction Ignore
        }
        if (![string]::IsNullOrWhiteSpace($subscriptionResultPath)) {
            Remove-Item -Path $subscriptionResultPath -Recurse -ErrorAction Ignore
        }
        if (![string]::IsNullOrWhiteSpace($archiveFilePath)) {
            Remove-Item -Path $archiveFilePath -Recurse -ErrorAction Ignore
        }
    }
}

function PersistToStorageAccount
{
    param(
        $StorageContext,
        $SubscriptionResultPath,
        $SvtResultPath
    )
    if (![string]::IsNullOrWhiteSpace($SubscriptionResultPath) -or ![string]::IsNullOrWhiteSpace($SvtResultPath)) {
        
        #Check if the passed storagecontext is null. This would be in the case of default scenario i.e non preview mode
        $containerName = "azsdkexecutionlogs"
        $archiveFilePath = "$parentFolderPath\AutomationLogs_" + $(Get-Date -format "yyyyMMdd_HHmmss") + ".zip"
            try {

            
                Get-AzureStorageContainer -Name $containerName -Context $StorageContext -ErrorAction Stop | Out-Null
            }
            catch {
                New-AzureStorageContainer -Name $containerName -Context $StorageContext | Out-Null
        }

        #Persist the files to the storage account using the passed storage context
        try {
            if (![string]::IsNullOrWhiteSpace($SvtResultPath)) {
                Compress-Archive -Path $SvtResultPath -CompressionLevel Optimal -DestinationPath $archiveFilePath -Update
            }
            if (![string]::IsNullOrWhiteSpace($SubscriptionResultPath)) {
                Compress-Archive -Path $SubscriptionResultPath -CompressionLevel Optimal -DestinationPath $archiveFilePath -Update
            }
            Set-AzureStorageBlobContent -File $archiveFilePath -Container $containerName -Context $StorageContext -ErrorAction Stop | Out-Null
            "Exported reports to storage $StorageAccountName"
            PublishEvent -EventName "CA Scan Reports Persisted" -Properties @{"StorageAccountName" = $StorageAccountName; "ArchiveFilePath" = $archiveFilePath } -Metrics @{"SuccessCount" = 1}
        }
        catch {
            "Could not export reports to storage $StorageAccountName"
            PublishEvent -EventName "CA Scan Reports Persist Error" -Properties @{"ErrorRecord" = ($_ | Out-String); "StorageAccountName" = $StorageAccountName; "ArchiveFilePath" = $archiveFilePath } -Metrics @{"SuccessCount" = 0}
            throw $_.Exception
        }        
    }
}

function PurgeOlderScanReports
{
    param(
        $StorageContext
    )
    $containerName = "azsdkexecutionlogs"
    $NotBefore = [DateTime]::Now.AddDays(-30);
    $OldLogCount = (Get-AzureStorageBlob -Container $containerName -Context $StorageContext | Where-Object { $_.LastModified -lt $NotBefore} | Measure-Object).Count

    Get-AzureStorageBlob -Container $containerName -Context $StorageContext | Where-Object { $_.LastModified -lt $NotBefore} | Remove-AzureStorageBlob -Force -ErrorAction SilentlyContinue

    if($OldLogCount -gt 0)
    {
        #deleted successfully all the old reports
        Write-Output ("Removed all the scan logs/reports older than date: $($NotBefore.ToShortDateString()) from storage account: [$StorageAccountName]")
    }
}

function CheckForSubscriptionsSnapshotData()
{            
    try {
        $CAScanDataBlobName = "CAScanObjects.json"    
        $CAActiveScanSnapshotBlobName = "ActiveScanSnapshot.json"
    

        $destinationFolderPath = $env:temp + "\AzSDKTemp\"
        if(-not (Test-Path -Path $destinationFolderPath))
        {
            mkdir -Path $destinationFolderPath -Force
        }
        $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG  -Name $StorageAccountName
        $currentContext = New-AzureStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $keys[0].Value -Protocol Https
        #Fetch if there is any existing active scan snapshot
        $CAScanSourceDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAScanDataBlobName -Context $currentContext -ErrorAction SilentlyContinue
        if($null -eq $CAScanSourceDataBlobObject)
        {
            $Global:IsPreviewMode = $false;
            return;
        }
        $CAScanDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $currentContext -ErrorAction SilentlyContinue 
        if($null -ne $CAScanDataBlobObject)
        {
            $CAActiveSnapshotBlobContentObject = Get-AzureStorageBlobContent -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $currentContext -Destination $destinationFolderPath -Force
            $Global:activeScanObjects = [array](Get-ChildItem -Path "$destinationFolderPath\$CAActiveScanSnapshotBlobName" -Force | Get-Content | ConvertFrom-Json)            
        }
        else
        {
            #Fetch the CA Scan objects
            $CAScanDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAScanDataBlobName -Context $currentContext -ErrorAction Stop | Out-Null
            $CAScanDataBlobContentObject = Get-AzureStorageBlobContent -Container $CentralScanContainerName -Blob $CAScanDataBlobName -Context $currentContext -Destination $destinationFolderPath -Force
            $CAScanDataBlobContent = Get-ChildItem -Path "$destinationFolderPath\$CAScanDataBlobName" -Force | Get-Content | ConvertFrom-Json

            #create the active snapshot from the ca scan objects
            $Global:activeScanObjects = @();
            if(($CAScanDataBlobContent | Measure-Object).Count -gt 0)
            {
                $CAScanDataBlobContent | ForEach-Object {
                    $CAScanDataInstance = $_;
                    $out = "" | Select SubscriptionId, Status, LoggingOption, CreatedTime, StartedTime, CompletedTime
                    $out.SubscriptionId = $CAScanDataInstance.SubscriptionId
                    $out.Status = "NA";
                    $out.LoggingOption = $CAScanDataInstance.LoggingOption;
                    $out.CreatedTime = [DateTime]::UtcNow.ToString();
                    $out.StartedTime = [DateTime]::MinValue.ToString();
                    $out.CompletedTime = [DateTime]::MinValue.ToString();
                    $Global:activeScanObjects += $out;
                }
                $snapshotFilePath = "$destinationFolderPath\$CAActiveScanSnapshotBlobName"
                $Global:activeScanObjects | ConvertTo-Json -Depth 10 | Out-File $snapshotFilePath
                Set-AzureStorageBlobContent -File $snapshotFilePath -Container $CentralScanContainerName -BlobType Block -Context $currentContext -Force
            }
        }
        if(($Global:activeScanObjects | Measure-Object).Count -gt 0)
        {
            $Global:IsPreviewMode = $true;
        }
    }
    catch {
        PublishEvent -EventName "CA Scan Error-PreviewSnapshotComputation" -Properties @{ "ErrorRecord" = ($_ | Out-String) } -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds; "SuccessCount" = 0}
        $Global:IsPreviewMode = $false;
    }
}

function PersistSubscriptionSnapshot
{
    param(
        $SubscriptionID,
        $Status,
        $StorageContext
    )
    try {
        $CAActiveScanSnapshotBlobName = "ActiveScanSnapshot.json"
        $destinationFolderPath = $env:temp + "\AzSDKTemp\"
        if(-not (Test-Path -Path $destinationFolderPath))
        {
            mkdir -Path $destinationFolderPath -Force
        }
        
        #Fetch if there is any existing active scan snapshot
        $CAScanDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $StorageContext -ErrorAction SilentlyContinue 
        if($null -ne $CAScanDataBlobObject)
        {
            $CAActiveSnapshotBlobContentObject = Get-AzureStorageBlobContent -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $StorageContext -Destination $destinationFolderPath -Force
            $activeScanObjects = [array](Get-ChildItem -Path "$destinationFolderPath\$CAActiveScanSnapshotBlobName" -Force | Get-Content | ConvertFrom-Json)

            $matchedSubId = $activeScanObjects | Where-Object {$_.SubscriptionId -eq $SubscriptionID}
            if(($matchedSubId | Measure-Object).Count -gt 0)
            {
                $matchedSubId[0].SubscriptionId = $SubscriptionID
                $matchedSubId[0].Status = $Status;
                if($Status -eq "COM")
                {
                    $matchedSubId[0].CompletedTime = [DateTime]::UtcNow.ToString();
                }
                elseif($Status -eq "INP")
                {
                    $matchedSubId[0].StartedTime = [DateTime]::UtcNow.ToString();
                }
            }
            if(($activeScanObjects | Where-Object { $_.Status -ne "COM"} | Measure-Object).Count -eq 0)
            {
                Remove-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $StorageContext -Force
            }
            else
            {
                $tempFilePath = "$destinationFolderPath\$CAActiveScanSnapshotBlobName"
                $activeScanObjects | ConvertTo-Json -Depth 10 | Out-File $tempFilePath
                Set-AzureStorageBlobContent -File $tempFilePath -Container $CentralScanContainerName -BlobType Block -Context $StorageContext -Force
            }
        }
    }
    catch {
        PublishEvent -EventName "CA Scan Error-PreviewSnapshotPersist" -Properties @{ "ErrorRecord" = ($_ | Out-String) } -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds; "SuccessCount" = 0}
        $Global:IsPreviewMode = $false;
    }
}

try {
    #start timer
    $scanAgentTimer = [System.Diagnostics.Stopwatch]::StartNew();

    #config start
    $ResourceGroupNames = Get-AutomationVariable -Name "AppResourceGroupNames"
    $OMSWorkspaceId = Get-AutomationVariable -Name "OMSWorkspaceId"
    $OMSWorkspaceSharedKey = Get-AutomationVariable -Name "OMSSharedKey"
    $StorageAccountName = Get-AutomationVariable -Name "ReportsStorageAccountName"
    $AzSDKModuleName = "AzSDK"
    $StorageAccountRG = "AzSDKRG"
    $RunbookName = "Continuous_Assurance_Runbook"
    $CAHelperScheduleName = "CA_Helper_Schedule"
    $CentralScanContainerName = "azsdk-scan-objects"
    ##config end

    #Set subscription id
    $SubscriptionID = $RunAsConnection.SubscriptionID
    $Global:IsPreviewMode = $false;
    $Global:activeScanObjects = @();
    Select-AzureRmSubscription -SubscriptionId $SubscriptionID;
    $isAzSDKAvailable = (Get-AzureRmAutomationModule -ResourceGroupName $AutomationAccountRG `
            -AutomationAccountName $AutomationAccountName `
            -Name $AzSDKModuleName -ErrorAction SilentlyContinue | `
            Where-Object {$_.ProvisioningState -eq "Succeeded" -or $_.ProvisioningState -eq "Created"} | `
            Measure-Object).Count -gt 0

    if ($isAzSDKAvailable) {
        Import-Module $AzSDKModuleName
    }

    #check if AzureRM is available (for scenario where AzSDK is available but AzureRM extraction might have failed)
    $isAzureRMAvailable = (Get-AzureRmAutomationModule -ResourceGroupName $AutomationAccountRG `
            -AutomationAccountName $AutomationAccountName `
            -Name AzureRM -ErrorAction SilentlyContinue | `
            Where-Object {$_.ProvisioningState -eq "Succeeded" -or $_.ProvisioningState -eq "Created"} | `
            Measure-Object).Count -gt 0

    #return if modules are not ready
    if ((Get-Command -Name "Get-AzSDKAzureServicesSecurityStatus" -ErrorAction SilentlyContinue|Measure-Object).Count -eq 0 -or !$isAzureRMAvailable) {
        "AzSDK module not available. Skipping AzSDK scan. Will retry in the next run."
        PublishEvent -EventName "CA Job Skipped" -Properties @{"SubscriptionId" = $RunAsConnection.SubscriptionID} -Metrics @{"TimeTakenInMs" = $timer.ElapsedMilliseconds; "SuccessCount" = 1}
        return;
    }
    # Sanitize Storage container
    SanitizeStorage
    #scan and save results to storage
    RunAzSDKScan

    #helper schedule not needed anymore
    Remove-AzureRmAutomationSchedule -Name $CAHelperScheduleName -ResourceGroupName $AutomationAccountRG -AutomationAccountName $AutomationAccountName -Force
    PublishEvent -EventName "CA Scan Completed" -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds}
}
catch {
    PublishEvent -EventName "CA Scan Error" -Properties @{ "ErrorRecord" = ($_ | Out-String) } -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds; "SuccessCount" = 0}
}