Winfield

1.0.0

UploadWinfield.ps1

# Copyright (c) Microsoft Corporation. All rights reserved.
# UploadWinfield.ps1 1.0.0 2023-04-27 18:34:10
# ASZ-ArcA-Deploy users/sarathys/addtiming Debug-x64

 param
 (
     [Parameter(Mandatory=$true)]
     $arcaBaseFolder
 )
 
 #Requires -RunAsAdministrator
 $ErrorActionPreference = "Stop"
 
 function Trace-Execution([string] $message)
 {
     $caller = (Get-PSCallStack)[1]
     Write-Host "[$(([DateTime]::UtcNow).ToString())][$($caller.Command)] $message"
 }
 
 function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries)
 {
     while($true)
     {
         try
         {
             Invoke-WebRequest $url -OutFile $downloadLocation -Verbose
             break
         }
         catch
         {
             $exceptionMessage = $_.Exception.Message
             Trace-Execution "Failed to download '$url': $exceptionMessage"
             if ($retries -gt 0) {
                 $retries--
                 Trace-Execution "Waiting 10 seconds before retrying. Retries left: $retries"
                 Start-Sleep -Seconds 10
             }
             else
             {
                 $exception = $_.Exception
                 throw $exception
             }
         }
     }
 }
 
 
 function DownloadAzCopy
 {
     param 
     (
         $DownloadURL = "https://aka.ms/downloadazcopy-v10-windows",
         $OutputPath = "C:\AzCopy\"
     )
 
     Trace-Execution "START: Installing AzCopy"
     $azCopyExe = Join-Path -Path $OutputPath -ChildPath "azcopy.exe"
     
     if(Test-Path $azCopyExe)
     {
         Trace-Execution "Azcopy is already installed"
     }
     else 
     {
         Remove-Item -Path $OutputPath -Recurse -Force -ErrorAction SilentlyContinue
         New-Item -Path $OutputPath -ItemType Directory -Force -Verbose | Out-Null
         $azcopyZipFile = Join-Path -Path $OutputPath -ChildPath "AzCopy.zip"
     
         # Download AzCopy and extract the zip file
         Trace-Execution "Downloading Azcopy.exe..."
         DownloadWithRetry -url $DownloadURL -downloadLocation $azcopyZipFile -retries 6
         Expand-Archive -Path $azcopyZipFile -DestinationPath $OutputPath -Force | Out-Null
     
         # Rename the AzCopy executable to "AzCopy.exe"
         $azCopyExeExtracted = Get-ChildItem -Path $OutputPath -Filter "azcopy.exe" -Recurse
         Move-Item -Path $azCopyExeExtracted.FullName -Destination $OutputPath | Out-Null
    
         if(-not (Test-Path $azCopyExe)) 
         {
             throw "$OutputPath\azcopy.exe not found."
         }
     }
     Trace-Execution "END: Installing AzCopy"
     return $azCopyExe
 }
 
 # check pre-requisites
 function Test-Prereqs
 {
     Trace-Execution "Checking if AZURE_DEVOPS_EXT_PAT and SAS_TOKEN environment variables are set."
     $pat = [System.Environment]::GetEnvironmentVariable('AZURE_DEVOPS_EXT_PAT')
     if([string]::IsNullOrEmpty($pat))
     {
         throw "ADO PAT is not set, exiting"
     }
 
     $Global:code = [System.Environment]::GetEnvironmentVariable('SAS_TOKEN')
     if([string]::IsNullOrEmpty($code))
     {
         throw "SAS_TOKEN environment variable isn't set."
     }
 
     Trace-Execution "Pre-req check passed."
     return $true
 }
 
 # Install CLI and AzCopy.exe
 function Init
 {
     Trace-Execution "[START] Init"
     $installed = Get-Command "az.cmd" -ErrorAction "SilentlyContinue"
     if (-not $installed)
     {
         Trace-Execution "Install/Update Azure-CLI"
         DownloadWithRetry -url "https://aka.ms/installazurecliwindows" -downloadLocation ".\AzureCLI.msi" -retries 6
         Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'
         $env:Path += ";c:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin"
         az extension add --name azure-devops
     }
 
     $Global:azCopyExe = DownloadAzCopy
     Trace-Execution "Azcopy installed at $azCopyExe"
     Trace-Execution "[END] Init"
 }
 
 
 # HEAD request on the blob to check if it exists
 function Test-BlobPath
 {
     [CmdletBinding()]
     param (
         [Parameter(Mandatory=$true)]
         [string]
         $Path
     )

    Trace-Execution "Test-BlobPath $Path"
    $blobUrlToCheck = "$Path$code"
    $blobUrlToCheck = $blobUrlToCheck.Replace('"','')
    $found = $false
    try 
    {
        $response = Invoke-WebRequest -Uri $blobUrlToCheck -Method Head -TimeoutSec 30     
        $responseStatusCode = [int] $response.StatusCode
    }
    catch 
    {
        Trace-Execution "Error calling $Path $($_)"
        if($null -ne $_.Exception.Response)
        {
            $responseStatusCode = [int] $_.Exception.Response.StatusCode
        }
    }

    Trace-Execution "HEAD $Path = http/$responseStatusCode"

    if($responseStatusCode -eq 200)
    {
        $found = $true
    }

    return $found
 }
 
 
 function New-BlobContainer 
 {
     param 
     (
         [string] $containerName
     )
 
     $newBlobContainerUrl = "$blobUrl/$containerName$code"
     Invoke-AzCopy -operation "create new blob container: $containerName" -azCopyParameters @("make", $newBlobContainerUrl)
 }
 
 function Get-BlobChildItem
 {
     param
     (
         [string] $containerName,
         [string] $pattern = "."
     )
 
     $containerUrl = "$blobUrl/$containerName$code"
     $blobFiles = @()
     Trace-Execution "list $blobUrl/$containerName"
     $output = Invoke-AzCopy -operation "list $blobUrl/$containerName" -azCopyParameters @("list", $containerUrl)
     $matchInfo = $output | Select-String -Pattern $pattern
     foreach($match in $matchInfo) 
     {
        $blobFiles += $match.Line.Split(";")[0].Split(":")[1].Trim()
     }
     Trace-Execution "Following files are located at $blobUrl/$containerName`:`r`n $($blobFiles | Out-String)"
     return $blobFiles
 }
 
 function Invoke-AzCopy
  {
      [CmdletBinding()]
      param
      (
          [string] $operation,
          [string[]] $azCopyParameters
      )
  
      Trace-Execution "[START][$operation]"
      try
      {
          if($azCopyParameters.Length -lt 2)
          {
              throw "expected atleast 2 azcopy parameters"
          }
  
          $azCopyCommand = $azCopyParameters[0]
          #$azCopyCmdParameters = $azCopyParameters[1..$azCopyParameters.Length] -join " "
  
          #Trace-Execution "Executing $azCopyExe $azCopyCommand $azCopyCmdParameters"
          #$output = & $azCopyExe $azCopyCommand $azCopyCmdParameters
 
          Trace-Execution "Executing $azCopyExe $azCopyCommand"
          Start-Process -FilePath $azCopyExe -ArgumentList $azCopyParameters -RedirectStandardOutput "output.txt" -Wait -NoNewWindow
          $output = Get-Content -Path "output.txt"
          Trace-Execution "azcopy output: $output"
      }
      catch 
      {
          if($ErrorActionPreference -eq 'Continue')
          {
              Trace-Execution "Ignoring error $($_)"
          }
          else 
          {
              throw $_.Exception
          }
      }
  
      Trace-Execution "[END][$operation]"
      return $output
  }
 
  function Test-ArtifactsOnDisk 
  {
      param 
      (
          [string] $Path
      )
  
      $expectedArtifacts = @('ArcA_EphemeralData_IRVM01_1.vhdx', 'ArcA_LocalData_IRVM01_1.vhdx', 'ArcA_SharedData_IRVM01.vhdx', 'Docker_IRVM01_1.vhdx', 'IRVM01.vhdx', '*.vmcx', '*.vmgs', '*.VMRS')
      foreach($expectedArtifact in $expectedArtifacts)
      {
          $artifactFile = Get-ChildItem -Path $Path -Recurse -File -Filter $expectedArtifact
  
          if($null -eq $artifactFile) 
          {
              return $false
          }
  
          if(-not (Test-Path $artifactFile.FullName))
          {
              return $false
          }
      }
      return $true
  }
  
function Get-ArcAFolder 
{
    $date = [datetime]::Now.ToString("MM-dd-yyyy") 
    $arcaFolder = "$arcaBaseFolder\$date"
    return $arcaFolder
}
  
 function DownloadLKG
 {
     param
     (
         $arcaBaseFolder = "F:\arca"
     )
 
     Trace-Execution "[START] Download LKG artifacts from AzureStackUniversalBuddy feed"
     $driveLetter = $arcaBaseFolder.Split(':')[0]
     $remainingSize = (Get-Volume -DriveLetter $driveLetter).SizeRemaining
     Trace-Execution "Remaining disk space in folder $arcaBaseFolder $($remainingSize/1Gb) Gb."
     if($remainingSize -lt 120Gb)
     {
         throw "expect atleast 120Gb to download artifacts"
     }
     
     $arcaFolder = Get-ArcAFolder
     mkdir $arcaFolder -Force | Out-Null
 
     if(Test-ArtifactsOnDisk -Path $arcaFolder)
     {
         Trace-Execution "Artifacts already exist in $arcaFolder skip download."
     }
     else
     {
         Trace-Execution "[START] download artifacts to $arcaFolder"
         # assumes PAT already set. See https://learn.microsoft.com/en-us/azure/devops/cli/log-in-via-pat?view=azure-devops&tabs=windows for details.
         $downloadOutput = az artifacts universal download --organization "https://msazure.visualstudio.com/" --feed "AzureStackUniversalBuddy" --name "arca.onenode.complete" --version "*" --path $arcaFolder
         $downloadedFiles = (Get-ChildItem -Path $arcaFolder -File -Recurse).FullName
         Trace-Execution "az artifacts universal download output: $downloadOutput"
         Trace-Execution "Following files were downloaded:`r`n$downloadedFiles`r`n"
     }

     Save-ArtifactMd5Hash -Path $arcaFolder
     Trace-Execution "[END] Download LKG artifacts"
     return $arcaFolder
 }
 
 # reads $blobUrl/control/control.json
 <#
 Sample control.json file:
 {
     "PreviousLKGContainer": "lkg",
     "CopyInProgress": false,
     "LKGContainer": "04-25-2023"
 }
 #>
 function Read-ControlData
 {
     $localDataFolder = "$env:APPDATA\Winfield"
     mkdir $localDataFolder -Force
     $downloadedControlDataFile = "$localDataFolder\downloadedControlDataFile.json"
     $azCopyParameters = @("cp", "$blobUrl/control/control.json$code", $downloadedControlDataFile)
     Invoke-AzCopy -operation "Read control data" -azCopyParameters $azCopyParameters
     $controlData = Get-Content $downloadedControlDataFile | ConvertFrom-Json
     Trace-Execution "LKG control data:`r`n $(Get-Content $downloadedControlDataFile)`r`n"
     return $controlData
 }
 
 # writes control.json file to blob. checks etag for concurrency control
 function Write-ControlData
 {
     param 
     (
         [string] $containerName,
         [string] $previousContainerName,
         [bool]   $copyInProgress,
         [string] $etag
     )
     
     $controlData = @{LKGContainer = $containerName; CopyInProgress=$copyInProgress; PreviousLKGContainer = $previousContainerName}
     $localDataFolder = "$env:APPDATA\Winfield"
     mkdir $localDataFolder -Force
     $controlDataFile = "$localDataFolder\control.json"
     $controlData | ConvertTo-Json | Out-File $controlDataFile -Encoding ascii
     
     $currEtag = Get-ControlDataEtag
     if($currEtag -eq $etag)
     {
         Trace-Execution "writing control data:`r`n $($controlData | Out-String) to $blobUrl/control/control.json"
         $azCopyParameters = @("cp", $controlDataFile, "$blobUrl/control$code")
         Invoke-AzCopy -operation "Write control data" -azCopyParameters $azCopyParameters
         Trace-Execution "control data file written to $blobUrl/control/control.json"
     }
     else 
     {
         Trace-Execution "Concurrency check failed. Given etag: $etag current etag: $currEtag"
         throw "Write-ControlData precondition failed. Given etag: $etag current etag: $currEtag"
     }
     
     return $true
 }
 
 # Get blob header via HEAD request, adds SAS token to the Uri
 function Get-BlobHeader
 {
     param
     (
         [string] $url,
         [string] $header
     )
 
     Trace-Execution "wget uri: $url"
     $urlWithCode = "$url$code"
     $urlWithCode = $urlWithCode.Replace('"','')
     $headerValue = $null
     try 
     {
        $response = Invoke-WebRequest -Uri $urlWithCode -Method Head -TimeoutSec 30
        $responseStatusCode = [int] $_.Exception.Response.StatusCode
        $headerValue = $response.Headers[$header]
     }
     catch 
     {
        if($null -ne $_.Exception.Response)
        {
            $responseStatusCode = [int] $_.Exception.Response.StatusCode
        }

        if($responseStatusCode -ne 404)
        {
            throw "$($_)"
        }
     }

     Trace-Execution "HEAD $url = http/$responseStatusCode"
     Trace-Execution "HEADER $header = $headerValue"
     return $headerValue
 }
 
 function Get-BlobEtag
 {
     param
     (
         [string] $url
     )
 
     $etag = Get-BlobHeader -url $url -header "ETag"
     Trace-Execution "ETAG $url = $etag"
     return $etag
 }

 function Get-BlobMD5Hash
 {
     param
     (
         [string] $url
     )
 
     $md5 = Get-BlobHeader -url $url -header "Content-MD5"
     $md5hash = $null
     if($null -ne $md5)
     {
        Trace-Execution "Content-MD5 $url = $md5"
        $md5Sum = [System.Convert]::FromBase64String($md5)
        $md5hash = [System.BitConverter]::ToString($md5Sum).Replace('-','')
        Trace-Execution "Blob MD5 $url = $md5hash"
     }
     return $md5hash
 }
 
 function Get-ControlDataEtag
 {
     $controlDataFile = "$blobUrl/control/control.json"
     return (Get-BlobEtag -url $controlDataFile)
 }
 


 # saves MD5 hash of downloaded artifacts in a json file
 function Save-ArtifactMd5Hash 
 {
    param 
    (
        [string] $Path
    )

    Trace-Execution "[START] compute md5 file hash for downloaded artifacts"
    $artifactFilePath = Join-Path $Path -ChildPath "artifacthash.json"
    
    if(-not (Test-Path $artifactFilePath))
    {
        Trace-Execution "$artifactFilePath doeesn't exist, computing MD5 file hashes."
        $fileHashes = @()
        $artifacts = Get-ChildItem -Path $Path -Recurse -File
        foreach($artifact in $artifacts)
        {
            $md5Hash = (Get-FileHash -Path $artifact.FullName -Algorithm MD5).Hash
            $fileHashes += @{FileName = $artifact.Name; FileFullName = $artifact.FullName; MD5 = $md5Hash}
        }

        $artifactHash = @{ArtifactHashInfo = $fileHashes}

        $artifactHash | ConvertTo-Json | Out-File $artifactFilePath -Force -Verbose
        
    }
    
    Trace-Execution "Downloaded Artifact file hashes stored at $artifactFilePath, contents:`r`n$(Get-Content $artifactFilePath)`r`n"
    Trace-Execution "[END] compute md5 file hashes"
 }

# compare MD5 file hash on file system file and blob file
# todo: cache MD5 hash results
 function Compare-Md5Hash 
{
    param 
    (
        $fileSource,
        $blobDestination
    )

    Trace-Execution "Comparing MD5 hash fileSource: $fileSource and blobDestination: $blobDestination"
    if(Test-Path $fileSource)
    {
        #$fileMd5 = (Get-FileHash -Algorithm MD5 -Path $fileSource).Hash
        $artifactHashFile = Join-Path -path (Get-ArcAFolder) -ChildPath "artifacthash.json"
        $fileHashes = (Get-Content -Path $artifactHashFile | ConvertFrom-Json).ArtifactHashInfo
        $fileMd5 = ($fileHashes | Where-Object{$_.FileFullName -eq $fileSource}).MD5
        $blobMd5 = Get-BlobMD5Hash -url "$blobDestination"
        Trace-Execution "File MD5 = $fileMd5 Blob MD5 = $blobMd5"
        return ($fileMd5 -eq $blobMd5)
    }
    return $false
}

 function UploadLKGtoBlob
 {
     param
     (
         [string] $arcaFolder = "F:\arca"
     )
 
     Trace-Execution "[START] Upload files from $arcaFolder to Blob storage"
 
     if(Test-Path -Path $arcaFolder)
     {
         $date = [datetime]::Now.ToString("MM-dd-yyyy")
         $blobLKG = "$blobUrl/lkg-$date"
 
         # indicate copy is in progress
         $controlData = Read-ControlData
         $currentEtag = Get-ControlDataEtag
         Write-ControlData -blobUrl $blobUri -containerName $date -previousContainerName $controlData.LKGContainer -copyInProgress $true -etag $currentEtag
 
         # azcopy cp "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true --put-md5
         $artifactsToUpload = Get-ChildItem -Path $arcaFolder -Recurse -File

         try
         {
             foreach($artifact in $artifactsToUpload)
             {
                $blobLocation = "$blobLKG/$($artifact.Name)"
                Trace-Execution "Checking to see if $blobLocation exists."
                $blobExists = Test-BlobPath -Path $blobLocation
                $md5AreEqual = Compare-Md5Hash -fileSource $artifact.FullName -blobDestination $blobLocation
                if($blobExists -and $md5AreEqual)
                {
                    Trace-Execution "Blob $blobLocation exists with same MD5 hash, skipping upload."
                }
                else 
                {
                    $artifact = '"' + $artifact.FullName + '"'        
                    Trace-Execution "[START] Copy $artifact to $blobLKG"
                    Invoke-AzCopy -operation "copy" -azCopyParameters @("cp", $artifact, "$blobLKG$code", "--recursive=true", "--put-md5")
                }
             }

            Trace-Execution "Winfield artifacts from folder $arcaFolder uploaded to $blobUrl/lkg-$date"
            $currentEtag = Get-ControlDataEtag
            Write-ControlData -blobUrl $blobUri -containerName $date -previousContainerName $controlData.LKGContainer -copyInProgress $false -etag $currentEtag
            Trace-Execution "control data updated."
         }
         catch 
         {
             Trace-Execution "error uploading artifacts:`r`n $($_). `r`nUpdating control data to use previous LKG."
             $currentEtag = Get-ControlDataEtag
             Write-ControlData -blobUrl $blobUri -containerName $date -previousContainerName $controlData.PreviousLKGContainer -copyInProgress $false -etag $currentEtag
         }
     }
     else 
     {
         Trace-Execution "ArcA artifact folder $arcaFolder does not exist."
     }
     Trace-Execution "[END] Upload to Blob storage"
 }
 
 function Verify
 {
     param
     (
         [string] $containerName,
         [string] $arcaFolder
     )
 
     $uploadVerified = $false
     
     Trace-Execution "[START] Verify upload"
     $blobArtifacts = Get-BlobChildItem -containerName $containerName -pattern "."
     Trace-Execution "Artifacts in $containerName`: $blobArtifacts"
     Trace-Execution "Get artifacts on local filesystem path: $arcaFolder"
     $fileArtifacts = Get-ChildItem -Path $arcaFolder -File -Recurse -Filter "*.*"
     if($fileArtifacts.count -ne $blobArtifacts.count)
     {
         Trace-Execution "Expected files: $($fileArtifacts.FullName) found files in blob: $vhdBlobFiles"
         throw "did not find expected vhdx files in $containerName"
     }
     else 
     {
        foreach($fileArtifact in $fileArtifacts)
        {
            $date = [datetime]::Now.ToString("MM-dd-yyyy")
            $blobLKG = "$blobUrl/lkg-$date"
            $blobLocation = "$blobLKG/$($fileArtifact.Name)"
            Trace-Execution "Checking to see if $blobLocation exists."
            $blobExists = Test-BlobPath -Path $blobLocation
            $md5AreEqual = Compare-Md5Hash -fileSource $fileArtifact.FullName -blobDestination $blobLocation
            if($blobExists -and $md5AreEqual)
            {
                Trace-Execution "[VERIFY] $($fileArtifact.FullName) and blob $blobLocation MD5 match."
            }
            else 
            {
                Trace-Execution "[VERIFY] $($fileArtifact.FullName) and blob $blobLocation do not match."
                throw "Verify failed"
            }
        }
        $uploadVerified = $true
     }
 
     Trace-Execution "[END] Verify upload"
     return $uploadVerified
 }
 
 #
 # main
 #

  # START Transcript
 $timestamp = [DateTime]::Now.ToString("yyyyMMdd-HHmmss")
 $logPath = (New-Item -Path "$env:ProgramData\Microsoft\Winfield\Logs" -ItemType Directory -Force).FullName
 $logFile = Join-Path -Path $logPath -ChildPath "ImportWinField_${timestamp}.txt"
 try { Start-Transcript -Path $logFile -Force | Out-String | Write-Verbose -Verbose } catch { Write-Warning -Message $_.Exception.Message }
 
 $Global:blobUrl = 'https://winfieldartifacts.blob.core.windows.net'
 $sw = [System.Diagnostics.Stopwatch]::StartNew()
 try 
 {
    Test-Prereqs
    Init

    # Download
    Trace-Execution "downloading to $arcaBaseFolder"
    $downloadedArtifactsFolder = DownloadLKG -arcaBaseFolder $arcaBaseFolder
    $downloadedArtifactsFolder += "\IRVM01"
    
    # Upload
    $containerName = "lkg-$([datetime]::Now.ToString("MM-dd-yyyy"))"
    Trace-Execution "Creating blob container $containerName"
    New-BlobContainer -containerName $containerName
    Trace-Execution "Uploading $downloadedArtifactsFolder folder contents recursively to blob container $containerName"
    UploadLKGtoBlob -arcaFolder "$downloadedArtifactsFolder"

    # Verify upload
    Trace-Execution "Verify $downloadedArtifactsFolder contents were uploaded to blob container $containerName"
    Verify -containerName $containerName -arcaFolder $downloadedArtifactsFolder
 }
 catch 
 {
    Trace-Execution "Error uploading artifacts:`r`n$($_)`r`n"
 }
 finally
 {
    $sw.Stop()
    Trace-Execution "Script completed execution in $($sw.Elapsed.TotalSeconds) sec."
    try { Stop-Transcript | Out-String | Write-Verbose -Verbose } catch { Write-Warning -Message $_.Exception.Message }
 }