Functions/Publish-DatabricksClusterLibrariesToWorkspaceByName.ps1

<#
.SYNOPSIS
Deploys DataBricks Cluster from configuration json file to a workspace
 
.DESCRIPTION
Deploys DataBricks Cluster from configuration json file to a workspace
 
.PARAMETER bearerToken
Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI)
 
.PARAMETER clusterConfig
The name path of the clusters configuration files.
 
.EXAMPLE
Publish-DatabricksClusterLibrariesToWorkspaceByName -librariesConfig '<path-to-file>'
 
.NOTES
Author: Sabin IO
 
#>
 
Function Publish-DatabricksClusterLibrariesToWorkspaceByName {
    [cmdletbinding()]
    Param(
        [parameter(Mandatory = $true)][string]$librariesConfig
    )
    try {

        $libraries = Get-Content -Raw -Path $librariesConfig | ConvertFrom-Json
        $ClusterId = Get-CachedDatabricksClusterId -ClusterName $libraries.cluster_name
        $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state
        $libs = Get-DatabricksLibraries -ClusterId $ClusterId

        $diffs = [PSCustomObject]@{
            pypi_in_source_not_in_target  = @{}
            pypi_in_target_not_in_source  = @{}
            cran_in_source_not_in_target  = @{}
            cran_in_target_not_in_source  = @{}
            egg_in_source_not_in_target   = @{}
            egg_in_target_not_in_source   = @{}
            whl_in_source_not_in_target   = @{}
            whl_in_target_not_in_source   = @{}
            jar_in_source_not_in_target   = @{}
            jar_in_target_not_in_source   = @{}
            maven_in_source_not_in_target = @{}
            maven_in_target_not_in_source = @{}
        }
        <# Build Source Lib Objects to Compare #>
        $sourceLibsPypiObjects = $libraries.libraries |
        ForEach-Object {
            if ($_.pypi) {
                [PSCustomObject]@{
                    package = $_.pypi.package
                    repo    = $_.pypi.repo         
                }
            }
        }

        $sourceLibsCranObjects = $libraries.libraries |
        ForEach-Object {
            if ($_.cran) {
                [PSCustomObject]@{
                    package = $_.cran.package
                    repo    = $_.cran.repo         
                }
            }
        }
        $sourceLibsEggObjects = $libraries.libraries |
        ForEach-Object {
            if ($_.egg) {
                [PSCustomObject]@{
                    egg = $_.egg
                }
            }
        }
        $sourceLibsWhlObjects = $libraries.libraries |
        ForEach-Object {
            if ($_.whl) {
                [PSCustomObject]@{
                    whl = $_.whl
                }
            }
        }
        $sourceLibsMavenObjects = $libraries.libraries |
        ForEach-Object {
            if ($_.maven) {
                [PSCustomObject]@{
                    coordinates = $_.maven.coordinates
                    repo        = $_.maven.repo     
                    exclusions  = $_.maven.exclusions      
                }
            }
        }

        $sourceLibsJarObjects = $libraries.libraries |
        ForEach-Object {
            if ($_.jar) {
                [PSCustomObject]@{
                    jar = $_.jar
                }
            }
        }
        
        <# Build Target Lib Objects to Compare #>
        $targetLibsPypiObjects = $libs |
        ForEach-Object {
            if ($_.library.pypi) {
                [PSCustomObject]@{
                    package = $_.library.pypi.package
                    repo    = $_.library.pypi.repo         
                }
            }
        }
        $targetLibsCranObjects = $libs |
        ForEach-Object {
            if ($_.library.cran) {
                [PSCustomObject]@{
                    package = $_.library.pypi.package
                    repo    = $_.library.pypi.repo         
                }
            }
        }
        $targetLibsEggObjects = $libs |
        ForEach-Object {
            if ($_.library.egg) {
                [PSCustomObject]@{
                    egg = $_.library.egg
                }
            }
        }
        $targetLibsWhlObjects = $libs |
        ForEach-Object {
            if ($_.library.whl) {
                [PSCustomObject]@{
                    whl = $_.library.whl
                }
            }
        }

        $targetLibsJarObjects = $libs |
        ForEach-Object {
            if ($_.library.jar) {
                [PSCustomObject]@{
                    jar = $_.library.jar
                }
            }
        }
        $targetLibsMavenObjects = $libs |
        ForEach-Object {
            if ($_.library.maven) {
                [PSCustomObject]@{
                    coordinates = $_.library.maven.coordinates
                    repo        = $_.library.maven.repo     
                    exclusions  = $_.library.maven.exclusions      
                }
            }
        }
        
        <# Get Libs that do not exist on either side #>
        $pypiLibsInSourceNotInTarget = $sourceLibsPypiObjects | Where-Object { $_.package -notin $targetLibsPypiObjects.package } 
        $pypiLibsInTargetNotInSource = $targetLibsPypiObjects | Where-Object { $_.package -notin $sourceLibsPypiObjects.package } 

        $cranLibsInSourceNotInTarget = $sourceLibsCranObjects | Where-Object { $_.package -notin $targetLibsCranObjects.package } 
        $cranLibsInTargetNotInSource = $targetLibsCranObjects | Where-Object { $_.package -notin $sourceLibsCranObjects.package } 

        $eggLibsInSourceNotInTarget = $sourceLibsEggObjects | Where-Object { $_.egg -notin $targetLibsEggObjects.egg } 
        $eggLibsInTargetNotInSource = $targetLibsEggObjects | Where-Object { $_.egg -notin $sourceLibsEggObjects.egg } 

        $whlLibsInSourceNotInTarget = $sourceLibsWhlObjects | Where-Object { $_.whl -notin $targetLibsWhlObjects.whl } 
        $whlLibsInTargetNotInSource = $targetLibsWhlObjects | Where-Object { $_.whl -notin $sourceLibsWhlObjects.whl } 

        $jarLibsInSourceNotInTarget = $sourceLibsJarObjects | Where-Object { $_.jar -notin $targetLibsJarObjects.jar } 
        $jarLibsInTargetNotInSource = $targetLibsJarObjects | Where-Object { $_.jar -notin $sourceLibsJarObjects.jar } 

        $mavenLibsInSourceNotInTarget = $sourceLibsMavenObjects | Where-Object { $_.coordinates -notin $targetLibsMavenObjects.coordinates } 
        $mavenLibsInTargetNotInSource = $targetLibsMavenObjects | Where-Object { $_.coordinates -notin $sourceLibsMavenObjects.coordinates } 
        
        [bool]$isDifferences = $false

        if ($pypiLibsInSourceNotInTarget) {
            $diffs.pypi_in_source_not_in_target = $pypiLibsInSourceNotInTarget
            # $diffCounts.pypi_in_source_not_in_target = 1
            Write-Host "PyPi Library/Libraries in Source not in Source `"$($libraries.cluster_name)`""
            Write-Host $pypiLibsInSourceNotInTarget 
            [bool]$isDifferences = $true
        }
        if ($pypiLibsInTargetNotInSource) {
            $diffs.pypi_in_target_not_in_source = $pypiLibsInTargetNotInSource
            # $diffCounts.pypi_in_target_not_in_source = 1
            Write-Host "PyPi Library/Libraries in Target not in Source `"$($libraries.cluster_name)`""
            Write-Host $pypiLibsInTargetNotInSource
            [bool]$isDifferences = $true
        }

        if ($cranLibsInSourceNotInTarget) {
            $diffs.cran_in_source_not_in_target = $cranLibsInSourceNotInTarget
            Write-Host "Cran Library/Libraries in Source not in Source `"$($libraries.cluster_name)`""
            Write-Host $cranLibsInSourceNotInTarget 
            [bool]$isDifferences = $true
        }
        if ($cranLibsInTargetNotInSource) {
            $diffs.cran_in_target_not_in_source = $cranLibsInTargetNotInSource
            Write-Host "Cran Library/Libraries in Target not in Source `"$($libraries.cluster_name)`""
            Write-Host $cranLibsInTargetNotInSource 
            [bool]$isDifferences = $true
        }

        if ($eggLibsInSourceNotInTarget) {
            $diffs.egg_in_source_not_in_target = $eggLibsInSourceNotInTarget
            Write-Host "Egg Library/Libraries in Source not in Source `"$($libraries.cluster_name)`""
            Write-Host $eggLibsInSourceNotInTarget
            [bool]$isDifferences = $true
        }
        if ($eggLibsInTargetNotInSource) {
            $diffs.egg_in_target_not_in_source = $eggLibsInTargetNotInSource
            Write-Host "Egg Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" 
            Write-Host $eggLibsInTargetNotInSource
            Write-Host "`$isDifferences is not being set to true as Prod still has egg files set to install across all clusters (Not Recommended)"
            # [bool]$isDifferences = $true
        }

        if ($whlLibsInSourceNotInTarget) {
            $diffs.whl_in_source_not_in_target = $whlLibsInSourceNotInTarget
            Write-Host "Whl Library/Libraries in Source not in Source `"$($libraries.cluster_name)`""
            Write-Host $whlLibsInSourceNotInTarget 
            [bool]$isDifferences = $true
        }
        if ($whlLibsInTargetNotInSource) {
            $diffs.whl_in_target_not_in_source = $whlLibsInTargetNotInSource
            Write-Host "Whl Library/Libraries in Target not in Source `"$($libraries.cluster_name)`""
            Write-Host $whlLibsInTargetNotInSource 
            [bool]$isDifferences = $true
        }

        if ($jarLibsInSourceNotInTarget) {
            $diffs.jar_in_source_not_in_target = $jarLibsInSourceNotInTarget
            Write-Host "Jar Library/Libraries in Source not in Source `"$($libraries.cluster_name)`""
            Write-Host $jarLibsInSourceNotInTarget 
            [bool]$isDifferences = $true
        }
        if ($jarLibsInTargetNotInSource) {
            $diffs.jar_in_target_not_in_source = $whlLibsInTargetNotInSource
            Write-Host "Jar Library/Libraries in Target not in Source `"$($libraries.cluster_name)`""
            Write-Host $jarLibsInTargetNotInSource 
            [bool]$isDifferences = $true
        }

        if ($mavenLibsInSourceNotInTarget) {
            $diffs.maven_in_source_not_in_target = $mavenLibsInSourceNotInTarget
            Write-Host "Maven Library/Libraries in Source not in Source `"$($libraries.cluster_name)`""
            Write-Host $mavenLibsInSourceNotInTarget 
            [bool]$isDifferences = $true
        }
        if ($mavenLibsInTargetNotInSource) {
            $diffs.maven_in_target_not_in_source = $mavenLibsInTargetNotInSource
            Write-Host "Maven Library/Libraries in Target not in Source `"$($libraries.cluster_name)`""
            Write-Host $mavenLibsInTargetNotInSource 
            [bool]$isDifferences = $true
        }
        
        Write-Host $diffs | Format-List
        Write-Host "Value of `$isDifferences is $($isDifferences) for `"$($libraries.cluster_name)`""
       
        if ($isDifferences -eq $true) {
            
            if ($ClusterState -eq 'TERMINATED') {
                Write-Host "Cluster State for `"$($libraries.cluster_name)`" is TERMINATED, starting cluster and waiting 30 seconds..."
                Start-DatabricksCluster -ClusterId $ClusterId -BearerToken $bearerToken 
                Start-Sleep -Seconds 30        
            }
        
            while ( $ClusterState -ne 'RUNNING') {
                Write-Host "Cluster State for `"$($libraries.cluster_name)`" is not RUNNING, waiting 30 seconds..."
                Start-Sleep -Seconds 30
                $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state
            }

            $libraries.PSObject.Properties.Remove('cluster_name')  
            $libraries | Add-Member -NotePropertyName 'cluster_id' -NotePropertyValue $ClusterId

            Add-DatabricksLibrary -InputObject $libraries
        }
    }    
    catch {
        #uh oh
        throw $_.Exception
    }
}