Functions/Publish-DatabricksClusterLibrariesToWorkspaceByName.ps1
|
<#
.SYNOPSIS Deploys DataBricks Cluster from configuration json file to a workspace .DESCRIPTION Deploys DataBricks Cluster from configuration json file to a workspace .PARAMETER bearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER clusterConfig The name path of the clusters configuration files. .EXAMPLE Publish-DatabricksClusterLibrariesToWorkspaceByName -librariesConfig '<path-to-file>' .NOTES Author: Sabin IO #> Function Publish-DatabricksClusterLibrariesToWorkspaceByName { [cmdletbinding()] Param( [parameter(Mandatory = $true)][string]$librariesConfig ) try { Write-Host "Setting libraries for $($librariesConfig.cluster_name)" $libraries = Get-Content -Raw -Path $librariesConfig | ConvertFrom-Json $ClusterId = Get-CachedDatabricksClusterId -ClusterName $libraries.cluster_name if ($null -ne $clusterId) { "clusterId $clusterId found" } else{ Write-Host " Cluster ID not found for $($libraries.cluster_name). This is the config." Write-Host $librariesConfig Throw } $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state $libs = Get-DatabricksLibraries -ClusterId $ClusterId $diffs = [PSCustomObject]@{ pypi_in_source_not_in_target = @{} pypi_in_target_not_in_source = @{} cran_in_source_not_in_target = @{} cran_in_target_not_in_source = @{} egg_in_source_not_in_target = @{} egg_in_target_not_in_source = @{} whl_in_source_not_in_target = @{} whl_in_target_not_in_source = @{} jar_in_source_not_in_target = @{} jar_in_target_not_in_source = @{} maven_in_source_not_in_target = @{} maven_in_target_not_in_source = @{} } <# Build Source Lib Objects to Compare #> $sourceLibsPypiObjects = $libraries.libraries | ForEach-Object { if ($_.pypi) { [PSCustomObject]@{ package = $_.pypi.package repo = $_.pypi.repo } } } $sourceLibsCranObjects = $libraries.libraries | ForEach-Object { if ($_.cran) { [PSCustomObject]@{ package = $_.cran.package repo = $_.cran.repo } } } $sourceLibsEggObjects = $libraries.libraries | ForEach-Object { if ($_.egg) { [PSCustomObject]@{ egg = $_.egg } } } $sourceLibsWhlObjects = $libraries.libraries | ForEach-Object { if ($_.whl) { [PSCustomObject]@{ whl = $_.whl } } } $sourceLibsMavenObjects = $libraries.libraries | ForEach-Object { if ($_.maven) { [PSCustomObject]@{ coordinates = $_.maven.coordinates repo = $_.maven.repo exclusions = $_.maven.exclusions } } } $sourceLibsJarObjects = $libraries.libraries | ForEach-Object { if ($_.jar) { [PSCustomObject]@{ jar = $_.jar } } } <# Build Target Lib Objects to Compare #> $targetLibsPypiObjects = $libs | ForEach-Object { if ($_.library.pypi) { [PSCustomObject]@{ package = $_.library.pypi.package repo = $_.library.pypi.repo } } } $targetLibsCranObjects = $libs | ForEach-Object { if ($_.library.cran) { [PSCustomObject]@{ package = $_.library.pypi.package repo = $_.library.pypi.repo } } } $targetLibsEggObjects = $libs | ForEach-Object { if ($_.library.egg) { [PSCustomObject]@{ egg = $_.library.egg } } } $targetLibsWhlObjects = $libs | ForEach-Object { if ($_.library.whl) { [PSCustomObject]@{ whl = $_.library.whl } } } $targetLibsJarObjects = $libs | ForEach-Object { if ($_.library.jar) { [PSCustomObject]@{ jar = $_.library.jar } } } $targetLibsMavenObjects = $libs | ForEach-Object { if ($_.library.maven) { [PSCustomObject]@{ coordinates = $_.library.maven.coordinates repo = $_.library.maven.repo exclusions = $_.library.maven.exclusions } } } <# Get Libs that do not exist on either side #> $pypiLibsInSourceNotInTarget = $sourceLibsPypiObjects | Where-Object { $_.package -notin $targetLibsPypiObjects.package } $pypiLibsInTargetNotInSource = $targetLibsPypiObjects | Where-Object { $_.package -notin $sourceLibsPypiObjects.package } $cranLibsInSourceNotInTarget = $sourceLibsCranObjects | Where-Object { $_.package -notin $targetLibsCranObjects.package } $cranLibsInTargetNotInSource = $targetLibsCranObjects | Where-Object { $_.package -notin $sourceLibsCranObjects.package } $eggLibsInSourceNotInTarget = $sourceLibsEggObjects | Where-Object { $_.egg -notin $targetLibsEggObjects.egg } $eggLibsInTargetNotInSource = $targetLibsEggObjects | Where-Object { $_.egg -notin $sourceLibsEggObjects.egg } $whlLibsInSourceNotInTarget = $sourceLibsWhlObjects | Where-Object { $_.whl -notin $targetLibsWhlObjects.whl } $whlLibsInTargetNotInSource = $targetLibsWhlObjects | Where-Object { $_.whl -notin $sourceLibsWhlObjects.whl } $jarLibsInSourceNotInTarget = $sourceLibsJarObjects | Where-Object { $_.jar -notin $targetLibsJarObjects.jar } $jarLibsInTargetNotInSource = $targetLibsJarObjects | Where-Object { $_.jar -notin $sourceLibsJarObjects.jar } $mavenLibsInSourceNotInTarget = $sourceLibsMavenObjects | Where-Object { $_.coordinates -notin $targetLibsMavenObjects.coordinates } $mavenLibsInTargetNotInSource = $targetLibsMavenObjects | Where-Object { $_.coordinates -notin $sourceLibsMavenObjects.coordinates } [bool]$isDifferences = $false if ($pypiLibsInSourceNotInTarget) { $diffs.pypi_in_source_not_in_target = $pypiLibsInSourceNotInTarget # $diffCounts.pypi_in_source_not_in_target = 1 Write-Host "PyPi Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Host $pypiLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($pypiLibsInTargetNotInSource) { $diffs.pypi_in_target_not_in_source = $pypiLibsInTargetNotInSource # $diffCounts.pypi_in_target_not_in_source = 1 Write-Host "PyPi Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Host $pypiLibsInTargetNotInSource [bool]$isDifferences = $true } if ($cranLibsInSourceNotInTarget) { $diffs.cran_in_source_not_in_target = $cranLibsInSourceNotInTarget Write-Host "Cran Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Host $cranLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($cranLibsInTargetNotInSource) { $diffs.cran_in_target_not_in_source = $cranLibsInTargetNotInSource Write-Host "Cran Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Host $cranLibsInTargetNotInSource [bool]$isDifferences = $true } if ($eggLibsInSourceNotInTarget) { $diffs.egg_in_source_not_in_target = $eggLibsInSourceNotInTarget Write-Host "Egg Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Host $eggLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($eggLibsInTargetNotInSource) { $diffs.egg_in_target_not_in_source = $eggLibsInTargetNotInSource Write-Host "Egg Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Host $eggLibsInTargetNotInSource Write-Host "`$isDifferences is not being set to true as Prod still has egg files set to install across all clusters (Not Recommended)" # [bool]$isDifferences = $true } if ($whlLibsInSourceNotInTarget) { $diffs.whl_in_source_not_in_target = $whlLibsInSourceNotInTarget Write-Host "Whl Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Host $whlLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($whlLibsInTargetNotInSource) { $diffs.whl_in_target_not_in_source = $whlLibsInTargetNotInSource Write-Host "Whl Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Host $whlLibsInTargetNotInSource [bool]$isDifferences = $true } if ($jarLibsInSourceNotInTarget) { $diffs.jar_in_source_not_in_target = $jarLibsInSourceNotInTarget Write-Host "Jar Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Host $jarLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($jarLibsInTargetNotInSource) { $diffs.jar_in_target_not_in_source = $whlLibsInTargetNotInSource Write-Host "Jar Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Host $jarLibsInTargetNotInSource [bool]$isDifferences = $true } if ($mavenLibsInSourceNotInTarget) { $diffs.maven_in_source_not_in_target = $mavenLibsInSourceNotInTarget Write-Host "Maven Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Host $mavenLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($mavenLibsInTargetNotInSource) { $diffs.maven_in_target_not_in_source = $mavenLibsInTargetNotInSource Write-Host "Maven Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Host $mavenLibsInTargetNotInSource [bool]$isDifferences = $true } Write-Host $diffs | Format-List Write-Host "Value of `$isDifferences is $($isDifferences) for `"$($libraries.cluster_name)`"" if ($isDifferences -eq $true) { if ($ClusterState -eq 'TERMINATED') { Write-Host "Cluster State for `"$($libraries.cluster_name)`" is TERMINATED, starting cluster and waiting 30 seconds..." Start-DatabricksCluster -ClusterId $ClusterId Start-Sleep -Seconds 30 } while ( $ClusterState -ne 'RUNNING') { Write-Host "Cluster State for `"$($libraries.cluster_name)`" is not RUNNING, waiting 30 seconds..." Start-Sleep -Seconds 30 $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state } $libraries.PSObject.Properties.Remove('cluster_name') $libraries | Add-Member -NotePropertyName 'cluster_id' -NotePropertyValue $ClusterId Add-DatabricksLibrary -InputObject $libraries } } catch { #uh oh throw $_.Exception } } |