Functions/Publish-DatabricksClusterSingletonToWorkspaceByName.ps1
|
<#
.SYNOPSIS Deploys DataBricks Cluster from configuration json file to a workspace .DESCRIPTION Deploys DataBricks Cluster from configuration json file to a workspace .PARAMETER config Configuration json file from the environment used to workout whether to deploy a clusters from a folder or file(s) .PARAMETER bearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER clusterConfig The name path of the clusters configuration files. .EXAMPLE Publish-DatabricksClusterSingletonToWorkspaceByName -config $config -bearerToken 'dapi1234567890' -clusterConfig '<path-to-file>' .NOTES Author: Sabin IO #> Function Publish-DatabricksClusterSingletonToWorkspaceByName { [cmdletbinding()] Param( [parameter(Mandatory = $true)]$config, [parameter(Mandatory = $true)][string]$bearerToken, [parameter(Mandatory = $true)][string]$clusterConfig ) try { $cluster = Get-Content -Raw -Path $clusterConfig | ConvertFrom-Json $Spark_conf = @{} $CustomTags = @{} $InitScripts = @() $SparkEnvVars = @{} if ($cluster.spark_conf) { $cluster.spark_conf.PSObject.Properties | ForEach-Object { $Spark_conf[$_.Name] = $_.Value } } if ($cluster.custom_tags) { $cluster.custom_tags.PSObject.Properties | ForEach-Object { $CustomTags[$_.Name] = $_.Value } } if ($cluster.init_scripts) { $cluster.init_scripts | ForEach-Object { $InitScripts += $_.dbfs.destination } } if ($cluster.spark_env_vars) { $cluster.spark_env_vars.PSObject.Properties | ForEach-Object { $SparkEnvVars[$_.Name] = $_.Value } } New-DatabricksCluster -BearerToken $bearerToken -Region $config.Region ` -ClusterName $cluster.cluster_name ` -SparkVersion $cluster.spark_version ` -NodeType $cluster.node_type_id ` -DriverNodeType $cluster.driver_node_type_id ` -MinNumberOfWorkers $cluster.autoscale.min_workers ` -MaxNumberOfWorkers $cluster.autoscale.max_workers ` -AutoTerminationMinutes $cluster.autotermination_minutes ` -Spark_conf $Spark_conf ` -CustomTags $CustomTags ` -InitScripts $InitScripts ` -SparkEnvVars $SparkEnvVars ` -PythonVersion 3 } catch { #uh oh throw $_.Exception } } |