Public/LibrariesCustom.ps1
|
Function Add-DatabricksClusterLocalLibrary { <# .SYNOPSIS Uploads a local library (.jar, .whl, ...) to DBFS and adds it to a cluster. .DESCRIPTION Retrieves the information for a cluster given its identifier. Clusters can be described while they are running, or up to 30 days after they are terminated. Official API Documentation: https://docs.databricks.com/api/latest/clusters.html#get .PARAMETER ClusterID The ID of the cluster on which you want to install the library. .PARAMETER LocalPath The local path where the library (.jar, .whl, ...) is located. .PARAMETER DBFSPath The DBFS path where to store the library file. Default is "/libraries/" .EXAMPLE Add-DatabricksClusterLocalLibrary -ClusterID "1202-211320-brick1" -LocalPath "C:\myLibrary.jar" #> param ( [Parameter(Mandatory = $true, ValueFromPipelineByPropertyName = $true)] [Alias("cluster_id")] [string] $ClusterID, [Parameter(Mandatory = $true, ValueFromPipelineByPropertyName = $true)] [Alias("local_path", "path")] [string] $LocalPath, [Parameter(Mandatory = $false)] [Alias("dbfs_path")] [string] $DBFSPath = "/libraries/", [Parameter(Mandatory = $false)] [Alias("library_type")] [string] [ValidateSet('jar', 'whl', 'egg', 'AUTO')]$LibraryType = "AUTO" ) begin { } process { $localFile = Get-Item $LocalPath $dbfsFullPath = "/" + $DBFSPath.Trim("/") + "/" + $localFile.Name Write-Verbose "Uploading local library ($($localFile.FullName)) to DBFS ($dbfsFullPath) ..." $x = Upload-DatabricksFSFile -Path $dbfsFullPath -LocalPath $localFile.FullName -Overwrite $true Write-Verbose "Getting Cluster $ClusterID ..." $cluster = Get-DatabricksCluster -ClusterID $ClusterID # we can only add libraries to running clusters! if ($cluster.state -eq "TERMINATED") { Write-Warning "Cluster $($cluster.cluster_name) is in TERMINATED state and libraries can only be installed on running clusters!" Write-Verbose "Temporary starting cluster $($cluster.cluster_name) (ID: $($cluster.cluster_id)) to install latest libraries ..." Start-DatabricksCluster -ClusterID $cluster.cluster_id Write-Verbose "Waiting 10 seconds for cluster startup ..." Start-Sleep -Seconds 10 } if($LibraryType -eq "AUTO") { Write-Verbose "Deriving LibraryType from filename ..." $LibraryType = $localFile.Extension.ToLower().Replace(".", "") Write-Verbose " LibraryType from filename: $LibraryType" } $libraries = @( @{ $LibraryType = "dbfs:$dbfsFullPath" } ) Write-Verbose "Installing library '$($locaFile.Name)' on cluster $($cluster.cluster_name) (ID: $($cluster.cluster_id) ..." $result = Add-DatabricksClusterLibraries -ClusterID $cluster.cluster_id -Libraries $libraries Write-Verbose "Libraries installed!" Write-Verbose "Waiting 10 seconds for library installation ..." Start-Sleep -Seconds 10 if ($cluster.state -eq "TERMINATED") { Write-Verbose "Terminating cluster $($cluster.cluster_name) (ID: $($cluster.cluster_id)) after installing libraries ..." Stop-DatabricksCluster -ClusterID $cluster.cluster_id } return } } |