Public/Import-DatabricksFolder.ps1

<#
.SYNOPSIS
Pushes the contents of a local folder (and subfolders) to Databricks
 
.DESCRIPTION
Use to deploy code from a repo
 
.PARAMETER BearerToken
Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI)
 
.PARAMETER Region
Azure Region - must match the URL of your Databricks workspace, example northeurope
 
.PARAMETER LocalPath
Path to your repo/local files that you would like to deploy to Databricks (should be in Source format)
 
.PARAMETER DatabricksPath
The Databricks folder to target
 
.EXAMPLE
PS C:\> Import-DatabricksFolder -BearerToken $BearerToken -Region $Region -LocalPath 'Samples\DummyNotebooks' -DatabricksPath 'Shared\ProjectX'
 
.NOTES
Author: Simon D'Morias / Data Thirst Ltd
 
#>
  

Function Import-DatabricksFolder
{
    [cmdletbinding()]
    param (
        [parameter(Mandatory=$false)][string]$BearerToken,
        [parameter(Mandatory=$false)][string]$Region,
        [parameter(Mandatory=$true)][string]$LocalPath,
        [parameter(Mandatory=$true)][string]$DatabricksPath
    )

    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    $Headers = GetHeaders $PSBoundParameters

    Push-Location
    $Files = Get-ChildItem $LocalPath -Recurse -Attributes !D
    Set-Location $LocalPath
    
    ForEach ($FileToPush In $Files)
    {
        $Path = $FileToPush.DirectoryName
        $LocalPath = $LocalPath.Replace("/","\")

        if ($FileToPush.DirectoryName -ne (Get-Location).Path) {
            $FolderFromTargetRoot = (Resolve-Path ($FileToPush.DirectoryName) -Relative)
            $Path = Join-Path $DatabricksPath $FolderFromTargetRoot
        }
        else{
            $Path = $DatabricksPath
        }
        
        $Path = $Path.Replace("\","/")
        $Path = $Path.Replace("/./","/")

        # Create folder in Databricks
        Add-DatabricksFolder -Path $Path
        Write-Verbose "Path: $Path"

        $BinaryContents = [System.IO.File]::ReadAllBytes($FileToPush.FullName)
        $EncodedContents = [System.Convert]::ToBase64String($BinaryContents)
        $TargetPath = $Path + '/'+ $FileToPush.BaseName

        $Body = @{}
        $Body['content'] = $EncodedContents
        $Body['path'] = $TargetPath
        $Body['overwrite'] = "true"
        switch ($FileToPush.Extension){
            ".py" {
                $Body['format'] = "SOURCE"
                $Body['language'] = "PYTHON"
            }

            ".scala" {
                $Body['format'] = "SOURCE"
                $Body['language'] = "SCALA"
            }

            ".r" {
                $Body['format'] = "SOURCE"
                $Body['language'] = "R"
            }

            ".sql" {
                $Body['format'] = "SOURCE"
                $Body['language'] = "SQL"
            }

            ".dbc" {
                $Body['format'] = "DBC"
            }

            ".ipynb" {
                $Body['format'] = "JUPYTER"
            }

            ".html" {
                $Body['format'] = "HTML"
            }

        }

        $BodyText = $Body | ConvertTo-Json -Depth 10

        if($null -eq $Body['format'])
        {
            Write-Warning "File $FileToPush has an unknown extension - skipping file"
        }
        else{
            Write-Verbose "Pushing file $FileToPush to $TargetPath"
            Invoke-RestMethod -Uri "$global:DatabricksURI/api/2.0/workspace/import" -Body $BodyText -Method 'POST' -Headers $Headers
        }
    }

    Pop-Location
}