public/Publish-AdfV2UsingArm.ps1

<#
.SYNOPSIS
Publishes Azure Data Factory from ARM Template files into target ADF service.

.DESCRIPTION
Publishes Azure Data Factory from ARM Template files into target ADF service.
Additionaly, creates a data factory with the specified resource group name and location, if that doesn't exist.
Uses standard New-AzResourceGroupDeployment method in order to create new deployment in a given Resource Group.

.PARAMETER TemplateFile
Path to ARM template file

.PARAMETER TemplateParameterFile
Path to ARM template parameter file

.PARAMETER ResourceGroupName
Resource Group Name of target instance of ADF

.PARAMETER DataFactoryName
Name of target ADF instance

.PARAMETER Location
Azure Region for target ADF. Used only for create new ADF instance.

.PARAMETER Option
This objects allows to define certain behaviour of deployment process. Use cmdlet "New-AdfPublishOption" to create new instance of the object and set required properties.

.PARAMETER WhatIf


.EXAMPLE
$RootFolder = '.\BigFactorySample2'
$ArmFile = "$RootFolder\ArmTemplate\ARMTemplateForFactory.json"
$ArmParamFile = "$RootFolder\ArmTemplate\ARMTemplateParametersForFactory.json"
$rg = 'rg-devops-factory'
$location = 'northeurope'
$DataFactoryName = 'BigFactorySample2-test'
$o = New-AdfPublishOption
$o.StopStartTriggers = $false
Publish-AdfV2UsingArm -TemplateFile $ArmFile -TemplateParameterFile $ArmParamFile `
    -ResourceGroupName $rg -Location $location -Option $o `
    -DataFactory $DataFactoryName

.LINK
Online version: https://github.com/SQLPlayer/azure.datafactory.tools/
#>


function Publish-AdfV2UsingArm {
    [CmdletBinding()]
    param
    (
        [parameter(Mandatory = $true)] 
        [String] $TemplateFile,

        [parameter(Mandatory = $true)] 
        [String] $TemplateParameterFile,
        
        [parameter(Mandatory = $true)] 
        [String] $ResourceGroupName,

        [parameter(Mandatory = $true)] 
        [String] $DataFactoryName,

        [parameter(Mandatory = $false)] 
        [String] $Stage = $null,
        
        [parameter(Mandatory = $false)] 
        [String] $Location,

        [parameter(Mandatory = $false)] 
        [AdfPublishOption] $Option,

        [switch] $WhatIf = $false
    )

    $m = Get-Module -Name "azure.datafactory.tools"
    $verStr = $m.Version.ToString(2) + "." + $m.Version.Build.ToString("000");
    Write-Host "======================================================================================";
    Write-Host "### azure.datafactory.tools Version $verStr ###";
    Write-Host "======================================================================================";
    Write-Host "Invoking Publish-AdfV2UsingArm (https://github.com/SQLPlayer/azure.datafactory.tools)";
    Write-Host "with the following parameters:";
    Write-Host "======================================================================================";
    Write-Host "TemplateFile: $TemplateFile";
    Write-Host "TemplateParameterFile: $TemplateParameterFile";
    Write-Host "ResourceGroupName: $ResourceGroupName";
    Write-Host "DataFactoryName: $DataFactoryName";
    Write-Host "Location: $Location";
    #Write-Host "Stage: $Stage";
    Write-Host "Options provided: $($null -ne $Option)";
    Write-Host "WhatIfPreference: $WhatIfPreference";
    Write-Host "======================================================================================";

    $script:StartTime = Get-Date

    if ($null -ne $Option) {
        Write-Host "Publish options are provided."
        $opt = $Option
    }
    else {
        Write-Host "Publish options are not provided."
        $opt = New-AdfPublishOption
    }

    if (!$WhatIfPreference) {
        Write-Host "STEP: Verifying whether ADF exists..."

        $targetAdf = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" -ErrorAction:Ignore
        if ($targetAdf) {
            Write-Host "Azure Data Factory exists."
        }
        else {
            $msg = "Azure Data Factory instance does not exist."
            if ($opt.CreateNewInstance) {
                Write-Host "$msg"
                Write-Host "Creating a new instance of Azure Data Factory..."
                $targetAdf = Set-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" -Location "$Location"
                $targetAdf | Format-List | Out-String
            }
            else {
                Write-Host "Creation operation skipped as publish option 'CreateNewInstance' = false"
                Write-Error "ADFT0027: $msg"
            }
        }

        if ($null -eq $targetAdf) {
            Write-Host "The process is exiting the function. Do fix the issue and run again."
            return 
        }
    }
    else {
        Write-Host "DRY RUN: Skipping ADF existence verification..."
    }
    
    Write-Host "===================================================================================";
    Write-Host "STEP: Reading Azure Data Factory from ARM Template files..."
    $adf = New-Object -TypeName 'adf'
    $adf.Location = Split-Path $TemplateFile -Parent
    $arm = Get-Content -Path $TemplateFile -Raw | ConvertFrom-Json 
    $armParamBody = Get-Content -Path $TemplateParameterFile -Raw
    $armParam = $armParamBody | ConvertFrom-Json
    $adf.ArmTemplateJson = $arm

    $arm.resources | ForEach-Object {
        $ArmType = $_.type
        $ArmName = $_.name
        $o = New-Object -TypeName 'AdfObject'
        $o.name = $ArmName.ToString().Substring(37, $ArmName.ToString().Length - 40)
        $o.type = ConvertTo-AdfType $ArmType
        $o.Adf = $adf
        $o.Body = $_
        $adf.Pipelines.Add($o)
        $collectionName = $ArmType.Substring(32)
        Invoke-Expression "`$adf.$collectionName.Add(`$o)"
    }
    if ($armParam.parameters.factoryName.value -ne $DataFactoryName) {
        Write-Error "Given factory name does not match name in ARMTemplate parameter file. The deployment stopped."
        return
    }
    $adf.Name = $DataFactoryName
    $adf.ResourceGroupName = $ResourceGroupName
    $adf.Region = $location
    $folder = Split-Path -Path $TemplateFile -Parent

    # Apply Deployment Options if applicable
    if ($null -ne $Option) {
        #ApplyExclusionOptions -adf $adf
        Write-Warning 'Selective deployment is not supported for this method yet.'
    }

    Write-Host "===================================================================================";
    Write-Host "STEP: Replacing all properties environment-related..."
    #Write-Warning 'Update parameters is not supported for this method yet.'
    if (![string]::IsNullOrEmpty($Stage)) {
        Update-PropertiesFromFile -adf $adf -stage $Stage

        #TODO: Remove 'never used' parameters from new ARM Template

        # Create new ARM Template file
        $OldTemplateFile = $TemplateFile
        $TemplateFile = $OldTemplateFile + '.~.json'
        Write-Verbose "Writing new ARM Template file: $TemplateFile"
        $output = ($adf.ArmTemplateJson | ConvertTo-Json -Compress:$true -Depth 100)
        $Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False
        [IO.File]::WriteAllLines($TemplateFile, $output, $Utf8NoBomEncoding)
    } else {
        Write-Host "Stage parameter was not provided - action skipped."
    }

    Write-Host "===================================================================================";
    Write-Host "STEP: Microsoft PreDeployment script..."
    if ($opt.StopStartTriggers -eq $true -and !$WhatIf) {
        #Stop-Triggers -adf $adf
        $script = Join-Path -Path $folder -ChildPath 'PrePostDeploymentScript.ps1'
        Write-Verbose "Running script: $script"
        . $script -armTemplate $TemplateFile -resourceGroupName $ResourceGroupName -dataFactoryName $DataFactoryName -predeployment $true
    } else {
        Write-Host "Operation skipped as WhatIf is enabled or publish option 'StopStartTriggers' = false"
    }


    Write-Host "===================================================================================";
    Write-Host "STEP: Deployment of ARM Template of ADF..."
    #$armParam = Get-Content -Path $TemplateParameterFile | ConvertFrom-Json #-AsHashtable
    #$t = (Get-Date).TOString('ddMM-HHmm')
    $DeploymentName = "ADFtools.deploy.$DataFactoryName"
    Write-Host " Deployment name: $DeploymentName"

    if (!$WhatIf) {
        New-AzResourceGroupDeployment -Name "$DeploymentName" -Mode 'Incremental' `
        -ResourceGroupName $ResourceGroupName `
        -TemplateFile $TemplateFile `
        -TemplateParameterFile $TemplateParameterFile
    }

    Write-Host "===================================================================================";
    Write-Host "STEP: Deployment of Global Parameters..."
    if ($opt.DeployGlobalParams -eq $true -and !$WhatIf) {
        Write-Verbose "Source folder for global params and script: $folder"
        #$script = Join-Path -Path $folder -ChildPath 'GlobalParametersUpdateScript.ps1'
        $globjson = (Get-ChildItem -Path $folder -Filter '*_GlobalParameters.json')[0]
        Write-Verbose "Global Param file: $($globjson.FullName)"
        #. $script -globalParametersFilePath $globjson.FullName -resourceGroupName $ResourceGroupName -dataFactoryName $DataFactoryName
        Microsoft-GlobalParametersUpdateScript -globalParametersFilePath $globjson.FullName -resourceGroupName $ResourceGroupName -dataFactoryName $DataFactoryName
    } else {
        Write-Host "Deployment of Global Parameters will be skipped as WhatIf is enabled or publish option 'DeployGlobalParams' = false"
    }

    Write-Host "===================================================================================";
    Write-Host "STEP: Microsoft PostDeployment script:"
    Write-Host " Deleting objects not in source = $($opt.DeleteNotInSource)"
    Write-Host " Starting triggers = $($opt.StopStartTriggers)"
    if (($opt.StopStartTriggers -eq $true -or $opt.DeleteNotInSource -eq $true) -and !$WhatIf) {
        #Start-Triggers -adf $adf
        #$script = Join-Path -Path $folder -ChildPath 'PrePostDeploymentScript.ps1'
        #Write-Verbose "Running script: $script"
        #. $script -armTemplate $TemplateFile -resourceGroupName $ResourceGroupName -dataFactoryName $DataFactoryName -predeployment $false
        Microsoft-PrePostDeploymentScript -armTemplate $TemplateFile `
            -resourceGroupName $ResourceGroupName `
            -dataFactoryName $DataFactoryName `
            -predeployment $false `
            -DeleteNotInSource $opt.DeleteNotInSource `
            -StartActiveTriggers $opt.StopStartTriggers
    } else {
        Write-Host "Operation skipped as WhatIf is enabled or publish option 'StopStartTriggers' or 'DeleteNotInSource' = false"
    }
    
    $elapsedTime = new-timespan $script:StartTime $(get-date)
    Write-Host "==============================================================================";
    Write-Host " ***** Azure Data Factory files have been deployed successfully. *****`n";
    Write-Host "Data Factory name: $DataFactoryName";
    Write-Host "Region (Location): $location";
    Write-Host ([string]::Format(" Elapsed time: {0:d1}:{1:d2}:{2:d2}.{3:d3}`n", $elapsedTime.Hours, $elapsedTime.Minutes, $elapsedTime.Seconds, $elapsedTime.Milliseconds))
    Write-Host "==============================================================================";

    return $adf
}