Functions/Internal/Remove-JobNewClusterMetaDataAsPSObject.ps1

function Remove-JobNewClusterMetaDataAsPSObject {
    [CmdletBinding()]
    param ( 
        [Parameter(  
            Position = 0,   
            Mandatory = $true,   
            ValueFromPipeline = $true,  
            ValueFromPipelineByPropertyName = $true  
        )] [PSObject]$customObject
    );
    process{
    <# Allowed values for Job new_cluster #>
    <# See - https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/jobs#--newcluster #>
    $allowedValues = @('num_workers', 'autoscale', 'spark_version', 'spark_conf', 'node_type_id', 'driver_node_type_id',
        'custom_tags', 'cluster_log_conf', 'init_scripts', 'spark_env_vars', 'enable_elastic_disk', 'instance_pool_id')

    <# Remove any properties that are not defined in the allowedValues array #>
    foreach ($prop in $customObject.PSObject.Properties) {
        if ($prop.Name -notin $allowedValues) {
          
            $customObject.PSObject.Properties.Remove($prop.Name)     
        }
    }

    return $customObject
}
}