Elastic.Helper

0.0.11

Elastic.Helper.psm1

# Convert to Newline Delimited JSON
function ConvertTo-NdJson {
  <#
  .SYNOPSIS
    Convert JSON/PSObject to Newline Delimited JSON (NDJSON) for bulk imports to ES
  .DESCRIPTION
    This function is intended to reformat a normal JSON array as a series of NDJSON lines. Will also work in a Pipeline
  .PARAMETER Input
    Input Array of PSObject
  #>

  [cmdletbinding()]
  param (
      # Input Data as PSObject to be converted to Newline Delimited JSON Entries
      [parameter(Mandatory = $true,
        ValueFromPipeline = $true)]
      $Input
  )

  Process {
    $Results = @()

    foreach ($item in $Input) {

      $Results += $item | ConvertTo-Json -Compress -Depth 8

    }

    Write-Output $Results
  }
}

Function Test-Guid {
    <#
    .SYNOPSIS
        Determine if a given string is a valid Guid.
    .DESCRIPTION
        Check for a valid Guid
    .PARAMETER Guid
        The string to be tested.
    .INPUTS
        System.String - Guid to be tested
    .OUTPUTS
        System.Boolean
            True if string is a valid Guid
            False if string is not a valid Guid
    .EXAMPLE
        PS > "edea82e3-8d0b-4370-86f0-d96bcd4b6c19" | Test-Guid
        True
    .EXAMPLE
        PS > Test-Guid "edea82e3-8d0b-4370-86f0-d96bcd4b6c19"
        True
    .LINK
        https://github.com/LogRhythm-Tools/LogRhythm.Tools
    #>

    [CmdletBinding()]
    [OutputType([System.Boolean])]

    Param(
        [Parameter(Mandatory = $false, Position = 0, ValueFromPipeline = $true)]
        [string] $Guid
    )

    Process {
        if ([string]::IsNullOrEmpty($Guid)) {
            return $false
        }
        $ValidGuid = [guid]::Empty
        if (! ([guid]::TryParse($Guid, [ref]$ValidGuid))) {
            return $false
        }
        return $true
    }
}

# Import the Elastic Helper Configuration

function Get-EsHelperConfig {
    <#
    .SYNOPSIS
        Import an Elastic Helper configuration.
    .DESCRIPTION
        Import an Elastic Helper configuration JSON file from a stored location.
    .PARAMETER ConfigName
        BaseName of Configuration. To determine the actual filename, the '.json' extension is appended.
    .PARAMETER Path
        (Optional) Path to configuration directory.

        Defaults to .eshelper in the user's home directory.

        On Windows this will be determined by Group Policy, but defaults to C:\Users\<username>

        On Linux this will be determined by OS configuration, but is usually /home/<username>

        On Mac OS/X this is usually /Users/<username>
    .INPUTS
        System.String -> Configuration Name
    .OUTPUTS
        PSCustomObject (Hash) representing the configuration file contents
    .EXAMPLE
        Load Configuration file 'elasticproject.json' from default folder into the $EsConf variable

        PS C:\> $EsConf = Get-EsHelperConfig -ConfigName elasticproject
    .EXAMPLE
        Load Configuration file 'elasticproject.json' from '/opt/scripts/project/etc'

        PS C:\> $EsConf = Get-EsHelperConfig -ConfigName elasticproject -Path '/opt/scripts/project/etc'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>

    [CmdletBinding()]

    param (
      [string] [Parameter(Mandatory=$true)] $ConfigName,
      [string] [Parameter(Mandatory=$false)] $Path = [io.path]::Combine($HOME,".eshelper")
    )

    # Build the full path to the config file
    $ConfigFileName = "{0}.json" -f $ConfigName
    $ConfigFilePath = [io.path]::Combine($Path, $ConfigFileName)

    Get-Content -Path $ConfigFilePath | ConvertFrom-Json -depth 10
}

# Get Index Definition from config
<#
.SYNOPSIS
  Get the definition from the named index from the supplied configuration object
.DESCRIPTION
  Provide the desired definition from the configuration file for the specified index
.PARAMETER EsConfig
  PSObject containing the read in configuration
.PARAMETER IndexName
  Name of ElasticSearch Index to return definition for
#>
function Get-EsIndexDefinition {
  param (
    [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
    [string] [Parameter(Mandatory=$true)] $IndexName,
    [switch] [Parameter(Mandatory=$false)] $Exact
  )

  foreach ($Index in $EsConfig.Indices) {
    if ($Exact) {
      if ($IndexName -eq $Index.name) {
        return $Index
      }
    } else {
      if ($IndexName -match $Index.name) {
        return $Index
      }
    }
  }
}

# Test for Enrich Policy Dependencies
function Test-EsEnrichPolicyDepends {
    <#
    .SYNOPSIS
        Test if the Enrichment Policy's Dependencies are met.
    .DESCRIPTION
        Using the provided configuration definition, determine if it's dependencies are met in ElasticSearch.

        Will check all defined enrichment policies unless one is specified.

        Checks ElasticSearch to confirm that the index on which the policy is based exists.
    .PARAMETER EsConfig
        ElasticHelper configuration loaded using Get-EsHelperConfig.
    .PARAMETER PolicyName
        (Optional) Name of Enrichment Policy to check for unmet dependencies.

        If not specified, checks all defined Enrichment Policies.
    .PARAMETER EsCreds
        PSCredential object containing username and password to access ElasticSearch
    .INPUTS
        PSCustomObject -> Defined ElasticHelper Configuration
    .OUTPUTS
        Success (1) or Failure (0)
    .EXAMPLE
        Test all defined Enrichment Policies for Dependencies

        PS C:\> $Result = Test-EsEnrichmentPolicyDepends -EsConfig $EsConfig
    .EXAMPLE
        Test the MyEnrichmentPolicy Enrichment Policy for Dependencies

        PS C:\> $Result = Test-EsEnrichmentPolicyDepends -EsConfig $EsConfig -PolicyName 'MyEnrichmentPolicy'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>

  [CmdletBinding()]

  param(
    [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
    [string] [Parameter(Mandatory=$false)] $PolicyName,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
)

# Find the Enrichment Policy in the config

foreach ($Policy in $EsConfig._enrich.policies) {
    # Check all policies if none specified, otherwise check just the one
    if(-not ($PSBoundParameters.ContainsKey('PolicyName')) -or $Policy.name -eq $PolicyName) {
        $EsIndex = $Policy.definition.match.indices
        Write-Debug " Checking Index: $EsIndex"
        if ($EsCreds) {
          $EsIndexStatus = Get-EsIndex -ESUrl $EsConfig.eshome -EsIndex $EsIndex -EsCreds $EsCreds
        } else {
          $EsIndexStatus = Get-EsIndex -ESUrl $EsConfig.eshome -EsIndex $EsIndex
        }

        Write-Debug "Index Status: $EsIndexStatus"
        if($EsIndexStatus.Error) {
          if ($EsIndexStatus.status -eq 404) {
              $msg = "Unmet Dependency: Index {0} not found" -f $EsIndex
          } else {
              $msg = "Error: {0}" -f $esIndexStatus.Error
          }
          Write-Output $msg
        } elseif ($EsIndexStatus.$EsIndex) {
          Write-Debug " Index $EsIndex Present..."
          if ($EsCreds) {
            $RunningPolicy = Get-EsEnrichmentPolicy -ESUrl $EsConfig.eshome -Policy $Policy.name -EsCreds $EsCreds | ConvertFrom-Json -Depth 8
          } else {
            $RunningPolicy = Get-EsEnrichmentPolicy -ESUrl $EsConfig.eshome -Policy $Policy.name | ConvertFrom-Json -Depth 8
          }
          if ($RunningPolicy.policies.Count -eq 0) {
            $msg = "Unmet Dependency: Enrichment Policy {0} not loaded" -f $Policy.name
            Write-Error $msg
          } else {
            $msg = "Dependency Met: Enrichment Policy {0} loaded" -f $Policy.name
            Write-Debug $msg
          }
        }
    }
}
}

# Test Index dependencies
function Test-EsIndexDepends {
    <#
    .SYNOPSIS
        Test if the Index's Dependencies are met.
    .DESCRIPTION
        Using the provided configuration definition, determine if it's dependencies are met in ElasticSearch.

        Will check all defined enrichment indices unless one is specified.

        Checks ElasticSearch to confirm that the pipeline, and associated dependencies, configured for the index exists.
    .PARAMETER EsConfig
        ElasticHelper configuration loaded using Get-EsHelperConfig.
    .PARAMETER IndexName
        (Optional) Name of Index to check for unmet dependencies.

        If not specified, checks all defined Indices.
    .PARAMETER EsCreds
        PSCredential object containing username and password to access ElasticSearch
    .INPUTS
        PSCustomObject -> Defined ElasticHelper Configuration
    .OUTPUTS
        Success (1) or Failure (0)
    .EXAMPLE
        Test all defined Indices for Dependencies

        PS C:\> $Result = Test-EsIndexDepends -EsConfig $EsConfig
    .EXAMPLE
        Test the MyIndex Index for Dependencies

        PS C:\> $Result = Test-EsIndexDepends -EsConfig $EsConfig -IndexName 'MyIndex'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>

  [CmdletBinding()]
  [OutputType([Int32])]

  param (
    [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
    [string] [Parameter(Mandatory=$false)] $IndexName,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
)

foreach ($Index in $EsConfig.indices) {
    # Check all indices if no index name is specified, otherwise check just the one index
    if ( -not($PSBoundParameters.ContainsKey('IndexName')) -or $Index.name -eq $IndexName ) {
        if ($Index.pipeline) {
            $PipelineDepends = Test-EsPipelineDepends -EsConfig $EsConfig -PipelineName $Index.pipeline
            if ($null -eq $PipelineDepends) {
                # Dependencies matched
                if ($EsCreds) {
                  $EsIndexStatus = Get-EsIndex -ESUrl $EsConfig.eshome -EsIndex $Index.name -EsCreds $EsCreds
                } else {
                  $EsIndexStatus = Get-EsIndex -ESUrl $EsConfig.eshome -EsIndex $Index.name
                }

                if($EsIndexStatus.Error) {
                    if ($EsIndexStatus.status -eq 404) {
                        $msg = "Unmet Dependency: Index {0} not found" -f $Index.name
                    } else {
                        $msg = "Error: {0}" -f $EsIndexStatus.Error
                    }
                    Write-Output $msg
                } elseif ($EsIndexStatus.{$Index.name}) {
                    return 1
                }

            }
        }
    }
}
}

# Test Pipeline Dependencies
function Test-EsPipelineDepends {
    <#
    .SYNOPSIS
        Test if the Pipeline's Dependencies are met.
    .DESCRIPTION
        Using the provided configuration definition, determine if it's dependencies are met in ElasticSearch.

        Will check all defined pipelines unless one is specified.

        Checks ElasticSearch to confirm that the pipeline, and associated dependencies exist.
    .PARAMETER EsConfig
        ElasticHelper configuration loaded using Get-EsHelperConfig.
    .PARAMETER PipelineName
        (Optional) Name of Pipeline to check for unmet dependencies.

        If not specified, checks all defined Pipelines.
    .PARAMETER EsCreds
        PSCredential object containing username and password to access ElasticSearch
    .INPUTS
        PSCustomObject -> Defined ElasticHelper Configuration
    .OUTPUTS
        Success (1) or Failure (0)
    .EXAMPLE
        Test all defined Pipelines for Dependencies

        PS C:\> $Result = Test-EsPipelineDepends -EsConfig $EsConfig
    .EXAMPLE
        Test the MyPipeline Pipeline for Dependencies

        PS C:\> $Result = Test-EsPipelineDepends -EsConfig $EsConfig -PipelineName 'MyPipeline'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>

  [CmdletBinding()]

  param(
    [PsCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
    [string] [Parameter(Mandatory=$false)] $PipelineName,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
)

  # Find the pipeline in the config
foreach($Pipeline in $EsConfig._ingest.pipelines) {
    # Check all pipelines if none specified, otherwise check just the one
  if (-not ($PSBoundParameters.ContainsKey('PipelineName')) -or $Pipeline.name -eq $PipelineName) {
    foreach($PipelineProcessor in $Pipeline.definition.processors) {
      if ($PipelineProcessor.enrich) {
        $EnrichPolicy = $PipelineProcessor.enrich.policy_name
        if ($EsCreds) {
          $EnrichPolicyDepends = Test-EsEnrichPolicyDepends -EsConfig $EsConfig -PolicyName $EnrichPolicy -EsCreds $EsCreds
        } else {
          $EnrichPolicyDepends = Test-EsEnrichPolicyDepends -EsConfig $EsConfig -PolicyName $EnrichPolicy
        }
        if ($null -eq $EnrichPolicyDepends){
          # Dependencies matched
          if ($EsCreds) {
            $PipelineStatus = Get-EsPipeline -ESUrl $EsConfig.eshome -Pipeline $Pipeline.name -EsCreds $EsCreds|  ConvertFrom-Json -Depth 8
          } else {
            $PipelineStatus = Get-EsPipeline -ESUrl $EsConfig.eshome -Pipeline $Pipeline.name | ConvertFrom-Json -Depth 8
          }
          if($PipelineStatus.($Pipeline.name)) {
            # Pipeline is there
          } else {
            $msg = "Unmet Dependency: Pipeline {0} not loaded" -f $Pipeline.name
            Write-Output $msg
          }
        } else {
          $msg = "Unmet Dependencies for Pipeline: {0} - " -f $Pipeline.Name
          $msg += $EnrichPolicyDepends
          Write-Output $msg
        }
      }
    }
  }
}
}

# Deploy ElasticSearch Configuration
# But only for resources that do not have dependencies, or where these dependencies are met
function Deploy-EsConfig {
    <#
    .SYNOPSIS
        Deploy specified ElasticSearch Resources to ElasticSearch cluster, with dependency checks.
    .DESCRIPTION
        Using the provided configuration definition, deploy the defined resources to ElasticSearch.

        Will check all dependencies are met before deploying to ElasticSeach, and will not attempt deployment of resources for which dependencies are not met.

        This may be used iteratively to defined the required resouces in ElasticSearch as data is populated.
    .PARAMETER EsConfig
        ElasticHelper configuration loaded using Get-EsHelperConfig.
    .PARAMETER ResourceType
        (Not Yet Implemented)
        (Optional) Type of Resources to deploy to ElasticSearch.

        One of: (index, pipeline, enrichmentpolicy)

        If not specified, attempts to deploy all resources without unmet dependencies.
    .PARAMETER ResourceName
        (Not Yet Implemented)
        (Optional) Name of Resource to deploy to ElasticSearch. If specified, must also specify resource type.

        If not specified, attempts to deploy all resources without unmet dependencies.
    .PARAMETER EsCreds
        PSCredential object containing username and password to access ElasticSearch
    .INPUTS
        PSCustomObject -> Defined ElasticHelper Configuration
    .OUTPUTS
        Success (1) or Failure (0)
    .EXAMPLE
        Deploy all defined Resources to ElasticSearch

        PS C:\> $Result = Deploy-EsConfig -EsConfig $EsConfig
    .EXAMPLE
        Load Configuration file 'elasticproject.json' from '/opt/scripts/project/etc'

        PS C:\> $EsConf = Get-EsHelperConfig -ConfigName elasticproject -Path '/opt/scripts/project/etc'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>

    [CmdletBinding()]

    param(
        [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
        [string] [Parameter(Mandatory=$false)] $ResourceType,
        [string] [Parameter(Mandatory=$false)] $ResourceName,
        [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
    )

    if ($PSBoundParameters.ContainsKey('ResourceType')) {
        if($PSBoundParameters.ContainsKey('ResourceName')) {
            # Process just the single Resource
        } else {
            # Can't have resource Type without Resource name
            Write-Error "Both ResourceType and ResourceName need to be specified"
        }
    } else {
        # In order, we need to do:
        # * Pipeline
        # * Index
        # * Enrichment Policy

        # Pipelines
        foreach ($Pipeline in $EsConfig._ingest.pipelines) {
            # Check dependencies
            if ($EsCreds) {
                $PipelineStatus = Test-EsPipelineDepends -EsConfig $EsConfig -PipelineName $Pipeline.name -EsCreds $EsCreds
            } else {
                $PipelineStatus = Test-EsPipelineDepends -EsConfig $EsConfig -PipelineName $Pipeline.name
            }
            if ($null -eq $PipelineStatus) {
                # Dependency check passed
                $msg = "Deploying Pipeline: {0}" -f $PipeLine.name
                Write-Output $msg
                if ($EsCreds) {
                    $result = Update-EsPipeline -ESUrl $EsConfig.eshome -Pipeline $Pipeline.name -PipelineDefinition ($PipeLine.definition | ConvertTo-Json -Depth 8) -EsCreds $EsCreds
                } else {
                    $result = Update-EsPipeline -ESUrl $EsConfig.eshome -Pipeline $Pipeline.name -PipelineDefinition ($PipeLine.definition | ConvertTo-Json -Depth 8)
                }
                if ($result) {
                    $msg = "Pipeline Deployed: {0}" -f $PipeLine.name
                } else {
                    $msg = "Pipeline Deployment Failed: {0} - {1}" -f $PipeLine.name, $result
                }
                Write-Output $msg
            } else {
                # Dependency check failed
                $msg = "Pipeline Deployment Skipped: {0} - Dependencies not met.`n {1}" -f $PipeLine.name, $PipelineStatus
                Write-Output $msg
            }
        }

        # Indices
        # We don't actually deploy indices, these are written by the data insert process

        # Enrichment Policies
        foreach ($Policy in $EsConfig._enrich.policies) {
            # Check Dependencies
            if ($EsCreds) {
                $PolicyStatus = Test-EsEnrichPolicyDepends -EsConfig $EsConfig -PolicyName $Policy.name -EsCreds $EsCreds
            } else {
                $PolicyStatus = Test-EsEnrichPolicyDepends -EsConfig $EsConfig -PolicyName $Policy.name
            }
            if ($null -eq $PolicyStatus) {
                # Dependency Check Passed
                $msg = "Deploying Enrichment Policy: {0}" -f $Policy.name
                Write-Output $msg
                if ($EsCreds) {
                    $result = Update-EsEnrichmentPolicy -ESUrl $EsConfig.eshome -Policy $Policy.name -PolicyDefinition ($Policy.definition | ConvertTo-Json -Depth 8) -EsCreds $EsCreds
                } else {
                    $result = Update-EsEnrichmentPolicy -ESUrl $EsConfig.eshome -Policy $Policy.name -PolicyDefinition ($Policy.definition | ConvertTo-Json -Depth 8)
                }
                if ($result) {
                    $msg = "Enrichment Policy Deployed: {0}" -f $Policy.name
                } else {
                    $msg = "Enrichment Policy Deployment Failed: {0} - {1}" -f $Policy.name, $result
                }
                Write-Output $msg
            } else {
                # Dependency Check failed
                $msg = "Enrichment Policy Deployment Skipped: {0} - Dependencies not met.`n {1}" -f $Policy.name, $PolicyStatus
                Write-Output $msg
            }
        }
    }
}

# Get the current enrichment policy definition
function Get-EsEnrichmentPolicy {
  <#
  .SYNOPSIS
      Get the currently configured Enrichment Policy configuration on the ElasticSearch server for the specified policy name
  .DESCRIPTION
      Get the configuration of the specified Enrichment Policy from the nomiated ElasticSearch server.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER Policy
      Name of ElasticSearch Enrichment Policy to get current configuration of.
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Definition of current Enrichment Policy on ElasticSearch Cluster
  .EXAMPLE
      Retrieve Enrichment Policy named MyEnrichmentPolicy without authentication

      PS C:\> $EnrichPol = Get-EsEnrichmentPolicy -EsUrl http://192.168.1.10:9200 -Policy 'MyEnrichmentPolicy'
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

  [CmdletBinding()]

  param (
    [string] [Parameter(Mandatory=$true)] $ESUrl,
    [string] [Parameter(Mandatory=$true)] $Policy,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
  )

  $Method = 'GET'
  $Uri = [io.path]::Combine($ESUrl, "_enrich/policy/", $Policy)

  if ($EsCreds){
    Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -User $EsCreds.Username -Password $EsCreds.Password -SkipCertificateCheck
  } else {
    Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -SkipCertificateCheck
  }

}

# Rebuild Enrichment Indices
function Update-EsEnrichmentIndices {
    <#
    .SYNOPSIS
        Rebuild all Enrichment indices associated with the specified Enrichment Policy
    .DESCRIPTION
        Each time the base/source index for an Enrichment Policy has documents added or updated, the system indices used to perform enrichment lookups need to be rebuilt.

        This operation triggers this task on the cluster.

        Optionally supports Authentication.
    .PARAMETER EsUrl
        Base URL for your ElasticSearch server/cluster
    .PARAMETER Policy
        Name of Enrichment Policy to rebuild enrichment indices for
    .PARAMETER EsCreds
        PSCredential object containing username and password to access ElasticSearch
    .INPUTS
        None
    .OUTPUTS
        Result of requested operation
    .EXAMPLE
        Rebuld enrichment indices associated to 'MyEnrichmentPolicy'

        PS C:\> $result = Update-EsEnrichmentIndices -EsUrl http://192.168.1.10:9200 -Policy 'MyEnrichmentPolicy'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>


  [CmdletBinding(SupportsShouldProcess)]

  param (
    [string] [Parameter(Mandatory=$true)] $ESUrl,
    [string] [Parameter(Mandatory=$true)] $Policy,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
)

$Method = 'POST'
$Uri = [io.path]::Combine($ESUrl, "_enrich/policy/", $Policy, "_execute")

if($PSCmdlet.ShouldProcess($Uri)) {
  if ($EsCreds) {
    $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -User $EsCreds.UserName -Password $EsCreds.Password  -SkipCertificateCheck| ConvertFrom-Json -depth 8
  } else {
    $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json'  -SkipCertificateCheck| ConvertFrom-Json -depth 8
  }
}

return $Result
}

# Execute Enrichment Policies tied to a specified index
function Update-EsEnrichmentIndicesFromIndex {
    <#
    .SYNOPSIS
        Test if the Index's Dependencies are met.
    .DESCRIPTION
        Each time the base/source index for an Enrichment Policy has documents added or updated, the system indices used to perform enrichment lookups need to be rebuilt.

        This operation triggers this task on the cluster, based on the index that is used in an enrichment policy.

        Optionally supports Authentication.
    .PARAMETER EsConfig
        ElasticHelper configuration loaded using Get-EsHelperConfig.
    .PARAMETER IndexName
        (Optional) Name of Index to check for unmet dependencies.

        If not specified, checks all defined Indices.
    .PARAMETER EsCreds
        PSCredential object containing username and password to access ElasticSearch
    .INPUTS
        PSCustomObject -> Defined ElasticHelper Configuration
    .OUTPUTS
        Success (1) or Failure (0)
    .EXAMPLE
        Rebuld enrichment indices associated to 'MyIndex'

        PS C:\> $result = Update-EsEnrichmentIndicesFromIndex -EsUrl http://192.168.1.10:9200 -IndexName 'MyIndex'
    .LINK
        https://github.com/jberkers42/Elastic.Helper
    #>

  [CmdletBinding(SupportsShouldProcess)]

  param(
    [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
    [string] [Parameter(Mandatory=$true)] $IndexName,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
)

# Look through the Enrich Policies in our configuration
foreach ($Policy in $EsConfig._enrich.policies) {
    # Find those that depend on our index
    if ($IndexName -match $Policy.definition.match.indices) {
        if ($PSCmdlet.ShouldProcess($Policy.name)) {
            # Update the Enrichment Index
            $msg = "Updating Enrichment Policy Index - Index: {0}; Policy: {1};" -f $IndexName, $Policy.name
            Write-Debug $msg
            if ($EsCreds) {
              $result = Update-EsEnrichmentIndices -ESUrl $EsConfig.eshome -Policy $Policy.name -EsCreds $EsCreds
            } else {
              $result = Update-EsEnrichmentIndices -ESUrl $EsConfig.eshome -Policy $Policy.name
            }
        }
        Write-Debug $result
    }
}
}

# Update the Enrichment Policy
function Update-EsEnrichmentPolicy {
  <#
  .SYNOPSIS
      Update the definition of the specified Enrichment Policy with the provided definition.
  .DESCRIPTION
      Use the supplied policy definition to update the running policy for the specified enrichment policy.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER Policy
      Name of ElasticSearch Enrichment Policy to get current configuration of.
  .PARAMETER PolicyDefinition
      PSCustomObject defining the desired state of the policy definition
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Definition of current Enrichment Policy on ElasticSearch Cluster
  .EXAMPLE
      Update the Enrichment Policy named MyEnrichmentPolicy without authentication

      PS C:\> $EnrichPol = Update-EsEnrichmentPolicy -EsUrl http://192.168.1.10:9200 -Policy 'MyEnrichmentPolicy' -PolicyDefinition $PolicyDef
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

  [CmdletBinding(SupportsShouldProcess)]

  param (
      [string] [Parameter(Mandatory=$true)] $ESUrl,
      [string] [Parameter(Mandatory=$true)] $Policy,
      [Parameter(Mandatory=$true)] $PolicyDefinition,
      [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
  )

  $Method = 'PUT'
  $Uri = [io.path]::Combine($ESUrl, "_enrich/policy/", $Policy)
  $EsBody = [Elastic.ElasticsearchRequestBody] $PolicyDefinition

  if($PSCmdlet.ShouldProcess($Uri)) {
    if ($EsCreds) {
      $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -Body $EsBody -User $EsCreds.UserName -Password $EsCreds.Password  -SkipCertificateCheck| ConvertFrom-Json -depth 8
    } else {
      $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -Body $EsBody  -SkipCertificateCheck| ConvertFrom-Json -depth 8
    }
  }

return $Result.acknowledged
}

# Convert to ElasticSearch Bulk Index request
function ConvertTo-EsBulkIndex {
  <#
  .SYNOPSIS
    Convert Array of JDJSON lines to interspersed Index requests followed by data to be indexed
  .DESCRIPTION
    This function is intended to turn a series of NDJSON lines into an ES Bulk Index request. Will also work in a Pipeline
  .PARAMETER Input
    Input Array of PSObject
  .PARAMETER Index
    Name of index, may be overridden by Pipeline configuration
  .PARAMETER Pipeline
    Name of Ingest Pipeline
  #>
    [OutputType([string])]
    [cmdletbinding()]
    param (
        # Input Data as Array of entries to be converted to Newline Delimited JSON Entries with specific base index
        [parameter(Mandatory = $true, ValueFromPipeline = $true)]  $Input,
        [parameter(Mandatory = $true)] $Index,
        [parameter(Mandatory = $true)] $Pipeline
    )

    Process {
        $Results = @()

        $Results += '{"index" : { "_index": "' + $Index + '", "pipeline": "' + $Pipeline + '"}}'
        $Results += ($Input | ConvertTo-Json -Compress -depth 8) -replace [char] 0x00a0,'-'

        Write-Output $Results

    }
    End {
      # Ensure newline at the end of the request
      Write-Output ''
    }

}

# Get index info

function Get-EsIndex {
  <#
  .SYNOPSIS
      Get the currently configured Index configuration on the ElasticSearch server for the specified index name
  .DESCRIPTION
      Get the configuration of the specified Index from the nomiated ElasticSearch server.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER EsIndex
      Name of ElasticSearch Index to get current information and configuration of.
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Information about the specified index on ElasticSearch Cluster
  .EXAMPLE
      Retrieve status and configuration about an index named MyIndex without authentication

      PS C:\> $EsIndex = Get-EsIndex -EsUrl http://192.168.1.10:9200 -EsIndex 'MyIndex'
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

  [CmdletBinding()]

  param(
    [string] [Parameter(Mandatory=$true)] $ESUrl,
    [string] [Parameter(Mandatory=$true)] $EsIndex,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
  )

  $Method = 'GET'
  $Uri = [io.path]::Combine($ESUrl, $EsIndex, "_settings")

  if ($EsCreds) {
    Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -User $EsCreds.UserName -Password $EsCreds.Password  -SkipCertificateCheck| ConvertFrom-Json -Depth 8
  } else {
    Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -SkipCertificateCheck | ConvertFrom-Json -Depth 8
  }
}

function Get-EsIndexSettings {
  <#
  .SYNOPSIS
      Get the currently configured Index configuration on the ElasticSearch server for the specified index name
  .DESCRIPTION
      Get the configuration of the specified Index from the nomiated ElasticSearch server.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER IndexName
      Name of ElasticSearch Index to get current information and configuration of.
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Information about the specified index on ElasticSearch Cluster
  .EXAMPLE
      Retrieve status and configuration about an index named MyIndex without authentication

      PS C:\> $EsIndex = Get-EsIndex -EsUrl http://192.168.1.10:9200 -IndexName 'MyIndex'
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

    [CmdletBinding()]

    param (
        [string] [Parameter(Mandatory=$true)] $ESUrl,
        [string] [Parameter(Mandatory=$true)] $IndexName,
        [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
    )

    $Method = 'GET'
    $Uri = [io.path]::Combine($ESUrl, $IndexName, "_settings")
    if ($EsCreds) {
      Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -User $EsCreds.UserName -Password $EsCreds.Password -SkipCertificateCheck
    } else {
      Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -SkipCertificateCheck
    }
}

# Bulk Index logs
# Index some records
function Invoke-EsBulkIndexRequest {
  <#
  .SYNOPSIS
    Bulk Index an array of PSObjects to the specified ElasticSearch instance and index
  .DESCRIPTION
    Takes an array of PSObjects and submits it to the ElasticSearch instance as a bulk index request denoted by the EsConfig object and index name
  .PARAMETER EsConfig
    PSObject containing the read in configuration
  .PARAMETER IndexName
    Name of ElasticSearch Index to return definition for
  .PARAMETER InputArray
    Array of PSObjects to sumbit to the index
  #>
  param (
    [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
    [string] [Parameter(Mandatory=$true)] $IndexName,
    [Parameter(Mandatory=$true)] $InputObject,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
  )

  Write-Debug "Processing Bulk Index Request..."

  $BulkIndexURI = '{0}/_bulk' -f $EsConfig.eshome, $IndexName

  Write-Debug " URI: $BulkIndexURI"

  $ndjson = $InputObject | ConvertTo-EsBulkIndex -Index $IndexName -Pipeline (Get-EsIndexDefinition -EsConfig $EsConfig -IndexName $IndexName -Exact).pipeline

  if ($EsCreds) {
    Write-Debug " Credentials Supplied"
    $ndjson -join "`n" | Invoke-Elasticsearch -Uri $BulkIndexURI -Method POST -ContentType 'application/x-ndjson' -User $EsCreds.UserName -Password $EsCreds.Password -SkipCertificateCheck
  } else {
    Write-Debug " No Credentials Supplied"
    $ndjson -join "`n" | Invoke-Elasticsearch -Uri $BulkIndexURI -Method POST -ContentType 'application/x-ndjson' -SkipCertificateCheck
  }

  Start-Sleep -Seconds 2

  # After indexing a bunch of records, update the enrichment indices (if any) associated to the index
  Write-Debug " Updating Enrichment Indices"
  if ($EsCreds) {
    Update-EsEnrichmentIndicesFromIndex -EsConfig $EsConfig -IndexName $IndexName -EsCreds $EsCreds
  } else {
    Update-EsEnrichmentIndicesFromIndex -EsConfig $EsConfig -IndexName $IndexName
  }

}

function Update-EsIndexSettings {
  <#
  .SYNOPSIS
      Update the settings for the specified index with the provided definition.
  .DESCRIPTION
      Use the supplied index definition to update the existing settings of the specified index.

      If additional settings are present on the ElasticSearch index, the new settings will merge with and override existing settings. Other settings will remain unchanged.

      This allows you to update just the 'number_of_replicas' setting without affecting any other settings.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER IndexName
      Name of ElasticSearch Index to update current configuration of.
  .PARAMETER IndexDefinition
      PSCustomObject defining the desired state of the index configuration
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Result of operation
  .EXAMPLE
      Update the index named MyIndex without authentication

      PS C:\> $IndexDef = @{'index' = @{ 'number_of_replicas' = '0'} }
      PS C:\> $EnrichPol = Update-EsIndexSettings -EsUrl http://192.168.1.10:9200 -IndexName 'MyIndex' -IndexDefinition $IndexDef
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

    [CmdletBinding(SupportsShouldProcess)]

    param (
        [string] [Parameter(Mandatory=$true)] $ESUrl,
        [string] [Parameter(Mandatory=$true)] $IndexName,
        [string] [Parameter(Mandatory=$true)] $IndexDefinition,
        [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
    )

    $Method = 'PUT'
    $Uri = [io.path]::Combine($ESUrl, $IndexName, "_settings")
    $EsBody = [Elastic.ElasticsearchRequestBody] $IndexDefinition

    if($PSCmdlet.ShouldProcess($Uri)) {
      if ($EsCreds) {
        $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -Body $EsBody -User $EsCreds.UserName -Password $EsCreds.Password  -SkipCertificateCheck| ConvertFrom-Json -depth 8
      } else {
        $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -Body $EsBody  -SkipCertificateCheck| ConvertFrom-Json -depth 8
      }
    }

    return $Result.acknowledged

}

function Update-EsIndexSettingsFromConfig {
  <#
  .SYNOPSIS
      Update the ElasticSearch index settings with values from the config file
  .DESCRIPTION
      Use the supplied configuration object to update the ElasticSearch running environment.

      This will recursively identify required objects to create, such as Pipeline, Enrichment Policy, etc

      Optionally supports Authentication.
  .PARAMETER EsConfig
      PSCustomObject containing configuration data loaded with Get-EsHelperConfig
  .PARAMETER IndexName
      Name of ElasticSearch Index to update current configuration of.
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Result of operation
  .EXAMPLE
      Update the index named MyIndex without authentication

      PS C:\> $EsConf = Get-EsHelperConfig -ConfigName 'esproject'
      PS C:\> $result = Update-EsIndexSettingsFromConfig -EsConf $EsConf -IndexName 'MyIndex'
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

    [CmdletBinding(SupportsShouldProcess)]

    param (
        [PSCustomObject] [Parameter(Mandatory=$true)] $EsConfig,
        [string] [Parameter(Mandatory=$true)] $IndexName,
        [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
    )

    foreach ($Index in $EsConfig.Indices) {
        if ($IndexName -match $Index.name) {
            If($PSCmdlet.ShouldProcess($IndexName)) {
              If ($EsCreds){
                Update-EsIndexSettings -ESUrl $EsConfig.eshome -IndexName $IndexName -IndexDefinition ($Index.settings | ConvertTo-Json -Depth 8) -EsCreds $EsCreds
              } else {
                Update-EsIndexSettings -ESUrl $EsConfig.eshome -IndexName $IndexName -IndexDefinition ($Index.settings | ConvertTo-Json -Depth 8)
              }
            }
        }
    }

}

# Get the current pipeline definition
function Get-EsPipeline {
  <#
  .SYNOPSIS
      Get the currently configured Pipeline configuration on the ElasticSearch server for the specified index name
  .DESCRIPTION
      Get the configuration of the specified Pipeline from the nomiated ElasticSearch server.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER Pipeline
      Name of ElasticSearch Pipeline to get current configuration of.
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Configuration of specified pipeline on ElasticSearch Cluster
  .EXAMPLE
      Retrieve status and configuration about an pipeline named MyPipeline without authentication

      PS C:\> $EsPipeline = Get-EsPipeline -EsUrl http://192.168.1.10:9200 -Pipeline 'MyPipeline'
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

  [CmdletBinding()]

  param (
    [string] [Parameter(Mandatory=$true)] $ESUrl,
    [string] [Parameter(Mandatory=$true)] $Pipeline,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
  )

  $Method = 'GET'
  $Uri = [io.path]::Combine($ESUrl, "_ingest/pipeline/", $Pipeline)
  if ($EsCreds) {
    Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -User $EsCreds.Username -Password $EsCreds.Password -SkipCertificateCheck
  } else {
    Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -SkipCertificateCheck
}
}

# Update the pipeline definition with a new one
function Update-EsPipeline {
  <#
  .SYNOPSIS
      Update the settings for the specified Pipeline with the provided definition.
  .DESCRIPTION
      Use the supplied Pipeline definition to update the existing settings of the specified Pipeline.

      If additional settings are present on the ElasticSearch Pipeline, the new settings will merge with and override existing settings. Other settings will remain unchanged.

      This allows you to update just the 'number_of_replicas' setting without affecting any other settings.

      Optionally supports Authentication.
  .PARAMETER EsUrl
      Base URL for your ElasticSearch server/cluster
  .PARAMETER Pipeline
      Name of ElasticSearch Pipeline to update current configuration of.
  .PARAMETER PipelineDefinition
      PSCustomObject defining the desired state of the Pipeline configuration
  .PARAMETER EsCreds
      PSCredential object containing username and password to access ElasticSearch
  .INPUTS
      None
  .OUTPUTS
      Result of operation
  .EXAMPLE
      Update the Pipeline named MyPipeline without authentication

      PS C:\> $EnrichPol = Update-EsPipelineSettings -EsUrl http://192.168.1.10:9200 -Pipeline 'MyPipeline' -PipelineDefinition $PipelineDef
  .LINK
      https://github.com/jberkers42/Elastic.Helper
  #>

  [CmdletBinding(SupportsShouldProcess)]

  param (
    [string] [Parameter(Mandatory=$true)] $ESUrl,
    [string] [Parameter(Mandatory=$true)] $Pipeline,
    [Parameter(Mandatory=$true)] $PipelineDefinition,
    [PSCustomObject] [Parameter(Mandatory=$false)] $EsCreds
  )

  $Method = 'PUT'
  $Uri = [io.path]::Combine($ESUrl, "_ingest/pipeline/", $Pipeline)
  $EsBody = [Elastic.ElasticsearchRequestBody] $PipelineDefinition

  if($PSCmdlet.ShouldProcess($Uri)) {
    if ($EsCreds) {
      $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -Body $EsBody -User $EsCreds.UserName -Password $EsCreds.Password  -SkipCertificateCheck| ConvertFrom-Json -depth 8
    } else {
      $Result = Invoke-Elasticsearch -Uri $Uri -Method $Method -ContentType 'application/json' -Body $EsBody  -SkipCertificateCheck| ConvertFrom-Json -depth 8
    }
  }

  if ($Result.acknowledged) {
    # Successfully deployed Pipeline Config
    return $Result.acknowledged
  } else {
    return $Result
  }
}

Export-ModuleMember -Function Get-EsHelperConfig, Get-EsIndexDefinition, Test-EsEnrichPolicyDepends, Test-EsIndexDepends, Test-EsPipelineDepends, Deploy-EsConfig, Get-EsEnrichmentPolicy, Update-EsEnrichmentIndices, Update-EsEnrichmentIndicesFromIndex, Update-EsEnrichmentPolicy, ConvertTo-EsBulkIndex, Get-EsIndex, Get-EsIndexSettings, Invoke-EsBulkIndexRequest, Update-EsIndexSettings, Update-EsIndexSettingsFromConfig, Get-EsPipeline, Update-EsPipeline