modules/Downloader/RedashMultipart/RedashQueryApi.psm1

function Get-RedashContent(){
    [CmdletBinding()]
    param(
        [string] $domain, [int] $queryId, [hashtable] $filterParams, [string] $fileExtension, [hashtable] $headers,
        [string] $outFile, [bool] $oldest, [bool] $descending, [int] $limitRows
    )

    #region Internal functions
    function Download([string] $domain, [string] $queryId, [hashtable] $filterParams, [string] $fileExtension, [hashtable] $authHeaders, [int] $chunk, [string] $source){
        $urlInit = '{0}/api/queries/{1}/results'
        $urlJob = '{0}/api/jobs/{1}'
        $urlResult = '{0}/api/query_results/{1}.{2}'

        if ($chunk -eq 1) { Write-Host ("Download {0} audit log..." -f $source) }

        # Initialize the download
        try {
            $urlInit = ($urlInit -f @($domain, $queryId))
            $json = (Invoke-RestMethod -Method POST -Uri $urlInit -Headers $authHeaders -Body (ConvertTo-Json $filterParams))
        }
        catch {
            throw ('Web request to ''{0}'' has failed! (Error: {1})' -f @($urlInit, $_))
        }

        # if there is job executed behind on Redash
        if ($json.job){
            $jobId = $json.job.id
            $urlJob = ($urlJob -f @($domain, $jobId))

            Write-Host ('Waiting for redash {0}' -f $jobId) -NoNewline

            while ($json.job.status -lt 3) {
                Start-Sleep -Seconds 1
                Write-Host '.' -NoNewline

                try {
                    $json = Invoke-RestMethod -Method GET -Uri $urlJob -Headers $authHeaders
                }
                catch {
                    Write-Host ''
                    throw ('Problems during the redash request: {0}' -f $_)
                }
            }
            if ($json.job.status -gt 3){
                Write-Host ''
                throw ('Redash request failed! {0}' -f $json.job.message)
            }

            $urlResult = ($urlResult -f @($domain, $json.job.query_result_id, $fileExtension))
        }
        else {
            Write-Host ('Download from redash cache {0}' -f $json.query_result.id) -NoNewline
            $urlResult = ($urlResult -f @($domain, $json.query_result.id, $fileExtension))
        }

        # Job is successfully executed (status: 3)
        $content = $null
        Write-Host ("[{0}]" -f @($chunk)) -NoNewline

        try {
            $content = (Invoke-WebRequest -Method GET -Uri $urlResult -Headers $authHeaders -TimeoutSec 300)
        }
        catch {
            Write-Host ''
            throw ('Error while downloading the logs, message: {0}' -f $_.Exception.Message)
        }

        Write-Host ' :: Downloaded!'

        return $content
    }

    function Merge-JsonContent([PSCustomObject] $downloaded, [PSCustomObject] $new, [bool] $oldest, [bool] $descending){

        if ($descending) { $new.query_result.data.rows = ($new.query_result.data.rows | Sort-Object { [int]$_.id }) }

        if (-not $downloaded) {
            $downloaded = $new
        }
        else {
            if ($oldest) { $downloaded.query_result.data.rows += $new.query_result.data.rows }
            else { $downloaded.query_result.data.rows = $new.query_result.data.rows + $downloaded.query_result.data.rows }
        }

        return $downloaded
    }

    function Merge-CsvContent([string[]] $downloaded, [string[]] $new, [bool] $oldest, [bool] $descending){

        # add the header line
        if (-not $downloaded) { $downloaded += $new[0] }
        # exclude the header for all others
        $new = $new[1..($new.Count-1)]

        if ($descending) { [array]::Reverse($new) }

        if ($oldest) { $downloaded += $new }
        elseif ($this.limitRange -eq $this.RANGE_NEWEST) { $downloaded = $new + $downloaded }

        return $downloaded
    }

    function NextDates([object] $content, [datetime] $from, [datetime] $to, [string] $fileExtension, [bool] $oldest){

        $bottom = $null
        $top = $null
        switch ($fileExtension) {
            'csv' {
                if (-not $content[1]) {
                    return $from, $from
                }

                $top = [datetime]($content[1].Split(',')[4])
                $last = $content.Count - 1
                while (-not $content[$last]) { $last-- }

                $bottom = [datetime]($content[$last].Split(',')[4])
            }
            'json' {
                if ($content.query_result.data.rows.Count -eq 0) {
                    return $from, $from
                }
                $bottom = $content.query_result.data.rows[0].created_at
                $last = $content.query_result.data.rows.Count - 1
                while (-not $content.query_result.data.rows[$last].created_at) {
                    $last--
                }
                $top = $content.query_result.data.rows[$last].created_at
            }
        }

        if ($oldest){
            if ($bottom -gt $top ){ $from = $bottom }
            else { $from = $top}
        }
        else {
            if ($bottom -gt $top ){ $to = $top }
            else { $to = $bottom }
        }

        return @($from, $to)
    }

    function NextMessageId([object] $content, [long] $messageId, [string] $fileExtension, [bool] $oldest){

        Write-Debug ("fileExtension: {0}, oldest: {1}" -f @($fileExtension, $oldest))

        $retValue = $messageId
        $bottom = $null; $top = $null
        switch ($fileExtension) {
            'csv' {
                if (-not $content[1]) {
                    return $messageId
                }

                $top = [long]($content[1].Split(',')[0])
                $last = $content.Count - 1
                while (-not $content[$last]) {
                    $last--
                }
                $bottom = [long]($content[$last].Split(',')[0])
            }
            'json' {
                if ($content.query_result.data.rows.Count -eq 0) {
                    return $messageId
                }
                $bottom = $content.query_result.data.rows[0].id
                $last = $content.query_result.data.rows.Count - 1
                while (-not $content.query_result.data.rows[$last].id) {
                    $last--
                }
                $top = $content.query_result.data.rows[$last].id
            }
        }

        Write-Debug ("Bottom: {0}, top: {1}" -f @($bottom, $top))
        if ($oldest){
            if ($bottom -gt $top ){ $retValue = $bottom }
            else { $retValue = $top}
        }
        else {
            if ($bottom -gt $top ){ $retValue = $top }
            else { $retValue = $bottom }
        }

        return $retValue
    }

    #endregion

    $temp = $outFile.split('-')
    $source = $temp[$temp.Count-1].Split('.')[0]

    $downloadContent = $null
    $messageId = 0

    $count = 1; $shouldBreak = $false
    while (-not $shouldBreak) {
        Write-Debug "Filter message id: $(Convertto-Json $filterParams)"
        $content = (Download -domain $domain -queryId $queryId $filterParams $filterParams -fileExtension $fileExtension -authHeaders $headers `
            -chunk $count -source $source)

        Write-Debug "Limit rows: $limitRows"
        switch ($fileExtension) {
            'csv' {
                $content = ($content -split '\r?\n')
                Write-Debug ("Total rows (csv): {0}" -f ($content.Count-1))
                if (($content.Count-1) -lt $limitRows){ $shouldBreak = $true; Write-Debug "Should stop (csv)" }
                $downloadContent = (Merge-CsvContent $downloadContent $content $oldest $descending)
            }
            'json' {
                $content = ($content | ConvertFrom-Json)
                Write-Debug ("Total rows (json): {0}" -f ($content.query_result.data.rows.Count))
                if (($content.query_result.data.rows.Count) -lt $limitRows) { $shouldBreak = $true; Write-Debug "Should stop (json)" }
                $downloadContent = (Merge-JsonContent $downloadContent $content $oldest $descending)
            }
        }
        $count++

        $messageId = (NextMessageId $content $messageId $fileExtension $oldest)
        $filterParams.parameters.messageId = ([string]$messageId)

        #Write-Debug "Filter message id: $($filterParams.messageId)"
    }

    switch ($fileExtension) {
        'csv' { $downloadContent | Out-File -Path $outFile }
        'json' { $downloadContent.query_result.data.rows | ConvertTo-Json -Depth 100 | Out-File -Path $outFile }
    }
}