Private/backup/Invoke-FilesBackup.ps1

function Invoke-FilesBackup {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true, Position = 0 )]
        [string]$Token,
        [Parameter(Mandatory = $true, Position = 1 )]
        [string]$Location
    )
    process {
        Write-Verbose "Cmdlet Invoke-FilesBackup - Process"

        $backupLoc = "$Location/files"
        Assert-Path $backupLoc

        $start = Get-ChildItem -Path $backupLoc -Filter *.json |`
            % { $_ | Get-Content -Encoding UTF8 | ConvertFrom-Json } |`
            Select-Object -ExpandProperty created |`
            Measure-Object -Maximum |`
            Select-Object -ExpandProperty Maximum |`
            % { $_ + 1 }

        if (!$start) {
            $start = Convert-DateToEpoch (Get-Date "2000-01-01")
        }

        # Collect every page: files.list returns newest first, one page at a time.
        $files = @()
        $page = 1
        do {
            $response = Get-FilesList -Token $Token -TsFrom $start -Page $page
            if (!$response.ok) {
                break
            }
            $files += $response.files
            $pages = if ($response.paging.pages) { [int]$response.paging.pages } else { 1 }
            $page++
        } while ($page -le $pages)

        # Process oldest first. Resuming relies on max(created) of the files already saved,
        # so a run that stops early (time limit, crash) must never leave older files behind.
        $files = @($files | Sort-Object -Property @{ Expression = { [long]$_.created } }, id)

        $cur = 0
        foreach ($f in $files) {
            $ext = $f.filetype
            $path = "$backupLoc/$($f.id)"
            $perc = ((100.0 * $cur++) / $files.Count)
            Write-Progress -Activity "File $($f.id)" -PercentComplete $perc -Status "Downloading"
            $slackFile = Get-SlackFile -Token $Token -Uri $f.url_private

            if ($slackFile.Length -eq 0) {
                Write-Host "File $($f.id) is empty or could not be downloaded."
                continue
            }

            Write-Progress -Activity "File $($f.id)" -PercentComplete $perc -Status "Running FileDataProcessingPipeline"
            $slackFile = Invoke-FileDataProcessingPipeline -SlackFile $slackFile -Metadata $f
            [System.IO.File]::WriteAllBytes("$path.$ext", $slackFile)
            Remove-Variable -Name slackFile

            Write-Progress -Activity "File $($f.id)" -PercentComplete $perc -Status "Running FileProcessingPipeline"
            $f | Invoke-FileProcessingPipeline | ConvertTo-Json -Depth 10 | Set-Content -Path "$path.json"
        }
    }
}