MediaValet.DAM

0.6.8

MediaValet.DAM.psm1


function Import-MvDamFilenameListFromCsv {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$Path
    )
    PROCESS {
        $csv = Import-Csv -Path $Path
        $filenameArray = $csv | Foreach {$_.Filename}
        return $filenameArray
    }
    
}

function Export-MvDamAssetInfo {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0,ParameterSetName="From Filename List")]
        [string[]]$FilenameList,
        [parameter(Mandatory=$true,Position=0,ParameterSetName="From Asset ID List")]
        [string[]]$AssetIdList,
        [parameter(Mandatory=$true, Position=0,ParameterSetName="All Assets")]
        [switch]$ListAllAssets,
        [parameter(Mandatory=$true,Position=1)]
        [string]$Path,
        [parameter(Mandatory=$false, Position=2)]
        [switch]$Resume
    )
    PROCESS {
    if ($ListAllAssets)
    {
            $previousResults = @()
            if (!$Resume)
            {
                $currentResults = Find-MvDamAssetInfo -SearchQuery "*&sort=record.createdAt+A" -Count 1000
            }
            else
            {
                $data = Import-Csv $Path

                ## I need to convert the CSV data into the correct obj so that I can add it to previous and do the compare.

                $result = $data[$data.Count - 1]
                $lastCreatedDate=$result."System.CreatedDate"

                if ($data.Length -gt 1000)
                {
                    $startIndex = ($data.Length - 1) - 999 
                    $endIndex = $data.Length - 1
                    $lastUniqueBatch = $data[$startIndex..$endIndex] 
                } 
                else
                {
                    $endIndex = $data.Length - 1
                    $lastUniqueBatch = $data[0..$endIndex] 
                }
                
                foreach ($asset in $lastuniqueBatch)
                {
                   $previousResults += [PSCustomObject]@{
                         Id = $asset.'System.Id'
                   }
                }

                $assetCtr = $data.length
                $data = $null
                $currentResults = Find-MvDamAssetInfo -SearchQuery "*&sort=record.createdAt+A&filters=(DateUploaded+GE+$lastCreatedDate)" -Count 1000
            }
            
            do
            {
                $firstCreatedDate = $currentResults[0].CreatedDate.ToString('o')
                $lastCreatedDate = $currentResults[$currentResults.Length-1].CreatedDate.ToString('o')
                
                $combinedAssets = $currentResults + $previousResults 
                $uniqueAssets = @{}
                $duplicateKeys = @()
                foreach ($asset in $combinedAssets)
                {
                    if(!$uniqueAssets.Contains($asset.Id))
                    {
                        $uniqueAssets.Add($asset.Id, $asset)
                    } 
                    else 
                    {
                        $duplicateKeys += $asset.Id
                    }
                }

                $convertedAssets = @()
                foreach ($result in $currentResults)
                {
                    if (!$duplicateKeys.Contains($result.Id))
                    {
                        $convertedAssets += ConvertAssetToCSVExport $result
                    }
                }

                $isNewAssets = ($currentResults.Count -ne $duplicateKeys.Count)

                if ($isNewAssets)
                {
                    $convertedAssets | Export-CSV -Path $Path -Encoding utf8 -Append -NoTypeInformation

                    $assetCtr +=  $convertedAssets.Count
                    Write-Host "Retrieved $assetCtr assets: "  $firstCreatedDate "to" $lastCreatedDate

                    $filter = "*&sort=record.createdAt+A&filters=(DateUploaded+GE+$lastCreatedDate)"
                    $previousResults = $currentResults

                    $currentResults = Find-MvDamAssetInfo -SearchQuery $filter -Count 1000
                }
            }
            while ($isNewAssets) 
        }
        else
        {
            $assets = @()
            if ($FilenameList)
            {
                $assets = Get-MvDamAssetInfo -FilenameList $FilenameList
            }
            if ($AssetIdList)
            {
                $assets = Get-MvDamAssetInfo -AssetIdList $AssetIdList
            }
            
            $convertedAssets = @()
            foreach ($asset in $assets)
            {
                $convertedAssets += ConvertAssetToCSVExport $asset
            }
        
            $convertedAssets | Export-CSV -Path $Path -Encoding utf8 -Append -NoTypeInformation
        }
    }
}

function Export-MvDamAssetAttribute {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [object[]]$AssetInfoList,
        [parameter(Mandatory=$false,Position=1)]
        [string[]]$AttributeList,
        [parameter(Mandatory=$true,Position=2)]
        [string]$Path
    )
    PROCESS {
        if ($AssetInfoList.GetType().Name -ne "AssetInfoModel[]" -or $AssetInfoList.Count -eq 0)
        {
            Write-Error -Message "Please pass an AssetInfoModel array with one or more items to the `$AssetInfoList parameter"
        }
        #Get AttributeList for Library
        if ($AttributeList -eq $null)
        {
            $AttributeList = (Get-MvDamAttributeDef) | Foreach {if ($_.IsSystemProperty){"System."+$_.Name}else{$_.Name}}
        }
        else #validate the AttributeList against the AttributeDefs
        {
            $objAttrNames = (Get-MvDamAttributeDef) | Foreach {if ($_.IsSystemProperty){"System."+$_.Name}else{$_.Name}}
            $AttributeList | Foreach { if (-not $objAttrNames.Contains($_)) {Write-Error "Invalid attribute name $_ specified"} } 
        }
        $assets = @()
        Foreach ($assetInfo in $AssetInfoList)
        {
            $asset = New-Object -TypeName PSObject
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Id' -Value $assetInfo.Id
            $asset | Add-Member -MemberType NoteProperty -Name 'System.FileName' -Value $assetInfo.FileName
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Title' -Value $assetInfo.Title
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Description' -Value $assetInfo.Description
            Foreach($attributeName in $AttributeList)
            {
                if ($attributeName -notlike "System.*")
                {
                    $attrValue = $assetInfo.Attributes | Where {$_.AttributeName -eq $attributeName} | Select -Property AttributeValue
                    $asset | Add-Member -MemberType NoteProperty -Name "$attributeName" -Value $attrValue.AttributeValue
                }
            }
            $assets += $asset  
        }
        $assets | Export-Csv -Path $Path -Encoding utf8 -Append -NoTypeInformation 
        return $assets
    }
}

function Import-MvDamAssetAttributesFromCsv {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$SourcePath,
        [parameter(Mandatory=$false,Position=1)]
        [string]$LogPath
    )
    PROCESS {
        $csv = @()
        $import = Import-Csv -Path $SourcePath -Encoding Default
        if ($import.GetType().ToString() -eq "System.Object[]")
        {
            $csv = $import
        }
        else
        {
            $csv = @($import)
        }
        for($i=0;$i -lt $csv.Length; $i++)
        {
            $line = $csv[$i]
            $assetId = $line.('System.Id')
            $percentComplete = ($i/$csv.Length)*100
            Write-Progress -Activity "Importing Asset Attributes" -Status "Updating attributes for Asset Id $assetId" -PercentComplete $percentComplete 
            $txnInfo = New-MvDamTxnInfo -StartTime (Get-Date) -Id $assetId  -Status "InProgress" -TxnType "Update-MvDamAssetAttribute"
            $attrKvps = @{}
            $line | Get-Member | Where {$_.MemberType -eq "NoteProperty"} | Foreach {if ($line.($_.Name) -ne "") {$attrKvps.Add($_.Name, $line.($_.Name))}}
            $errorInfo = $null
            $txnResult = Update-MvDamAssetAttribute -AssetId $assetId -Attributes $attrKvps -ErrorAction Continue -ErrorVariable $errorInfo
            $txnInfo.EndTime = Get-Date
            if ($errorInfo)
            {
                $txnInfo.Status = "Failed"
                $txnInfo.Notes = $errorInfo.Message
            }
            else
            {
                $txnInfo.Status = "Succeeded"
            }
            if ($LogPath)
            {
                $txnInfo | Export-Csv -Path $LogPath -Encoding utf8 -Append
            }
        }
        Write-Progress -Activity "Importing Asset Attributes" -Completed 

        return 
    }
}

function Test-MvDamAssetAttributeCsv {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$SourcePath,
        [parameter(Mandatory=$true,Position=1)]
        [string]$ResultPath
    )
    PROCESS {
        $csv = @()
        $import = Import-Csv -Path $SourcePath
        if ($import.GetType().ToString() -eq "System.Object[]")
        {
            $csv = $import
        }
        else
        {
            $csv = @($import)
        }
        $assetsWithErrors = 0
        $totalErrors = 0
        for($i=0;$i -lt $csv.Length; $i++)
        {
            $line = $csv[$i]
            $assetId = $line.('System.Id')
            $percentComplete = ($i/$csv.Length)*100
            Write-Progress -Activity "Validating Asset Attributes" -Status "Testing attributes for Asset Id $assetId" -PercentComplete $percentComplete 
            $attrKvps = @{}
            $line | Get-Member | Where {$_.MemberType -eq "NoteProperty"} | Foreach {if ($line.($_.Name) -ne "") {$attrKvps.Add($_.Name, $line.($_.Name))}}
            $errorInfo = $null
            $txnResult = Test-MvDamAssetAttribute -AssetId $assetId -Attributes $attrKvps -ErrorAction Continue -ErrorVariable $errorInfo
            if ($txnResult.Errors -eq 0)
            {
                $line | Add-Member -MemberType NoteProperty -Name 'Errors' -Value ""
            }
            else
            {
                $assetsWithErrors++
                $totalErrors += $txnResult.Errors
                $errorMsgs = ($txnResult.AttributeValidatonResults | Foreach ValidationResult) -join "; "
                $line | Add-Member -MemberType NoteProperty -Name 'Errors' -Value $errorMsgs
            }
            if ($i -eq 0)
            {
                $line | Export-Csv -Path $ResultPath -Encoding utf8  -NoTypeInformation
            }
            else
            {
                $line | Export-Csv -Path $ResultPath -Encoding utf8  -Append -NoTypeInformation
            }
        }
        Write-Progress -Activity "Validating Asset Attributes" -Completed 
        Write-Host "Total Assets Validated:"$csv.Length
        Write-Host "Assets with Error(s):"$assetsWithErrors
        Write-Host "Total Errors Detected"$totalErrors
        if ($totalErrors -eq 0)
        {
            Write-Host "`r`nCONGRATULATIONS! Your asset attribute CSV file is ready to import."
        }
        return 
    }
}

function Import-MvDamAssetKeywordFromCsv
{
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$Path,
        [parameter(Mandatory=$false,Position=1)]
        [string]$ErrorLogPath = ".\UpdateKeywordErrors.csv"
    )
    PROCESS 
    {
        $Assets = Import-Csv -Path $Path | Select 'System.Id', Keywords

        if ($Assets -ne $null)
        {
            "Validating Ids"
            $Count = 0
            foreach ($id in ($Assets | select Id))
            {
                if ($id -match '<<[a-z]*>>')
                {
                    $Count += 1;
                }
            }

            if ($Count -gt 0)
            {
                throw "$Count issues were detected with the Asset Ids provided. Please resolve this before continuing."
            }

            "Ids are valid"
            "Starting keyword import"

            $err = @()
            $lastErrorCount = 0
            foreach ($asset in $Assets)
            {
                "AssetId: {0}" -f $asset.'System.Id'
               if(!$asset.Keywords.Equals(""))
               {
                    "Keywords: $asset.Keywords"
                    $keywords = $asset.Keywords.Split(",")
                    Update-MvDamAssetKeyword -AssetId $asset.'System.Id' -Keywords $keywords -ErrorAction Continue -ErrorVariable +err

                    if ($err.Count -gt $lastErrorCount)
                    {
                        $updateKeywordError = New-Object -TypeName PSObject
                        $updateKeywordError | Add-Member -NotePropertyName "System.Id" -NotePropertyValue $asset.'System.Id'
                        $updateKeywordError | Add-Member -NotePropertyName "Keywords" -NotePropertyValue $asset.Keywords
                        $updateKeywordError | Add-Member -NotePropertyName "Error" -NotePropertyValue $err[$lastErrorCount]

                        $updateKeywordError | Export-Csv -Path $ErrorLogPath -Encoding utf8 -NoTypeInformation -Append

                        $lastErrorCount += 1
                    }
               }
            }

            if ($lastErrorCount -gt 0)
            {
                "Import completed with {0} errors. Logs can be found here: {1}" -f $lastErrorCount, $ErrorLogPath 
            } 
            else 
            {
                "Import complete"
            }
        }
    }
}

function Import-MvDamAssetCategoryFromCsv
{
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$Path,
        [parameter(Mandatory=$false)]
        [string]$ErrorLogPath = ".\ImportAssetCategoryErrorLog.csv" 
    )
    PROCESS 
    {
        $Assets = Import-Csv -Path $Path | Select 'System.Id', Path

        Test-MvDamSystemId -Ids ($Assets | Select -ExpandProperty System.Id)

        $err = @()
        $lastErrorCount = 0
        foreach ($asset in $Assets)
        {
            if(!$asset.Path.Equals(""))
            {
                $asset
                $category = Get-MvDamCategory -CategoryPath $asset.Path -ErrorAction Continue -ErrorVariable +err
                Update-MvDamAssetCategory -AssetId $asset.'System.Id' -CategoryIds $category.Id -ErrorAction Continue -ErrorVariable +err

                if ($err.Count -gt $lastErrorCount)
                {
                    $updateCategoryError = New-Object -TypeName PSObject
                    $updateCategoryError | Add-Member -NotePropertyName "System.Id" -NotePropertyValue $asset.'System.Id'
                    $updateCategoryError | Add-Member -NotePropertyName "Path" -NotePropertyValue $asset.Path
                    $updateCategoryError | Add-Member -NotePropertyName "Error" -NotePropertyValue $err[$lastErrorCount]

                    $updateCategoryError | Export-Csv -Path $ErrorLogPath -Encoding utf8 -NoTypeInformation -Append

                    $lastErrorCount += 1
                }
            }
        }
    }
}

function Import-MvDamAssetStatusFromCsv {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$Path,
        [parameter(Mandatory=$false)]
        [string]$ErrorLogPath = ".\ImportAssetStatusErrorLog.csv" 
    )
    PROCESS
    {
        $Assets = Import-CSV -Path $Path | Select 'System.Id', Status
        
        Test-MvDamSystemId -Ids ($Assets | Select -ExpandProperty 'System.Id')
        
        $err = @()
        $lastErrorCount = 0
        foreach ($asset in $Assets)
        {
            if(!$asset.Status.Equals(""))
            {
                Update-MvDamAssetStatus -AssetId $asset.'System.Id' $asset.Status -ErrorAction Continue -ErrorVariable +err
            }
            
                if ($err.Count -gt $lastErrorCount)
                {
                    $updateCategoryError = New-Object -TypeName PSObject
                    $updateCategoryError | Add-Member -NotePropertyName "System.Id" -NotePropertyValue $asset.'System.Id'
                    $updateCategoryError | Add-Member -NotePropertyName "Status" -NotePropertyValue $asset.Status
                    $updateCategoryError | Add-Member -NotePropertyName "Error" -NotePropertyValue $err[$lastErrorCount]

                    $updateCategoryError | Export-Csv -Path $ErrorLogPath -Encoding utf8 -NoTypeInformation -Append

                    $lastErrorCount += 1
                }
        }
    }
}

function Test-MvDamSystemId
{
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string[]]$Ids
    )   

    PROCESS 
    {
        $Count = 0
        foreach ($id in $Ids)
        {
            try {
                [System.Guid]::Parse($id) | Out-Null

            } catch {
                $Count += 1
            }
        }

        if ($Count -gt 0)
        {
            throw "$Count issues were detected with the System.Ids provided. Please resolve this before continuing."
        } 
        else 
        {
            "Validation complete. No issues were detected."
        }
    }
}

function Initialize-MvDamUploadBatch {
        [CmdletBinding()]
        param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$SourcePath,
        [parameter(Mandatory=$true,Position=1)]
        [string]$ManifestFilePath
    )
    PROCESS {
        $AttributeList = (Get-MvDamAttributeDef) | Foreach {if ($_.IsSystemProperty){"System."+$_.Name}else{$_.Name}} 
        if ($AttributeList -eq $null)
        {
            return
        }
        Write-Progress -Activity "Reading folder structure..." -Status "Initializing"
        $files = Get-ChildItem -Path $SourcePath -File -Recurse
        $ctr = 0
        $md5 = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider
        foreach($file in $files)
        {
            $ctr++
            Write-Progress -Activity "Adding $file to manifest" -Status "In-Progress" -PercentComplete ($ctr/$files.Count*100)
            #Write-Host $file.DirectoryName, $file.Name, $file.Basename $file.Length, $file.CreationTimeUtc, $file.LastWriteTimeUtc
            $relPath = $file.DirectoryName.Replace($SourcePath, "")
            if (!($env:OS).StartsWith("Windows")) 
            {
                $relPath = $relPath -replace '/','\'
            }
            $hash = [System.Convert]::ToBase64String($md5.ComputeHash([System.IO.File]::ReadAllBytes($file.FullName)))
            $asset = New-Object -TypeName PSObject
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Id' -Value ""
            $asset | Add-Member -MemberType NoteProperty -Name 'Process.LocalPath' -Value $file.DirectoryName
            $asset | Add-Member -MemberType NoteProperty -Name 'Process.LocalName' -Value $file.Name
            $asset | Add-Member -MemberType NoteProperty -Name 'System.FileName' -Value $file.Name
            $asset | Add-Member -MemberType NoteProperty -Name 'System.CategoryPath' -Value $relPath
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Title' -Value $file.Basename
            $asset | Add-Member -MemberType NoteProperty -Name 'System.MD5Checksum' -Value $hash
            $asset | Add-Member -MemberType NoteProperty -Name 'Process.PreflightStatus' -Value "Pending"
            $asset | Add-Member -MemberType NoteProperty -Name 'Process.PreflightInfo' -Value ""
            $asset | Add-Member -MemberType NoteProperty -Name 'Process.UploadStatus' -Value "Preflight"
            $asset | Add-Member -MemberType NoteProperty -Name 'Process.UploadInfo' -Value ""
            $asset | Add-Member -MemberType NoteProperty -Name 'System.File Size' -Value $file.Length
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Created Date' -Value $file.CreationTimeUtc
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Last Modified' -Value $file.LastWriteTimeUtc
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Description' -Value ""
            $asset | Add-Member -MemberType NoteProperty -Name 'System.Keywords' -Value ""
            Foreach($attributeName in $AttributeList)
            {
                if ($attributeName -notlike "System.*")
                {
                    $attrValue = $assetInfo.Attributes | Where {$_.AttributeName -eq $attributeName} | Select -Property AttributeValue
                    $asset | Add-Member -MemberType NoteProperty -Name "$attributeName" -Value $attrValue.AttributeValue
                }
            }
            if ($PSCmdlet.MyInvocation.BoundParameters["Verbose"].IsPresent -eq $true)
            {
                Write-Host "Adding to manifest: $asset `r`n"
            }
            $asset | Export-Csv -Path $ManifestFilePath -Encoding uft8 -Append -NoTypeInformation 
        }
        Write-Progress -Activity "Finalizing" -PercentComplete 100 -Completed
    }
}

function Test-MvDamUploadBatch {
        [CmdletBinding()]
        param (
        [parameter(Mandatory=$true,Position=0)]
        [string]$ManifestFilePath,
        [parameter(Mandatory=$false,Position=1)]
        [Guid]$ParentCategoryId,
        [parameter(Mandatory=$false,Position=2)]
        [switch]$AutoCreateCategories
    )
    PROCESS {
    }
}

function ConvertAssetToCSVExport ($assetInfo)
{    
    $asset = New-Object -TypeName PSObject    
    foreach ($property in $assetInfo.PSObject.Properties)
    {
        if (!$property.Name.Equals("Attributes"))
        {
            if($property.Name.Equals("Keywords") -or $property.Name.Equals("Categories"))
            {
                $asset | Add-Member -MemberType NoteProperty -Name ("System." + $property.Name) -Value ($property.Value -join ", ")
            }
            elseif ($property.Value -is [DateTime])
            {
                $asset | Add-Member -MemberType NoteProperty -Name ("System." + $property.Name) -Value ($property.Value.ToString("o", $null))
            }
            else
            {
                $asset | Add-Member -MemberType NoteProperty -Name ("System." + $property.Name) -Value $property.Value
            }
        }
     }
   
    foreach($attribute in $assetInfo.Attributes)
    {
        if (!$attribute.IsSystemProperty)
        {
            $attrValue = $assetInfo.Attributes | Where {$_.AttributeName -eq $attribute.AttributeName} | Select -ExpandProperty AttributeValue
            $asset | Add-Member -MemberType NoteProperty -Name $attribute.AttributeName -Value $attrValue
        }
    }

    return $asset
}

function Import-MvDamCategoryTreeFromManifest {
    [CmdletBinding()]
    param (
        [parameter(Mandatory = $true, Position = 0, ParameterSetName = "By File Path")]
        [string]$ManifestFilePath,
        [parameter(Mandatory = $true, Position = 1, ParameterSetName = "By File Path")]
        [string]$LogPath,
        [parameter(Mandatory = $true, Position = 1, ParameterSetName = "By Job Folder")]
        [string]$JobFolder
    )
    PROCESS {
        $jobResult = 0
        $errorCount = 0
        $JobStartTime = Get-Date 
        $RootCategoryName = (Get-MvDamCategory -RootCategory).Name

        if ($PSCmdlet.ParameterSetName -eq "By Job Folder") {
            $ManifestFilePath = [System.IO.Path]::Combine($JobFolder, 'category-manifest.csv')
            $LogFolder = [System.IO.Path]::Combine($JobFolder, 'Logs')
            if (-not(Test-Path $LogFolder)) {
                New-Item -Path $LogFolder -ItemType Directory > $null
            }
            $LogPath = [System.IO.Path]::Combine($JobFolder, 'Logs', 'category-import-log.csv')
        }

        $categories = Import-Csv -Path $ManifestFilePath
        $ctr = 0
        $categoryCount = $categories.Length
        if ($categories.GetType().BaseType -ne [System.Array]) {
            $categoryCount = 1
        }
        ForEach ($category in $categories) {
                
            Write-Progress -Activity 'Creating categories from import file...' -PercentComplete ($ctr/$categoryCount*100)
            $result = New-Object PSObject
            $result | Add-Member -NotePropertyName 'CategoryTreePath' -NotePropertyValue $category.CategoryPath
            $result | Add-Member -NotePropertyName 'Status' -NotePropertyValue 'Pending'
            $result | Add-Member -NotePropertyName 'CategoryId' -NotePropertyValue ([System.Guid]::Empty).ToString()
            $result | Add-Member -NotePropertyName 'BatchStarted' -NotePropertyValue $JobStartTime
            $result | Add-Member -NotePropertyName 'Submitted' -NotePropertyValue (Get-Date -Format FileDateTimeUniversal)
            $result | Add-Member -NotePropertyName 'Completed' -NotePropertyValue ''
            $result | Add-Member -NotePropertyName 'Notes' -NotePropertyValue ''
            if (($category.CategoryPath.ToLower().StartsWith($RootCategoryName.ToLower())) -or ($category.CategoryPath.ToLower().StartsWith('\root\'))) {
                $opsErr = $null
                #Check if access token still valid
                if ((Get-Date).AddMinutes(5) -ge $MvDamContext.ClientRunspace.AccessTokenExpiry) {
                    Connect-MvDamAccount -Username $MvDamContext.ClientRunspace.Username -Region $region -Password $MvDamContext.ClientRunspace.Password
                }

                $catOperation = New-MvDamCategory -Name $category.CategoryPath -UseRootPath -CreateTreeIfNotExists -ErrorAction SilentlyContinue -ErrorVariable opsErr
                if ($opsErr.Count -eq 0) {
                    $leafCat = $catOperation[$catOperation.count - 1]
                    $result.CategoryId = $leafCat.Id
                    $result.Status = 'Success'
                    $result.Completed = Get-Date -Format FileDateTimeUniversal
                    $catOperation | ForEach-Object { Write-Host "Created category $($_.Tree.Path)" }
                }
                else {
                    $result.Status = 'Fail'
                    if ($opsErr -ne $null){
                        if ($opsErr[0].InnerException.InnerException -ne $null) 
                        {
                            $result.Notes = $opsErr[0].InnerException.InnerException.Message
                        }
                        else
                        {
                            $result.Notes = $opsErr[0].Message
                        }

                    }
                    Write-Warning "Failed to create $($category.CategoryPath)"
                    $errorCount++
                }
                $catOperation = $null
            }
            else {
                $result.Status = 'Failed'
                $result.Notes = "Invalid category path."
            }
            $result | Export-Csv -Path $LogPath -Encoding utf8 -NoTypeInformation -Append 
            $ctr++
        }
        Write-Progress -Activity 'Creating categories from import file...' -PercentComplete 100 -Completed
        if ($errorCount -gt 0) {
            Write-Warning "Failed creating $errorCount categories. Please retry the Import-MvDamCategoryTreeFromManifest command and/or review the category manifest for invalid category paths."
            $jobResult = -1
        }
        return $jobResult
    }
}


function Get-MvDamCategoryFromBlobPath {
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $BlobPath
        )
    PROCESS {

    $pathSegments = $BlobPath.Replace('/','\').Split("\\") #Stanardize MacOs Paths to windows
    $categoryPath = ''
    if ($pathSegments.Count -ge 2)
    {
        $categoryPath = [string]::Join("\", $pathSegments[0..$($pathSegments.Count-2)])
    }
    return $categoryPath
    }
}

function Get-MvDamFilenameFromBlobPath {
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $BlobPath
        )
    PROCESS {

    $pathSegments = $BlobPath.Replace('/','\').Split("\\") #Stanardize MacOs Paths to windows
    $filename = Remove-MvDamInvalidFileNameChars($pathSegments[$pathSegments.Count-1]) ##Last segment of a blob path is the blob
    return $filename
    }
}

function Export-MvDamContainerManifest {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $SrcSasUri,
        [parameter(Mandatory=$false,Position=1)]
        [string] $ParentCategoryPath = '\Root',
        [parameter(Mandatory=$true,Position=2)]
        [string] $JobFolder
        )
    PROCESS {
        #Validate that the Az.Storage module is installed
        if ($null -eq (Get-Module -Name Az.Storage -ListAvailable | Where-Object {(($_.Version.Major * 100) + ($_.Version.Minor * 10)) -ge 270 }))
        {
            Write-Warning "Az.Storage Powershell Module v2.7 is not installed. Please run 'Install-Module -Name Az -AllowClobber' as Administrator."
            return -1
        } 

        #Validate MvDamContext
        if ($null -eq $MvDamContext)
        {
            Write-Warning "You are not connected to a MediaValet DAM Account. Please run 'Connect-MvDamAccount' to connect to an account."
            return -1
        }

        #validate ParentCategoryPath
        if ($ParentCategoryPath.EndsWith("\"))
        {
            $ParentCategoryPath = $ParentCategoryPath.Substring(0, $ParentCategoryPath.Length - 1)
        }
        $parentCategory = Get-MvDamCategory -CategoryPath $ParentCategoryPath 
        if ($null -eq $parentCategory)
        {
            Write-Warning "ParentCategoryPath does not exist. Please create before proceeding."
            return -1
        }

        $assetManifestFilePath = [System.IO.Path]::Combine($JobFolder, "asset-manifest.csv")
        $categoryManifestFilePath = [System.IO.Path]::Combine($JobFolder, "category-manifest.csv")
        if ((Test-Path $assetManifestFilePath) -or (Test-Path $categoryManifestFilePath))
        {
            Write-Warning "Manifest files already exist. Please specify a different JobFolder or clear the specified JobFolder. Stopping execution."
            return -1
        }
         
        $sasUri = [System.Uri]$SrcSasUri
        $storageAccountName = $sasUri.Host.Split('.')[0]
        $containerName = $sasUri.LocalPath.Remove(0,1) #remove the leading /
        $containerBaseUri = $sasUri.GetLeftPart(1)
        $sasToken = $sasUri.Query

        $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName -SasToken $sasToken -Protocol Https 
        $concurrentTasks = [System.Environment]::ProcessorCount 

        if (-not(Test-Path $JobFolder))
        {
            New-Item -Path $JobFolder -ItemType Directory > $null
        }
        $token = $null
        $categories = [System.Collections.ArrayList] @()
        $runningTotal = 0
        do {
                Write-Progress -Activity "Retrieved $runningTotal blobs."
                $blobs = Get-AzStorageBlob -Container $containerName -Context $storageContext -ConcurrentTaskCount $concurrentTasks -MaxCount 10000 -ContinuationToken  $token     
                if ($null -eq $blobs)
                {
                    break
                } 
                $token = $blobs[$blobs.Count - 1].ContinuationToken
                $uploadRequests = $blobs | Select-Object @{Label='CategoryPath';Expression={$ParentCategoryPath + '\' + (Get-MvDamCategoryFromBlobPath($_.Name))}}, `
                    @{Label='Filename';Expression={Get-MvDamFilenameFromBlobPath($_.Name)}}, @{Label='Filesize';Expression={$_.Length}} , 
                    ContentType, LastModified, @{Label='ContentMD5';Expression={$_.ICloudBlob.Properties.ContentMD5}}, `
                    @{Label='SasUrl';Expression={"$($_.ICloudBlob.Uri.AbsoluteUri)$sasToken"}}
                $uploadRequests | Export-Csv -Path $assetManifestFilePath -Encoding utf8 -NoTypeInformation -Append
                $currentCategories = $uploadRequests | Select-Object CategoryPath -Unique
                if ($null -ne $currentCategories)
                {
                    $incrementalCategories = $currentCategories | Where-Object {$_.CategoryPath -NotIn $categories.CategoryPath}
                    if ($null -ne $incrementalCategories)
                    {
                        if ($incrementalCategories.GetType().Name -eq 'PSCustomObject') #single object returned, not an array
                        {
                            $categories.Add($incrementalCategories) > $null
                        }
                        else {
                            $categories.AddRange($incrementalCategories) > $null
                        }
                    }
                }
                $runningTotal += $blobs.Count
        } while ($null -ne $token)
        $categories | Sort-Object -Property CategoryPath | Export-Csv -Path $categoryManifestFilePath -Encoding utf8 -NoTypeInformation
        return 0
    }
}

function Export-MvDamDupeAssetReport {
   [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $JobFolder,
        [parameter(Mandatory=$true,Position=1)]
        [string] $ReportFilePathCsv
        )
    PROCESS {
        if (Test-Path $ReportFilePathCsv)
        {
            Write-Warning "Report file path already exists. Please specify a different name"
            return -1
        }
        if (!(Test-Path $JobFolder))
        {
            Write-Warning "JobFolder $JobFolder does not exist."
            return -1
           
        }
        $manifestFilePath = Join-Path $JobFolder -ChildPath "asset-manifest.csv"
        if (!(Test-Path $manifestFilePath))
        {
            Write-Warning "Asset manifest is missing. Please run the Export-MvDamContainerManifest cmdlet first"
            return -1
        }

        $blobs = Import-Csv -Path $manifestFilePath |  Sort-Object -Property ContentMD5, Filename
        $lastCategoryPath = ""
        $lastFileName = ""
        $lastMD5 = ""
        foreach($blob in $blobs)
        {
            $isDupe = $blob.ContentMD5 -eq $lastMD5
            if ($isDupe -eq $false)
            {
                $lastCategoryPath = ""
                $lastFileName = ""
            }
            $blob | Select CategoryPath, Filename, Filesize, ContentType, LastModified,ContentMD5,`
                @{Label="Duplicate";Expression={$isDupe}}, @{Label="OriginCategoryPath";Expression={$lastCategoryPath}}, @{Label="OriginFilename";Expression={$lastFileName}}, SasUrl | `
                Export-Csv -Path $ReportFilePathCsv -Encoding utf8 -Append -NoTypeInformation
            if ($isDupe -eq $false)
            {
                $lastMD5 = $blob.ContentMD5
                $lastCategoryPath = $blob.CategoryPath
                $lastFileName = $blob.Filename
            }
        }
    }
 }

 function Import-MvDamAssetClassificationFromDupeCsv {
     [CmdletBinding()]
     param (
            [parameter(Mandatory=$true,Position=0)]
            [string] $JobFolder
     )
     PROCESS {
            if (!(Test-Path $JobFolder))
            {
                Write-Warning "JobFolder $JobFolder does not exist."
                return -1
            }

            $logPath = Join-Path $JobFolder -ChildPath 'Logs'
            $uploads = [System.Collections.Arraylist] @()
            $dupesToCategorizeFilePath = Join-Path $JobFolder -ChildPath 'dupes-to-categorize.csv'
            $logFiles = Get-ChildItem -Path $logPath -Filter "asset-import-log-*.csv" 
            $dupeClassificationLog = Join-Path $logPath -ChildPath "dupe-classification-log.csv"
            foreach($logfile in $logfiles)
            {
                $currentLog = Import-Csv -Path $logfile.FullName -Encoding UTF8
                $uploads.AddRange($currentLog)
            }
            $dupesToCategorize = Import-Csv -Path $dupesToCategorizeFilePath -Encoding UTF8
            $originals = $dupesToCategorize | Select OriginFileName, ContentMD5 -Unique
            foreach($original in $originals)
            {
                $dupes = $dupesToCategorize | Where {$_.ContentMD5 -eq $original.ContentMD5}
                $categories =[System.Collections.ArrayList] @()
                $originalCategory = $null 
                foreach($dupe in $dupes)
                {
                    if ($originalCategory -eq $null)
                    {
                        $originalCategory = Get-MvDamCategory -CategoryPath $dupe.OriginCategoryPath
                        $categories.Add($originalCategory) > $null
                    }
                    $category = Get-MvDamCategory -CategoryPath $dupe.CategoryPath
                    $categories.Add($category) > $null
                }
                $upload = $uploads | Where {($_.Filename -eq $dupe.OriginFilename) -and ($_.Filesize -eq $dupe[0].Filesize) -and ($_.CateoryIds -eq $originalCategory.Id)} 
                if (($upload -eq $null) -or ($upload.Status -ne 'Success (Creation)'))
                {
                   foreach ($dupe in $dupes)
                   {
                       Write-Warning "Asset $($original.OriginFilename) was not added to $($dupe.CategoryPath)"
                       $dupe | Select CategoryPath, Filename, Filesize, ContentType, LastModified, ContentMD5, OriginCategoryPath,`
                            OriginFilename, SasUrl, @{Label="Result";Expression={"Fail"}} | `
                            Export-Csv -Path $dupeClassificationLog -Encoding utf8 -Append -NoTypeInformation 
                   }
                   continue   
                }
                $categoryIds = ($categories | ForEach {$_.Id}) | Select -Unique
                $results = Update-MvDamAssetCategory -AssetId $upload.AssetId -CategoryIds $categoryIds
                if ($results.CategoryIds.Count -eq $categoryIds.Count)
                {
                   foreach ($dupe in $dupes)
                   {
                       Write-Host "Asset $($original.OriginFilename) successfully added to $($dupe.CategoryPath)"
                       $dupe | Select CategoryPath, Filename, Filesize, ContentType, LastModified, ContentMD5, OriginCategoryPath,`
                            OriginFilename, SasUrl, @{Label="Result";Expression={"Success"}} | `
                            Export-Csv -Path $dupeClassificationLog -Encoding utf8 -Append -NoTypeInformation
                   }   
                }
                else
                {
                   foreach ($dupe in $dupes)
                   {
                       Write-Warning "Asset $($original.OriginFilename) was not added to $($dupe.CategoryPath)"
                       $dupe | Select CategoryPath, Filename, Filesize, ContentType, LastModified, ContentMD5, OriginCategoryPath,`
                            OriginFilename, SasUrl, @{Label="Result";Expression={"Fail"}} | `
                            Export-Csv -Path $dupeClassificationLog -Encoding utf8 -Append -NoTypeInformation 
                   }   
                }
            }
     }
 }

 function Split-MvDamAssetManifestForDeduping {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $JobFolder
        )
    PROCESS {
        if (!(Test-Path $JobFolder))
        {
            Write-Warning "JobFolder $JobFolder does not exist."
            return -1
        }

        $tempfilename = -join ((48..57) + (97..122) | Get-Random -Count 10 | % {[char]$_})
        $tempcsvpath = "$JobFolder\$tempfilename.csv"

        $manifestFilePath = [System.IO.Path]::Combine($JobFolder,"asset-manifest.csv")
        $manifestFileOriginalPath = [System.IO.Path]::Combine($JobFolder,"asset-manifest-original.csv")
        if (!(Test-Path $manifestFilePath))
        {
            Write-Warning "Asset manifest is missing. Please run the Export-MvDamContainerManifest cmdlet first"
            return -1
        }

        $dupesToCategorizeFilepPath = [System.IO.Path]::Combine($JobFolder,"dupes-to-categorize.csv")
        $dupesToResolveFilepPath = [System.IO.Path]::Combine($JobFolder,"dupes-to-resolve.csv")
        if ((Test-Path $dupesToCategorizeFilepPath) -or (Test-Path $dupesToResolveFilepPath))
        {
            Write-Warning "Asset manifest has already been split for deduping"
            return -1
        }

        #rename old asset-manifest.csv
        Export-MvDamDupeAssetReport -JobFolder $JobFolder -ReportFilePathCsv $tempcsvpath
        $assets = Import-Csv -Path $tempcsvpath -Encoding utf8
        Move-Item $manifestFilePath -Destination $manifestFileOriginalPath
        Remove-Item -Path $tempcsvpath
        $missingMD5s = $assets | Where {[string]::IsNullOrEmpty($_.ContentMD5)}
        if ($missingMD5s -ne $null)
        {
            Write-Warning "Some blobs are missing ContentMD5 hash. Cannot proceed."
            return -1
        }

        #Create new asset manifest
        $assets | Where {$_.Duplicate -eq $false} | `
            Select CategoryPath, Filename, Filesize, ContentType, LastModified, ContentMD5, SasUrl | `
            Export-Csv -Path "$JobFolder\asset-manifest.csv" -Encoding utf8 -NoTypeInformation 

        #Create DupesToCategorize.csv
        $assets | Where {($_.Duplicate -eq $true) -and ($_.Filename -eq $_.OriginFilename) } | `
            Select CategoryPath, Filename, Filesize, ContentType, LastModified, ContentMD5, OriginCategoryPath, OriginFilename, SasUrl | `
            Export-Csv -Path $dupesToCategorizeFilepPath -Encoding utf8 -NoTypeInformation 

        #Create DupesToResolve.csv
        $assets | Where {($_.Duplicate -eq $true) -and ($_.Filename -ne $_.OriginFilename) } | `
            Select CategoryPath, Filename, Filesize, ContentType, LastModified, ContentMD5, OriginCategoryPath, OriginFilename, SasUrl | `
            Export-Csv -Path $dupesToResolveFilepPath -Encoding utf8 -NoTypeInformation

    }
 }


function Get-MvDamFileSizeClass {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [long] $FileSize
        )
    PROCESS {
        $sizeInMB = $fileSize/1024/1024
        $fileClass = `
        switch ($sizeInMB) {
            {$_ -le 1} {'A';break;}
            {$_ -le 2} {'B';break;}
            {$_ -le 4} {'C';break;}
            {$_ -le 8} {'D';break;}
            {$_ -le 16} {'E';break;}
            {$_ -le 32} {'F';break;}
            {$_ -le 64} {'G';break;}
            {$_ -le 128} {'H';break;}
            {$_ -le 256} {'I';break;}
            {$_ -le 512} {'J';break;}
            {$_ -le 1024} {'K';break;}
            {$_ -le 2048} {'L';break;}
            {$_ -gt 2048} {'M';break;}
            }
        return $fileClass
    }
}

function New-MvDamCreateAssetBatchStats {
    PROCESS 
    {

        $data = @(
            [PSCustomobject]@{FileClass='A';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=16;TargetFreqSecs=6}
            [PSCustomobject]@{FileClass='B';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=16;TargetFreqSecs=10}
            [PSCustomobject]@{FileClass='C';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=18}
            [PSCustomobject]@{FileClass='D';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=24}
            [PSCustomobject]@{FileClass='E';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=48}
            [PSCustomobject]@{FileClass='F';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=80}
            [PSCustomobject]@{FileClass='G';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=140}
            [PSCustomobject]@{FileClass='H';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=180}
            [PSCustomobject]@{FileClass='I';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=10;TargetFreqSecs=240}
            [PSCustomobject]@{FileClass='J';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=10;TargetFreqSecs=420}
            [PSCustomobject]@{FileClass='K';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=10;TargetFreqSecs=640}
            [PSCustomobject]@{FileClass='L';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=8;TargetFreqSecs=1200}
            [PSCustomobject]@{FileClass='M';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;`
                TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=4;TargetFreqSecs=1200}
        )
        return $data

    }
}

 function Initialize-MvDamUploadJob {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $JobFolder,
        [parameter(Mandatory=$false,Position=1)]
        [int] $ConcurrencyLevel = 2,
        [parameter(Mandatory=$false,Position=2)]
        [int] $MaxBatchSize= 20
        )
    PROCESS {
            if (!(Test-Path $JobFolder)) 
            {
                New-Item -Path $JobFolder -ItemType Directory > $null
            }

            $pendingJobFolder = [System.IO.Path]::Combine($jobFolder,'PendingBatches')
            if (!(Test-Path $pendingJobFolder)) 
            {
                New-Item -Path $pendingJobFolder -ItemType Directory > $null
            }

            $excludedJobFolder =  [System.IO.Path]::Combine($jobFolder,'SkippedBatches')
            if (!(Test-Path $excludedJobFolder)) 
            {
                New-Item -Path $excludedJobFolder -ItemType Directory > $null
            }

            $logPath = [System.IO.Path]::Combine($JobFolder,'Logs')
            if (!(Test-Path $logPath)) 
            {
                New-Item -Path $logPath -ItemType Directory > $null
            }
            $logFilePath = [System.IO.Path]::Combine($logPath, "asset-import-log-00001.csv")

            $categoryImportLogPath = [System.IO.Path]::Combine($JobFolder,'Logs','category-import-log.csv')
            if (-not(Test-Path $categoryImportLogPath))
            {
                Write-Warning "Category import log file is missing in the Job Folder. Please make sure that the category import completed."
                return -1
            }

            $stats = New-MvDamCreateAssetBatchStats
            $batchFileSizeCapMB = 12000 
            $processorCount = [Math]::Min($ConcurrencyLevel, [System.Environment]::ProcessorCount) #Do not exceed the number of cores

            $assetManifestFile = [System.IO.Path]::Combine($JobFolder,'asset-manifest.csv')
            $uploadRequests = Import-Csv -Path $assetManifestFile  | Sort-Object -Property LastModified -Descending
            $reqId = 1
            ForEach($request in $uploadRequests)
            {
                Write-Progress -Activity "Preparing asset upload plan..." -PercentComplete ($reqId/$uploadRequests.Count*100)
                $requestCategory = $null
                $requestCategory = Get-MvDamCategory -CategoryPath $request.CategoryPath
                $fileclass = Get-MvDamFileSizeClass -FileSize $request.Filesize

                $fileclassStat = $stats | Where {$_.FileClass -eq $fileclass}
                if (($fileclassStat.CurrentBatchCount -eq 0) -and ($fileclassStat.TotalBatchCount -eq 0))
                {
                    $fileclassStat.TotalBatchCount++
                }
                if (($fileclassStat.CurrentBatchFileCount -ge ($processorCount * $fileclassStat.TaskPerCore)) `
                    -or ($fileclassStat.CurrentBatchFileCount -ge $MaxBatchSize) `
                    -or ($fileclassStat.CurrentBatchFileSizeMB -gt $batchFileSizeCapMB))
                {
                    $fileclassStat.TotalBatchCount++
                    $fileclassStat.CurrentBatchFileCount = 0
                    $fileclassStat.CurrentBatchFileSizeMB = 0
                }

                $batchId = [Math]::Max($fileclassStat.BatchCount, 1)

                $refId = $reqId.ToString("0000000")
                if (-not ([String]::IsNullOrEmpty($request.RefId)))
                {
                    $refId = $request.RefId
                }

                if (IsValidUploadRequest($request))
                {
                    $fileclassStat.CurrentBatchFileCount++
                    $fileclassStat.CurrentBatchFileSizeMB += ($request.Filesize/1024/1024)
                    $fileclassStat.TotalFileCount++
                    $fileclassStat.TotalFileSizeMB += ($request.Filesize/1024/1024)
                    $targetFilePath = [System.IO.Path]::Combine($pendingJobFolder, $fileclass + '-' + $fileclassStat.TotalBatchCount.ToString('00000') + ".csv")
                    $request | Select-Object @{Label='RefId';Expression={$refId}}, @{Label='CategoryId';Expression={$requestCategory.Id}}, Filename, Filesize,  @{Label="SourceUrl";Expression={$_.SasUrl}} | Export-Csv -Path $targetFilePath -Append -NoTypeInformation
                }
                else
                {
                    $targetFilePath = [System.IO.Path]::Combine($excludedJobFolder, $fileclass + '-00000.csv')
                    $request | Select-Object @{Label='RefId';Expression={$refId}}, @{Label='CategoryId';Expression={$requestCategory.Id}}, CategoryPath, Filename, Filesize,  @{Label="SourceUrl";Expression={$_.SasUrl}} | Export-Csv -Path $targetFilePath -Append -NoTypeInformation
                    #Log the skipped file as well so that asset manifest count
                    $request | Select-Object @{Label='BatchId';Expression={$fileclass + '-00000'}}, RefId, `
                        @{Label='AssetId';Expression={[Guid]::Empty.Guid.ToString()}}, `
                        @{Label='CateoryIds';Expression={$requestCategory.Id}}, `
                        Filename, Filesize, `
                        @{Label="Status";Expression={"Skipped"}}, `
                        @{Label="SrcSasUrl";Expression={$_.SasUrl}}, `
                        @{Label="DestSasUrl";Expression={''}}, `
                        @{Label='StartedOn';Expression={(Get-Date)}},  `
                        @{Label='CreatedOn';Expression={''}},  `
                        @{Label='BlobCopiedOn';Expression={''}},  `
                        @{Label='TitledOn';Expression={''}},  `
                        @{Label='CategorizedOn';Expression={''}},  `
                        @{Label='EndedOn';Expression={(Get-Date)}}  |`
                        Export-Csv -Path $logFilePath -Encoding utf8 -Append -NoTypeInformation 
                }
                #Write-Host $request.filename $request.Filesize $fileclass
                $requestCategory = $null
                $reqId++
            }
            $stats | ConvertTo-Json | Out-File  $jobFolder\BatchJobMaster.json -Encoding utf8
            return 0
    }
} 

function IsValidUploadRequest($request)
{
    if (($request.Filesize -gt 0))
    {
        return (($MvDamContext.UploadExclusions | Where {$request.Filename -like $_} | Select -First 1) -eq $null);
    }
    else
    {
        return $false;
    }
}

 function Start-MvDamUploadJob {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $JobFolder,
        [parameter(Mandatory=$false,Position=1)]
        [int] $MaxUploadRate = 40,
        [parameter(Mandatory=$false)]
        [Switch] $UseAzCopy = $false
        )
    PROCESS 
    {

        if ($UseAzCopy)
        {
            $azcopyVersion = Invoke-Expression "azcopy --version"
            if (($azcopyVersion -eq $null) -or ($azcopyVersion.Split(' ')[2] -notlike "10.*.*"))
            {
                Write-Error "AzCopy10 not found. Please install AzCopy first."
            }
        }

        $jobResult = 0
        Write-Host "Starting batch asset submission...`n`n`n "
        try
        {
            $logPath = [System.IO.Path]::Combine($JobFolder,'Logs')
            $pendingJobFolder = [System.IO.Path]::Combine($JobFolder,'PendingBatches')
            $completedJobFolder = [System.IO.Path]::Combine($JobFolder,'CompletedBatches')
            $faultedJobFolder = [System.IO.Path]::Combine($JobFolder,'FaultedBatches')

            if (!(Test-Path $JobFolder)) 
            {
                New-Item -Path $JobFolder -ItemType Directory > $null
            }

            if (!(Test-Path $pendingJobFolder)) 
            {
                New-Item -Path $pendingJobFolder -ItemType Directory > $null
            }

            if (!(Test-Path $logPath)) 
            {
                New-Item -Path $logPath -ItemType Directory > $null
            }

            if (!(Test-Path $completedJobFolder)) 
            {
                New-Item -Path $completedJobFolder -ItemType Directory > $null
            }

            if (!(Test-Path $faultedJobFolder)) 
            {
                New-Item -Path $faultedJobFolder -ItemType Directory > $null
            }


            $jobMasterFilePath = [System.IO.Path]::Combine($JobFolder,'BatchJobMaster.json')
            $jobSummaryFilePath = [System.IO.Path]::Combine($JobFolder,'BatchJobSummary.csv')

            $jobMaster = (Get-Content -Path $jobMasterFilePath -Encoding UTF8 |  Out-String | ConvertFrom-Json)
            $currentBatchCount = ($jobMaster | Measure-Object -Property 'CurrentBatchCount' -Sum).Sum
            $totalBatchCount = ($jobMaster | Measure-Object -Property 'TotalBatchCount' -Sum).Sum
            $pullFromTop = $true 
            $fileclassCode = $jobMaster | Where-Object {($_.TotalBatchCount -gt 0) -and ($_.CurrentBatchCount -le $_.TotalBatchCount)} | Sort-Object -Property TargetFrequencySecs | Select-Object FileClass -First 1
            $maxUploadsPerMin = [Math]::Min(200, $MaxUploadRate) #Do not exceed 200 assets per minute
            $uploadLimitCounter = 0
            $uploadLimitDuration = 0

            $maxLinesPerLogFile = 10000
            $linesLogged = 0
            $logFiles = Get-ChildItem -Path $logPath -Filter "asset-import-log-*.csv" | Select @{Label="Id";Expression={[int]($_.Name.Substring(18,5))}}, FullName, Name
            $logFileCounter = [Math]::Max(($logFiles | Measure-Object -Property Id -Maximum).Maximum,1)

            if ($logFiles.Count -gt 0)
            {
                $lastLogFilePath = $logFiles | Where Id -eq $logFileCounter | Select FullName
                $linesLogged = (Import-Csv -Path $lastLogFilePath.FullName).Count
            }

            While($currentBatchCount -le $totalBatchCount)
            {
                Write-Progress -Activity 'Submitting upload batches...' -PercentComplete ($currentBatchCount/$totalBatchCount*100)
                if ($fileclassCode -eq $null)
                {
                    break
                }
                $fileclass = $fileclassCode.FileClass
                $jobClass = $jobMaster | Where {$_.FileClass -eq $fileclass}
                if ($jobClass.CurrentBatchCount -le $jobClass.TotalBatchCount)
                {

                    #Check if access token still valid even for a long running copy task
                    if ((Get-Date).AddMinutes(45) -ge $MvDamContext.ClientRunspace.AccessTokenExpiry)
                    {
                        $region = $MvDamContext.ClientRunspace.Region.ToString().ToLower().Insert(2,'-')
                        $password = $pass = ConvertTo-SecureString -String $MvDamContext.ClientRunspace.Password -AsPlainText -Force
                        Connect-MvDamAccount -Username $MvDamContext.ClientRunspace.Username -Region $region -Password $pass
                    }

                    $batchId = $fileclass + '-' + ([Math]::Max($jobClass.CurrentBatchCount,1)).ToString('00000')
                    $batchFilePath = [System.IO.Path]::Combine($pendingJobFolder, $batchId + ".csv")
                    $jobClass.CurrentBatchCount++
                    if (!(Test-Path -Path $batchFilePath))
                    {
                        Write-Host "$BatchId already processed."
                        Continue
                    }

                    $batchStart = Get-Date
                    $batch = Import-Csv -Path $batchFilePath -Encoding UTF8
                    #submit batch
                    $jobClass.CurrentBatchFileSizeMB = [Math]::Round(($batch | Measure-Object -Property 'FileSize' -Sum).Sum/1024/1024, 4)
                    Write-Host "Submitting $batchId with $($batch.Count) assets for ingestion - $($jobClass.CurrentBatchFileSizeMB)MB"
                    #$ingestionResult = Submit-MvDamIngestionBatch -IngestionRequests $batch
                    $initializedBatch = $null
                    if (($UseAzCopy) -and ($fileclass -in @('J','K','L','M')) )
                    {
                        $initializedBatch = Initialize-MvDamIngestionBatch -IngestionRequests $batch -NoBlobCopy
                        $initializedBatch = Copy-MvDamIngestionBatchBlobs -IngestionBatch $initializedBatch
                    }
                    else
                    {
                        $initializedBatch = Initialize-MvDamIngestionBatch -IngestionRequests $batch 
                    }

                    #Check if access token still valid even for a long running copy task
                    if ((Get-Date).AddMinutes(45) -ge $MvDamContext.ClientRunspace.AccessTokenExpiry)
                    {
                        $region = $MvDamContext.ClientRunspace.Region.ToString().ToLower().Insert(2,'-')
                        $password = $pass = ConvertTo-SecureString -String $MvDamContext.ClientRunspace.Password -AsPlainText -Force
                        Connect-MvDamAccount -Username $MvDamContext.ClientRunspace.Username -Region $region -Password $pass
                    }
                    $ingestionResult = Complete-MvDamIngestionBatch -IngestionRequests $initializedBatch

                    #Get stats
                    $batchEnd = Get-Date
                    $elapsedSecs = ($batchEnd - $batchStart).TotalSeconds
                    $uploadLimitCounter += $batch.Count
                    $batchFileSizeMBCopied = [Math]::Round(($ingestionResult | Where {$_.Status.StartsWith('Success')} | Measure-Object -Property 'FileSize' -Sum).Sum/1024/1024, 4)
                    $batchFileSuccessCount = ($ingestionResult | Where {$_.Status.StartsWith('Success')}).Count
                    $batchFileFailCount = ($ingestionResult | Where {$_.Status.StartsWith('Fail')}).Count
                    $jobClass.CurrentBatchFileSizeMB = $batchFileSizeMBCopied
                    $jobClass.CurrentBatchFileCount = $batchFileSuccessCount
                    $batchSummary = [PSCustomObject]@{BatchId=$batchId;TransferredMB=$batchFileSizeMBCopied;SuccessCount=$batchFileSuccessCount;FailCount=$batchFileFailCount;StartTime=$batchStart;EndTime=$batchEnd}
                    $batchSummary | Export-Csv -Path $jobSummaryFilePath -Encoding utf8 -Append -NoTypeInformation

                    #Log the result
                    if (($linesLogged + $ingestionResult.Count) -gt $maxLinesPerLogFile)
                    {
                        $linesLogged = 0
                        $logFileCounter++
                    } 
                    $logFilePath = [System.IO.Path]::Combine($logPath, "asset-import-log-$($logFileCounter.ToString("00000")).csv")
                    $ingestionResult | Select-Object @{Label='BatchId';Expression={$batchId}}, RefId, AssetId, `
                        @{Label='CateoryIds';Expression={$_.CategoryIds -join ','}}, `
                        Filename, Filesize, Status, SrcSasUrl, DestSasUrl, `
                        @{Label='StartedOn';Expression={$_.PerfStats.StartTime}},  `
                        @{Label='CreatedOn';Expression={$_.PerfStats.CreatedTime}},  `
                        @{Label='BlobCopiedOn';Expression={$_.PerfStats.CopiedTime}},  `
                        @{Label='TitledOn';Expression={$_.PerfStats.TitledTime}},  `
                        @{Label='CategorizedOn';Expression={$_.PerfStats.ClassifiedTime}},  `
                        @{Label='EndedOn';Expression={$_.PerfStats.EndTime}}  |`
                        Export-Csv -Path $logFilePath -Encoding utf8 -Append -NoTypeInformation
                    $linesLogged =+ $ingestionResult.Count

                    #update job master
                    $jobmaster | ConvertTo-Json | Out-File  $JobFolder\BatchJobMaster.json -Encoding utf8

                    #move pending file to completed or faulted
                    $destinationPath = [System.IO.Path]::Combine($completedJobFolder, $batchId + ".csv")
                    if ($batchFileFailCount -gt 0)
                    {
                        $destinationPath = [System.IO.Path]::Combine($faultedJobFolder, $batchId + ".csv")        
                    }
                    Move-Item -Path $batchFilePath -Destination $destinationPath

                    Write-Host "$batchId completed in $($elapsedSecs)s. $batchFileSuccessCount successfully submitted. $batchFileFailCount failed. `n"

                    $uploadLimitDuration += $elapsedSecs
                    if ($uploadLimitCounter -ge $maxUploadsPerMin)
                    {
                        if ($uploadLimitDuration -lt 60){
                            Write-Host "Throttling upload requests for $(60-$uploadLimitDuration)s...."
                            Start-Sleep -Seconds (60-$uploadLimitDuration)
                        }
                        $uploadLimitCounter = 0
                        $uploadLimitDuration = 0    
                    }

                    $currentBatchCount++
               }
               #Distribute ingestion workload
               $pullFromTop = -not $pullFromTop
               if ($pullFromTop) {
                    $fileclassCode = $jobMaster | Where-Object {($_.TotalBatchCount -gt 0) -and ($_.CurrentBatchCount -le $_.TotalBatchCount)}  | Sort-Object TargetFrequencySecs, {($_.CurrentBatchCount/$_.TotalBatchCount)} | Select-Object FileClass -First 1
               }
               else {
                    $fileclassCode = $jobMaster | Where-Object {($_.TotalBatchCount -gt 0) -and ($_.CurrentBatchCount -le $_.TotalBatchCount)}  | Sort-Object TargetFrequencySecs, {($_.CurrentBatchCount/$_.TotalBatchCount)} | Select-Object FileClass -Last 1       
               }
            } #endwhile

        }
        catch
        {
            $jobResult = -1
        }
        return $jobResult
    }
}

function Copy-MvDamIngestionBatchBlobs {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [object[]]$IngestionBatch
    )
    PROCESS {
        foreach($request in $IngestionBatch)
        {
            $startTime = Get-Date
            $srcSasUrl = $request.SrcSasUrl.Replace("'","''")
            $dstSasUrl = $request.DestSasUrl.Replace("'","''")
            $copyCmd = "azcopy copy '$srcSasUrl' '$dstSasUrl' --output-type json"
            $copyResult = Invoke-Expression $copyCmd
            $initResults = $copyResult | ConvertFrom-Json | Where {$_.MessageType -EQ 'Init'} | foreach {$_.MessageContent} | ConvertFrom-Json 
            $jobResults = $copyResult | ConvertFrom-Json | Where {$_.MessageType -EQ 'EndOfJob'} |  foreach {$_.MessageContent} | ConvertFrom-Json
            if ($jobResults[0].JobStatus.ToLower() -eq 'completed')
            {
                $request.Status = 'Success (Copy Blob)'
                $cleanupResult = Invoke-Expression  "azcopy.exe jobs remove $($initResults[0].JobID)"
            }
            else
            {
                $request.Status = 'Fail (Copy Blob)'
            }
            $request.PerfStats.CopiedTime = Get-Date
            $elapsedSecs = ($request.PerfStats.CopiedTime - $startTime).TotalSeconds
            Write-Host " >> azcopy $($jobResults[0].JobStatus.ToLower()) $($request.Filename) ($(($request.Filesize/1024/1024).ToString("0.0000")) MB) in $elapsedSecs seconds"

        }
        return $IngestionBatch
    }
}

function Test-MvDamManifestImport {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $SrcSasUri,
        [parameter(Mandatory=$false,Position=1)]
        [string] $ParentCategoryPath = '\Root',
        [parameter(Mandatory=$true,Position=2)]
        [string] $JobFolder
        )
    PROCESS {

        #Valid container SAS url
        $uriResult = $null
        $result = [Uri]::TryCreate($SrcSasUri, [UriKind]::Absolute, [ref] $uriResult)
        if ($result)
        {
            #check if signature is present
            if ([string]::IsNullOrEmpty($uriResult.Query))
            {
                Write-Warning "Invalid blob container SAS URI specified. Missing SAS Signature."
                $result = $false
            }
        }
        else
        {
            Write-Warning "Invalid blob container SAS URI specified."
        }

        #Validate that the Az.Storage module is installed
        if ($null -eq (Get-Module -Name Az.Storage -ListAvailable))
        {
            Write-Warning "Az.Storage Powershell Module is not installed. Please run 'Install-Module -Name Az -AllowClobber' as Administrator."
            $result = $false
        } 

        #Validate MvDamContext
        if ($null -eq $MvDamContext)
        {
            Write-Warning "You are not connected to a MediaValet DAM Account. Please run 'Connect-MvDamAccount' to connect to an account."
            $result = $false
        }

        if ((Get-MvDamContext).FeatureFlags['FileNameVersioning'] -eq 'Active')
        {
            Write-Warning "Pre-requisite not met: FilenNameVersioning should be inactive when performing a batch upload from a container"   
            $result = $false
        }


        #validate ParentCategoryPath
        $parentCategory = Get-MvDamCategory -CategoryPath $ParentCategoryPath 
        if ($null -eq $parentCategory)
        {
            Write-Warning "ParentCategoryPath does not exist. Please create before proceeding."
            $result = $false
        }

        #ensure that we have a clean jobfolder
        if ((Test-Path $JobFolder) -and ((Get-ChildItem -Path $JobFolder).Count -gt 0))
        {
            Write-Warning "JobFolder $JobFolder is not empty. Please use a clean job folder for every job."
            $result = $false
        }

        return $result;

    }
}

function Import-MvDamAssetsFromContainer  {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $SrcSasUri,
        [parameter(Mandatory=$false,Position=1)]
        [string] $ParentCategoryPath = '\Root',
        [parameter(Mandatory=$true,Position=2)]
        [string] $JobFolder
        )
    PROCESS {
        if (Test-MvDamManifestImport -SrcSasUri $SrcSasUri -ParentCategoryPath $ParentCategoryPath -JobFolder $JobFolder)
        {
            $result = Export-MvDamContainerManifest -SrcSasUri $SrcSasUri -ParentCategoryPath $ParentCategoryPath -JobFolder $JobFolder
            if ($result -ne 0)
            {
                Write-Warning "Error encountered exporting container manifest. Stopping job execution."
                return
            }

            $result = Import-MvDamCategoryTreeFromManifest -JobFolder $JobFolder
            if ($result -ne 0)
            {
                Write-Warning "Error encountered creating categories. Stopping job execution."
                return
            }

            $result = Initialize-MvDamUploadJob -JobFolder $JobFolder
            if ($result -ne 0)
            {
                Write-Warning "Error preparing DAM ingestion plan. Stopping job execution."
                return
            }

            $result = Start-MvDamUploadJob -JobFolder $JobFolder
            if ($result -ne 0)
            {
                Write-Warning "Error(s) encountered executing the ingestion plan. Please review the logs at $JobFolder\Logs."
                return
            }
        }
        else
        {
            Write-Warning "Import-MvDamAssetsFromContainer pre-flight check failed. Stopping job execution."
            return
        }
    }
}

 function Reset-MvDamUploadJob {
    [CmdletBinding()]
    param (
        [parameter(Mandatory=$true,Position=0)]
        [string] $JobFolder
        )
    PROCESS 
    {
        $pendingJobFolder = [System.IO.Path]::Combine($JobFolder,'PendingBatches')
        $completedJobFolder = [System.IO.Path]::Combine($JobFolder,'CompletedBatches')
        $faultedJobFolder = [System.IO.Path]::Combine($JobFolder,'FaultedBatches')


        $jobMasterFilePath = [System.IO.Path]::Combine($JobFolder,'BatchJobMaster.json')
        $jobMaster = (Get-Content -Path $jobMasterFilePath -Encoding UTF8 |  Out-String | ConvertFrom-Json)
        foreach($fileclass in $jobMaster)
        {
            $firstPendingBatchId = $lastPendingBatchId = $lastCompletedBatchId =$lastFailedBatchId = 0
            $fileClassCode = $fileclass.FileClass
            $pendingBatches = Get-ChildItem -Path $pendingJobFolder | Where {$_.Name -like "$fileClassCode-*.csv"} | Sort-Object -Property Name
            $firstPendingBatch =  $pendingBatches  | Select Name -First 1
            if (-not [string]::IsNullOrEmpty($firstPendingBatch))
            {
                 $firstPendingBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($firstPendingBatch.Name)).Substring(2))
            }
            $lastPendingBatch = $pendingBatches | Select Name -Last 1
            if (-not [string]::IsNullOrEmpty($lastPendingBatch))
            {
                $lastPendingBatchId =  [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($lastPendingBatch.Name)).Substring(2))
            }
            $lastCompletedBatch = Get-ChildItem -Path $completedJobFolder | Where {$_.Name -like "$fileClassCode-*.csv"} | Sort-Object -Property Name  | Select Name -Last 1
            if (-not [string]::IsNullOrEmpty($lastCompletedBatch))
            {
                $lastCompletedBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($lastCompletedBatch.Name)).Substring(2))
            }
            $lastFailedBatch = Get-ChildItem -Path $faultedJobFolder | Where {$_.Name -like "$fileClassCode-*.csv"} | Sort-Object -Property Name  | Select Name -Last 1
            if (-not [string]::IsNullOrEmpty($lastFailedBatch))
            {
                $lastFailedBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($lastFailedBatch.Name)).Substring(2))
            }
            Write-Host "Resetting FileClass $fileClassCode First Pending: $firstPendingBatchId Last Pending: $lastPendingBatchId Completed: $lastCompletedBatchId Faulted: $lastFailedBatchId"
            $fileclass.CurrentBatchCount = [Math]::Max($lastCompletedBatchId, $lastFailedBatchId)
            $fileclass.CurrentBatchFileCount= 0
            $fileclass.CurrentBatchFileSizeMB = 0
            $fileclass.TotalBatchCount = [Math]::Max($lastPendingBatchId, [Math]::Max($lastCompletedBatchId, $lastFailedBatchId))
        } 
        Write-Host "Updating job master file at $jobMasterFilePath" 
        $jobmaster | ConvertTo-Json | Out-File  $JobFolder\BatchJobMaster.json -Encoding utf8
     }
 }

 function Remove-MvDamInvalidFileNameChars {
    param(
        [Parameter(Mandatory=$true,
        Position=0,
        ValueFromPipeline=$true,
        ValueFromPipelineByPropertyName=$true)]
        [String]$Name
        )

    $invalidChars = [IO.Path]::GetInvalidFileNameChars() -join ''
    $re = "[{0}]" -f [RegEx]::Escape($invalidChars)
    $newName = [Regex]::Replace(($Name -replace $re), "[^\u0000-\u007F]+",'')
    $newName = $newName.Replace(':',' ').Replace('?',' ').Replace('*',' ').Replace('"','')
    return ($newName)
}

function Copy-MvDamUploadFailuresForRetry {
    param(
      [Parameter(Mandatory=$true,
        Position=0)]
      [String]$srcJobFolder,
            [Parameter(Mandatory=$true,
        Position=0)]
      [String]$dstJobFolder
    )


    if (!(Test-Path -Path $dstJobFolder))
    {
        New-Item -Path $dstJobFolder -ItemType Directory
        New-Item -Path "$dstJobFolder\Logs" -ItemType Directory
    }

   if (!(Test-Path -Path "$dstJobFolder\Logs"))
    {
        New-Item -Path "$dstJobFolder\Logs" -ItemType Directory
    }
    $assetManifest = Import-Csv -Path ($srcJobFolder+"\asset-manifest.csv")
    $processedItems = [System.Collections.ArrayList] @()

    #Copy pending assets for upload
    $pendingFilesFolder = [System.IO.Path]::Combine($srcJobFolder, "PendingBatches")
    $pendingFiles = Get-ChildItem -Path $pendingFilesFolder -File -Filter "*.csv"
    foreach($pendingFile in $pendingFiles)
    {
        $pendingItems = Import-Csv -Path $pendingFile.FullName
        foreach ($pendingItem in $pendingItems)
        {
            if (-not $processedItems.Contains($pendingItem.SourceUrl))
            {
                $asset = $assetManifest | Where-Object -Property SasUrl -eq $pendingItem.SourceUrl
                if ($asset -ne $null)
                {
                    Write-Host "Copying pending manifest record $($asset.Filename)"
                    $asset | Select CategoryPath, @{Label='Filename';Expression={Remove-MvDamInvalidFileNameChars $_.Filename}}, Filesize, ContentType, LastModified, SasUrl  | Export-Csv -Path ($dstJobFolder+"\asset-manifest.csv") -Encoding utf8 -Append -NoTypeInformation
                    $processedItems.Add($pendingItem.SourceUrl)
                }
            }
            else
            {
                Write-Warning "Skipping pending manifest record $($pendingFile.Filename) since it has been processed already."
            }
        }
    }

    #Copy failed assets for retry
    $logFilePath = [System.IO.Path]::Combine($srcJobFolder, "Logs")
    $logFiles = Get-ChildItem -Path $logFilePath -Filter "asset-import-log-*.csv"
    $importedLogFile = "$dstJobFolder\Logs\imported-asset-import-log.csv"
    foreach($logFile in $logFiles)
    {
        $failures = Import-Csv -Path $logFile.FullName | Where {($_.Status -like 'Fail*')}
        if (($failures -ne $null) -and ($failures.Count -gt 0))
        {
            $failures | Export-Csv -Path $importedLogFile -Encoding utf8 -NoTypeInformation -Append 
        }
    }
    $failedUploads= Import-Csv -Path $importedLogFile
    foreach ($failedUpload in $failedUploads)
    {
        $asset = $assetManifest | Where-Object -Property SasUrl -eq $failedUpload.SrcSasUrl
        if ($asset -ne $null)
        {
            if (-not $processedItems.Contains($failedUpload.SrcSasUrl))
            {
                $newFileName = Remove-MvDamInvalidFileNameChars($asset.Filename)
                if ($newFileName -ne $asset.Filename)
                {
                    Write-Host "Renaming $($asset.Filename) to $newFileName in manifest"
                }
                else
                {
                    Write-Host "Copying manifest record $($asset.Filename)"
                }
                $asset.Filename = $newFileName
                $asset | Export-Csv -Path ($dstJobFolder+"\asset-manifest.csv") -Encoding utf8 -Append -NoTypeInformation 
                $processedItems.Add($failedUpload.SrcSasUrl)
            }
            else
            {
                Write-Warning "Skipping manifest record $($asset.Filename) since it has already been processed."
            }

        }
        else
        {
            Write-Warning "Asset $($failedUpload.Filename)[$($failedUpload.RefId)] not found. "
        }
    }

    Copy-Item -Path ($srcJobFolder+"\category-manifest.csv") -Destination ($dstJobFolder +"\category-manifest.csv")
    Copy-Item -Path ($srcJobFolder+"\Logs\category-import-log.csv") -Destination ($dstJobFolder+"\Logs\category-import-log.csv")
}