MediaValet.DAM.psm1
|
function Import-MvDamFilenameListFromCsv { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$Path ) PROCESS { $csv = Import-Csv -Path $Path $filenameArray = $csv | Foreach {$_.Filename} return $filenameArray } } function Export-MvDamAssetInfo { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0,ParameterSetName="From Filename List")] [string[]]$FilenameList, [parameter(Mandatory=$true,Position=0,ParameterSetName="From Asset ID List")] [string[]]$AssetIdList, [parameter(Mandatory=$true, Position=0,ParameterSetName="All Assets")] [switch]$ListAllAssets, [parameter(Mandatory=$true,Position=1)] [string]$Path, [parameter(Mandatory=$false, Position=2)] [switch]$Resume ) PROCESS { if ($ListAllAssets) { if (!$Resume) { $assets = Find-MvDamAssetInfo -SearchQuery "*&sort=record.createdAt+A" -Count 1000 } else { $data = Import-Csv $Path $result = $data[$data.Count - 1] $lastCreatedDate=$result."System.CreatedDate" $assets = Find-MvDamAssetInfo -SearchQuery "*&sort=record.createdAt+A&filters=(DateUploaded+GT+$lastCreatedDate)" -Count 1000 } $assetCtr = $assets.Count while ($assets.Count -gt 0) { $firstCreatedDate = $assets[0].CreatedDate.ToString('o') $lastCreatedDate = $assets[$assets.Length-1].CreatedDate.ToString('o') Write-Host "Retrieved $assetCtr assets: " $firstCreatedDate "to" $lastCreatedDate $convertedAssets = @() foreach ($asset in $assets) { $convertedAssets += ConvertAssetToCSVExport $asset } $convertedAssets | Export-CSV -Path $Path -Encoding Default -Append -NoTypeInformation $filter = "*&sort=record.createdAt+A&filters=(DateUploaded+GT+$lastCreatedDate)" $assets = Find-MvDamAssetInfo -SearchQuery $filter -Count 1000 $assetCtr = $assetCtr + $assets.Count } } else { $assets = @() if ($FilenameList) { $assets = Get-MvDamAssetInfo -FilenameList $FilenameList } if ($AssetIdList) { $assets = Get-MvDamAssetInfo -AssetIds $AssetIdList } $convertedAssets = @() foreach ($asset in $assets) { $convertedAssets += ConvertAssetToCSVExport $asset } $convertedAssets | Export-CSV -Path $Path -Encoding Default -Append -NoTypeInformation } } } function Export-MvDamAssetAttribute { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [object[]]$AssetInfoList, [parameter(Mandatory=$false,Position=1)] [string[]]$AttributeList, [parameter(Mandatory=$true,Position=2)] [string]$Path ) PROCESS { if ($AssetInfoList.GetType().Name -ne "AssetInfoModel[]" -or $AssetInfoList.Count -eq 0) { Write-Error -Message "Please pass an AssetInfoModel array with one or more items to the `$AssetInfoList parameter" } #Get AttributeList for Library if ($AttributeList -eq $null) { $AttributeList = (Get-MvDamAttributeDef) | Foreach {if ($_.IsSystemProperty){"System."+$_.Name}else{$_.Name}} } else #validate the AttributeList against the AttributeDefs { $objAttrNames = (Get-MvDamAttributeDef) | Foreach {if ($_.IsSystemProperty){"System."+$_.Name}else{$_.Name}} $AttributeList | Foreach { if (-not $objAttrNames.Contains($_)) {Write-Error "Invalid attribute name $_ specified"} } } $assets = @() Foreach ($assetInfo in $AssetInfoList) { $asset = New-Object -TypeName PSObject $asset | Add-Member -MemberType NoteProperty -Name 'System.Id' -Value $assetInfo.Id $asset | Add-Member -MemberType NoteProperty -Name 'System.FileName' -Value $assetInfo.FileName $asset | Add-Member -MemberType NoteProperty -Name 'System.Title' -Value $assetInfo.Title $asset | Add-Member -MemberType NoteProperty -Name 'System.Description' -Value $assetInfo.Description Foreach($attributeName in $AttributeList) { if ($attributeName -notlike "System.*") { $attrValue = $assetInfo.Attributes | Where {$_.AttributeName -eq $attributeName} | Select -Property AttributeValue $asset | Add-Member -MemberType NoteProperty -Name "$attributeName" -Value $attrValue.AttributeValue } } $assets += $asset } $assets | Export-Csv -Path $Path -Encoding Default -Append -NoTypeInformation return $assets } } function Import-MvDamAssetAttributesFromCsv { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$SourcePath, [parameter(Position=1)] [string]$LogPath ) PROCESS { $csv = @() $import = Import-Csv -Path $SourcePath -Encoding Default if ($import.GetType().ToString() -eq "System.Object[]") { $csv = $import } else { $csv = @($import) } for($i=0;$i -lt $csv.Length; $i++) { $line = $csv[$i] $assetId = $line.('System.Id') $percentComplete = ($i/$csv.Length)*100 Write-Progress -Activity "Importing Asset Attributes" -Status "Updating attributes for Asset Id $assetId" -PercentComplete $percentComplete $txnInfo = New-MvDamTxnInfo -StartTime (Get-Date) -Id $assetId -Status "InProgress" -TxnType "Update-MvDamAssetAttribute" $attrKvps = @{} $line | Get-Member | Where {$_.MemberType -eq "NoteProperty"} | Foreach {if ($line.($_.Name) -ne "") {$attrKvps.Add($_.Name, $line.($_.Name))}} $errorInfo = $null $txnResult = Update-MvDamAssetAttribute -AssetId $assetId -Attributes $attrKvps -ErrorAction Continue -ErrorVariable $errorInfo $txnInfo.EndTime = Get-Date if ($errorInfo) { $txnInfo.Status = "Failed" $txnInfo.Notes = $errorInfo.Message } else { $txnInfo.Status = "Succeeded" } if ($LogPath) { $txnInfo | Export-Csv -Path $LogPath -Encoding Default -Append } } Write-Progress -Activity "Importing Asset Attributes" -Completed return } } function Test-MvDamAssetAttributeCsv { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$SourcePath, [parameter(Mandatory=$true,Position=1)] [string]$ResultPath ) PROCESS { $csv = @() $import = Import-Csv -Path $SourcePath if ($import.GetType().ToString() -eq "System.Object[]") { $csv = $import } else { $csv = @($import) } $assetsWithErrors = 0 $totalErrors = 0 for($i=0;$i -lt $csv.Length; $i++) { $line = $csv[$i] $assetId = $line.('System.Id') $percentComplete = ($i/$csv.Length)*100 Write-Progress -Activity "Validating Asset Attributes" -Status "Testing attributes for Asset Id $assetId" -PercentComplete $percentComplete $attrKvps = @{} $line | Get-Member | Where {$_.MemberType -eq "NoteProperty"} | Foreach {if ($line.($_.Name) -ne "") {$attrKvps.Add($_.Name, $line.($_.Name))}} $errorInfo = $null $txnResult = Test-MvDamAssetAttribute -AssetId $assetId -Attributes $attrKvps -ErrorAction Continue -ErrorVariable $errorInfo if ($txnResult.Errors -eq 0) { $line | Add-Member -MemberType NoteProperty -Name 'Errors' -Value "" } else { $assetsWithErrors++ $totalErrors += $txnResult.Errors $errorMsgs = ($txnResult.AttributeValidatonResults | Foreach ValidationResult) -join "; " $line | Add-Member -MemberType NoteProperty -Name 'Errors' -Value $errorMsgs } if ($i -eq 0) { $line | Export-Csv -Path $ResultPath -Encoding Default -NoTypeInformation } else { $line | Export-Csv -Path $ResultPath -Encoding Default -Append -NoTypeInformation } } Write-Progress -Activity "Validating Asset Attributes" -Completed Write-Host "Total Assets Validated:"$csv.Length Write-Host "Assets with Error(s):"$assetsWithErrors Write-Host "Total Errors Detected"$totalErrors if ($totalErrors -eq 0) { Write-Host "`r`nCONGRATULATIONS! Your asset attribute CSV file is ready to import." } return } } function Import-MvDamAssetKeywordFromCsv { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$Path ) PROCESS { $Assets = Import-Csv -Path $Path | Select 'System.Id', Keywords "Validating Ids" $Count = 0 foreach ($id in ($Assets | select Id)) { if ($id -match '<<[a-z]*>>') { $Count += 1; } } if ($Count -gt 0) { throw "$Count issues were detected with the Asset Ids provided. Please resolve this before continuing." } "Ids are valid" "Starting keyword import" $err = @() $lastErrorCount = 0 $errorLogPath = ".\UpdateKeywordErrors.csv" foreach ($asset in $Assets) { "AssetId: {0}" -f $asset.'System.Id' if(!$asset.Keywords.Equals("")) { "Keywords: $asset.Keywords" $keywords = $asset.Keywords.Split(",") Update-MvDamAssetKeyword -AssetId $asset.'System.Id' -Keywords $keywords -ErrorAction Continue -ErrorVariable +err if ($err.Count -gt $lastErrorCount) { $updateKeywordError = New-Object -TypeName PSObject $updateKeywordError | Add-Member -NotePropertyName "System.Id" -NotePropertyValue $asset.'System.Id' $updateKeywordError | Add-Member -NotePropertyName "Keywords" -NotePropertyValue $asset.Keywords $updateKeywordError | Add-Member -NotePropertyName "Error" -NotePropertyValue $err[$lastErrorCount] $updateKeywordError | Export-Csv -Path $errorLogPath -Encoding Default -NoTypeInformation -Append $lastErrorCount += 1 } } } "Import complete" } } function Import-MvDamAssetCategoryFromCsv { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$Path, [parameter(Mandatory=$false)] [string]$ErrorLogPath = ".\ImportAssetCategoryErrorLog.csv" ) PROCESS { $Assets = Import-Csv -Path $Path | Select 'System.Id', Path Test-MvDamSystemId -Ids ($Assets | Select -ExpandProperty System.Id) $err = @() $lastErrorCount = 0 foreach ($asset in $Assets) { if(!$asset.Path.Equals("")) { $asset $category = Get-MvDamCategory -CategoryPath $asset.Path -ErrorAction Continue -ErrorVariable +err Update-MvDamAssetCategory -AssetId $asset.'System.Id' -CategoryIds $category.Id -ErrorAction Continue -ErrorVariable +err if ($err.Count -gt $lastErrorCount) { $updateCategoryError = New-Object -TypeName PSObject $updateCategoryError | Add-Member -NotePropertyName "System.Id" -NotePropertyValue $asset.'System.Id' $updateCategoryError | Add-Member -NotePropertyName "Path" -NotePropertyValue $asset.Path $updateCategoryError | Add-Member -NotePropertyName "Error" -NotePropertyValue $err[$lastErrorCount] $updateCategoryError | Export-Csv -Path $ErrorLogPath -Encoding Default -NoTypeInformation -Append $lastErrorCount += 1 } } } } } function Import-MvDamAssetStatusFromCsv { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$Path, [parameter(Mandatory=$false)] [string]$ErrorLogPath = ".\ImportAssetStatusErrorLog.csv" ) PROCESS { $Assets = Import-CSV -Path $Path | Select 'System.Id', Status Test-MvDamSystemId -Ids ($Assets | Select -ExpandProperty 'System.Id') $err = @() $lastErrorCount = 0 foreach ($asset in $Assets) { if(!$asset.Status.Equals("")) { Update-MvDamAssetStatus -AssetId $asset.'System.Id' $asset.Status -ErrorAction Continue -ErrorVariable +err } if ($err.Count -gt $lastErrorCount) { $updateCategoryError = New-Object -TypeName PSObject $updateCategoryError | Add-Member -NotePropertyName "System.Id" -NotePropertyValue $asset.'System.Id' $updateCategoryError | Add-Member -NotePropertyName "Status" -NotePropertyValue $asset.Status $updateCategoryError | Add-Member -NotePropertyName "Error" -NotePropertyValue $err[$lastErrorCount] $updateCategoryError | Export-Csv -Path $ErrorLogPath -Encoding Default -NoTypeInformation -Append $lastErrorCount += 1 } } } } function Test-MvDamSystemId { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string[]]$Ids ) PROCESS { $Count = 0 foreach ($id in $Ids) { try { [System.Guid]::Parse($id) | Out-Null } catch { $Count += 1 } } if ($Count -gt 0) { throw "$Count issues were detected with the System.Ids provided. Please resolve this before continuing." } else { "Validation complete. No issues were detected." } } } function Initialize-MvDamUploadBatch { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$SourcePath, [parameter(Mandatory=$true,Position=1)] [string]$ManifestFilePath ) PROCESS { $AttributeList = (Get-MvDamAttributeDef) | Foreach {if ($_.IsSystemProperty){"System."+$_.Name}else{$_.Name}} if ($AttributeList -eq $null) { return } Write-Progress -Activity "Reading folder structure..." -Status "Initializing" $files = Get-ChildItem -Path $SourcePath -File -Recurse $ctr = 0 $md5 = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider foreach($file in $files) { $ctr++ Write-Progress -Activity "Adding $file to manifest" -Status "In-Progress" -PercentComplete ($ctr/$files.Count*100) #Write-Host $file.DirectoryName, $file.Name, $file.Basename $file.Length, $file.CreationTimeUtc, $file.LastWriteTimeUtc $relPath = $file.DirectoryName.Replace($SourcePath, "") if (!($env:OS).StartsWith("Windows")) { $relPath = $relPath -replace '/','\' } $hash = [System.Convert]::ToBase64String($md5.ComputeHash([System.IO.File]::ReadAllBytes($file.FullName))) $asset = New-Object -TypeName PSObject $asset | Add-Member -MemberType NoteProperty -Name 'System.Id' -Value "" $asset | Add-Member -MemberType NoteProperty -Name 'Process.LocalPath' -Value $file.DirectoryName $asset | Add-Member -MemberType NoteProperty -Name 'Process.LocalName' -Value $file.Name $asset | Add-Member -MemberType NoteProperty -Name 'System.FileName' -Value $file.Name $asset | Add-Member -MemberType NoteProperty -Name 'System.CategoryPath' -Value $relPath $asset | Add-Member -MemberType NoteProperty -Name 'System.Title' -Value $file.Basename $asset | Add-Member -MemberType NoteProperty -Name 'System.MD5Checksum' -Value $hash $asset | Add-Member -MemberType NoteProperty -Name 'Process.PreflightStatus' -Value "Pending" $asset | Add-Member -MemberType NoteProperty -Name 'Process.PreflightInfo' -Value "" $asset | Add-Member -MemberType NoteProperty -Name 'Process.UploadStatus' -Value "Preflight" $asset | Add-Member -MemberType NoteProperty -Name 'Process.UploadInfo' -Value "" $asset | Add-Member -MemberType NoteProperty -Name 'System.File Size' -Value $file.Length $asset | Add-Member -MemberType NoteProperty -Name 'System.Created Date' -Value $file.CreationTimeUtc $asset | Add-Member -MemberType NoteProperty -Name 'System.Last Modified' -Value $file.LastWriteTimeUtc $asset | Add-Member -MemberType NoteProperty -Name 'System.Description' -Value "" $asset | Add-Member -MemberType NoteProperty -Name 'System.Keywords' -Value "" Foreach($attributeName in $AttributeList) { if ($attributeName -notlike "System.*") { $attrValue = $assetInfo.Attributes | Where {$_.AttributeName -eq $attributeName} | Select -Property AttributeValue $asset | Add-Member -MemberType NoteProperty -Name "$attributeName" -Value $attrValue.AttributeValue } } if ($PSCmdlet.MyInvocation.BoundParameters["Verbose"].IsPresent -eq $true) { Write-Host "Adding to manifest: $asset `r`n" } $asset | Export-Csv -Path $ManifestFilePath -Encoding Default -Append -NoTypeInformation } Write-Progress -Activity "Finalizing" -PercentComplete 100 -Completed } } function Test-MvDamUploadBatch { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string]$ManifestFilePath, [parameter(Mandatory=$false,Position=1)] [Guid]$ParentCategoryId, [parameter(Mandatory=$false,Position=2)] [switch]$AutoCreateCategories ) PROCESS { } } function ConvertAssetToCSVExport ($assetInfo) { $asset = New-Object -TypeName PSObject foreach ($property in $assetInfo.PSObject.Properties) { if (!$property.Name.Equals("Attributes")) { if($property.Name.Equals("Keywords") -or $property.Name.Equals("Categories")) { $asset | Add-Member -MemberType NoteProperty -Name ("System." + $property.Name) -Value ($property.Value -join ", ") } elseif ($property.Value -is [DateTime]) { $asset | Add-Member -MemberType NoteProperty -Name ("System." + $property.Name) -Value ($property.Value.ToString("o", $null)) } else { $asset | Add-Member -MemberType NoteProperty -Name ("System." + $property.Name) -Value $property.Value } } } foreach($attribute in $assetInfo.Attributes) { if (!$attribute.IsSystemProperty) { $attrValue = $assetInfo.Attributes | Where {$_.AttributeName -eq $attribute.AttributeName} | Select -ExpandProperty AttributeValue $asset | Add-Member -MemberType NoteProperty -Name $attribute.AttributeName -Value $attrValue } } return $asset } function Import-MvDamCategoryTreeFromManifest { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0,ParameterSetName="By File Path")] [string]$ManifestFilePath, [parameter(Mandatory=$true,Position=1,ParameterSetName="By File Path")] [string]$LogPath, [parameter(Mandatory=$true,Position=1,ParameterSetName="By Job Folder")] [string]$JobFolder ) PROCESS { $jobResult = 0; $JobStartTime = Get-Date $RootCategoryName = (Get-MvDamCategory -RootCategory).Name if ($PSCmdlet.ParameterSetName -eq "By Job Folder") { $ManifestFilePath = [System.IO.Path]::Combine($JobFolder,'category-manifest.csv') $LogFolder = [System.IO.Path]::Combine($JobFolder,'Logs') if (-not(Test-Path $LogFolder)) { New-Item -Path $LogFolder -ItemType Directory > $null } $LogPath = [System.IO.Path]::Combine($JobFolder,'Logs','category-import-log.csv') } try { $categories = Import-Csv -Path $ManifestFilePath $ctr = 0 ForEach($category in $categories) { Write-Progress -Activity 'Creating categories from import file...' -PercentComplete ($ctr/$categories.Length*100) $result = New-Object PSObject $result | Add-Member -NotePropertyName 'CategoryTreePath' -NotePropertyValue $category.CategoryPath $result | Add-Member -NotePropertyName 'Status' -NotePropertyValue 'Pending' $result | Add-Member -NotePropertyName 'CategoryId' -NotePropertyValue ([System.Guid]::Empty).ToString() $result | Add-Member -NotePropertyName 'BatchStarted' -NotePropertyValue $JobStartTime $result | Add-Member -NotePropertyName 'Submitted' -NotePropertyValue (Get-Date -Format FileDateTimeUniversal) $result | Add-Member -NotePropertyName 'Completed' -NotePropertyValue '' $result | Add-Member -NotePropertyName 'Notes' -NotePropertyValue '' if (($category.CategoryPath.ToLower().StartsWith($RootCategoryName.ToLower())) -or ($category.CategoryPath.ToLower().StartsWith('\root\'))) { $opsErr = $null #Check if access token still valid if ((Get-Date).AddMinutes(5) -ge $MvDamContext.ClientRunspace.AccessTokenExpiry) { Connect-MvDamAccount -Username $MvDamContext.ClientRunspace.Username -Region $region -Password $MvDamContext.ClientRunspace.Password } $catOperation = New-MvDamCategory -Name $category.CategoryPath -UseRootPath -CreateTreeIfNotExists -ErrorAction SilentlyContinue -ErrorVariable opsErr if ($opsErr.Count -eq 0) { $leafCat = $catOperation[$catOperation.count -1] $result.CategoryId = $leafCat.Id $result.Status = 'Success' $result.Completed = Get-Date -Format FileDateTimeUniversal $catOperation | ForEach-Object {Write-Host "Created category $($_.Tree.Path)"} } else { $result.Status = 'Fail' $result.Notes = $opsErr[0].InnerException.InnerException.Message } $catOperation = $null } else { $result.Status = 'Failed' $result.Notes = "Invalid category path." } $result | Export-Csv -Path $LogPath -NoTypeInformation -Append -Encoding UTF8 $ctr++ } Write-Progress -Activity 'Creating categories from import file...' -PercentComplete 100 -Completed } catch { $jobResult = -1 } return $jobResult } } function Export-MvDamContainerManifest { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string] $SrcSasUri, [parameter(Mandatory=$false,Position=1)] [string] $ParentCategoryPath = '\Root', [parameter(Mandatory=$true,Position=2)] [string] $JobFolder ) PROCESS { #Validate that the Az.Storage module is installed if ($null -eq (Get-Module -Name Az.Storage -ListAvailable)) { Write-Warning "Az.Storage Powershell Module is not installed. Please run 'Install-Module -Name Az -AllowClobber' as Administrator." return -1 } #Validate MvDamContext if ($null -eq $MvDamContext) { Write-Warning "You are not connected to a MediaValet DAM Account. Please run 'Connect-MvDamAccount' to connect to an account." return -1 } #validate ParentCategoryPath $parentCategory = Get-MvDamCategory -CategoryPath $ParentCategoryPath if ($null -eq $parentCategory) { Write-Warning "ParentCategoryPath does not exist. Please create before proceeding." return -1 } #TODO:make sure that the destination manifest file does not exist $assetManifestFilePath = [System.IO.Path]::Combine($JobFolder, "asset-manifest.csv") $categoryManifestFilePath = [System.IO.Path]::Combine($JobFolder, "category-manifest.csv") if ((Test-Path $assetManifestFilePath) -or (Test-Path $categoryManifestFilePath)) { Write-Warning "Manifest files already exist. Please specify a different JobFolder or clear the specified JobFolder. Stopping execution." return -1 } $sasUri = [System.Uri]$SrcSasUri $storageAccountName = $sasUri.Host.Split('.')[0] $containerName = $sasUri.LocalPath.Remove(0,1) #remove the leading / $containerBaseUri = $sasUri.GetLeftPart(1) $sasToken = $sasUri.Query $storageContext = New-AzStorageContext -StorageAccountName $storageAccountName -SasToken $sasToken -Protocol Https $concurrentTasks = [System.Environment]::ProcessorCount if (-not(Test-Path $JobFolder)) { New-Item -Path $JobFolder -ItemType Directory > $null } $excludedFiles = @('.DS_Store','thumbs.db') $token = $null $categories = [System.Collections.ArrayList] @() $runningTotal = 0 do { Write-Progress -Activity "Retrieved $runningTotal blobs." $blobs = Get-AzStorageBlob -Container $containerName -Context $storageContext -ConcurrentTaskCount $concurrentTasks -MaxCount 10000 -ContinuationToken $token $token = $blobs[$blobs.Count - 1].ContinuationToken $uploadRequests = $blobs | Where {[System.IO.Path]::GetFileName($_.Name) -notin $excludedFiles} | Select-Object @{Label='CategoryPath';Expression={$ParentCategoryPath + '\' + [System.IO.Path]::GetDirectoryName($_.Name)}}, ` @{Label='Filename';Expression={[System.IO.Path]::GetFileName($_.Name)}}, @{Label='Filesize';Expression={$_.Length}} , ContentType, LastModified, ` @{Label='SasUrl';Expression={"$($_.ICloudBlob.Uri.AbsoluteUri)$sasToken"}} $uploadRequests | Export-Csv -Path $assetManifestFilePath -NoTypeInformation -Append -Encoding utf8 $currentCategories = $uploadRequests | Select-Object CategoryPath -Unique if ($null -ne $currentCategories) { $incrementalCategories = $currentCategories | Where-Object {$_.CategoryPath -NotIn $categories.CategoryPath} if ($null -ne $incrementalCategories) { if ($incrementalCategories.GetType().Name -eq 'PSCustomObject') #single object returned, not an array { $categories.Add($incrementalCategories) > $null } else { $categories.AddRange($incrementalCategories) > $null } } } $runningTotal += $blobs.Count } while ($null -ne $token) $categories | Sort-Object -Property CategoryPath | Export-Csv -Path $categoryManifestFilePath -NoTypeInformation -Encoding utf8 return 0 } } function Get-MvDamFileSizeClass { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [long] $FileSize ) PROCESS { $sizeInMB = $fileSize/1024/1024 $fileClass = ` switch ($sizeInMB) { {$_ -le 1} {'A';break;} {$_ -le 2} {'B';break;} {$_ -le 4} {'C';break;} {$_ -le 8} {'D';break;} {$_ -le 16} {'E';break;} {$_ -le 32} {'F';break;} {$_ -le 64} {'G';break;} {$_ -le 128} {'H';break;} {$_ -le 256} {'I';break;} {$_ -le 512} {'J';break;} {$_ -le 1024} {'K';break;} {$_ -le 2048} {'L';break;} {$_ -gt 2048} {'M';break;} } return $fileClass } } function New-MvDamCreateAssetBatchStats { PROCESS { $data = @( [PSCustomobject]@{FileClass='A';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=16;TargetFreqSecs=6} [PSCustomobject]@{FileClass='B';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=16;TargetFreqSecs=10} [PSCustomobject]@{FileClass='C';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=18} [PSCustomobject]@{FileClass='D';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=24} [PSCustomobject]@{FileClass='E';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=48} [PSCustomobject]@{FileClass='F';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=80} [PSCustomobject]@{FileClass='G';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=140} [PSCustomobject]@{FileClass='H';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=12;TargetFreqSecs=180} [PSCustomobject]@{FileClass='I';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=10;TargetFreqSecs=240} [PSCustomobject]@{FileClass='J';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=10;TargetFreqSecs=420} [PSCustomobject]@{FileClass='K';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=10;TargetFreqSecs=640} [PSCustomobject]@{FileClass='L';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=8;TargetFreqSecs=1200} [PSCustomobject]@{FileClass='M';CurrentBatchCount=0;CurrentBatchFileCount=0;CurrentBatchFileSizeMB=0;TotalBatchCount=0;` TotalFileCount=0;TotalFileSizeMB=0;TaskPerCore=4;TargetFreqSecs=1200} ) return $data } } function Initialize-MvDamUploadJob { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string] $JobFolder, [parameter(Mandatory=$false,Position=1)] [int] $ConcurrencyLevel = 2, [parameter(Mandatory=$false,Position=2)] [int] $MaxBatchSize= 200 ) PROCESS { if (!(Test-Path $JobFolder)) { New-Item -Path $JobFolder -ItemType Directory > $null } $pendingJobFolder = [System.IO.Path]::Combine($jobFolder,'PendingBatches') if (!(Test-Path $pendingJobFolder)) { New-Item -Path $pendingJobFolder -ItemType Directory > $null } $categoryImportLogPath = [System.IO.Path]::Combine($JobFolder,'Logs','category-import-log.csv') if (-not(Test-Path $categoryImportLogPath)) { Write-Warning "Category import log file is missing in the Job Folder. Please make sure that the category import completed." return -1 } $stats = New-MvDamCreateAssetBatchStats $processorCount = [Math]::Min($ConcurrencyLevel, [System.Environment]::ProcessorCount) #Do not exceed the number of cores $assetManifestFile = [System.IO.Path]::Combine($JobFolder,'asset-manifest.csv') $uploadRequests = Import-Csv -Path $assetManifestFile | Sort-Object -Property LastModified -Descending $reqId = 1 ForEach($request in $uploadRequests) { Write-Progress -Activity "Preparing asset upload plan..." -PercentComplete ($reqId/$uploadRequests.Count*100) $requestCategory = $null $requestCategory = Get-MvDamCategory -CategoryPath $request.CategoryPath $fileclass = Get-MvDamFileSizeClass -FileSize $request.Filesize $fileclassStat = $stats | Where {$_.FileClass -eq $fileclass} if (($fileclassStat.CurrentBatchCount -eq 0) -and ($fileclassStat.TotalBatchCount -eq 0)) { $fileclassStat.TotalBatchCount++ } if (($fileclassStat.CurrentBatchFileCount -ge ($processorCount * $fileclassStat.TaskPerCore)) -or ($fileclassStat.CurrentBatchFileCount -ge $MaxBatchSize)) { $fileclassStat.TotalBatchCount++ $fileclassStat.CurrentBatchFileCount = 0 $fileclassStat.CurrentBatchFileSizeMB = 0 } $fileclassStat.CurrentBatchFileCount++ $fileclassStat.CurrentBatchFileSizeMB =+ $request.Filesize $fileclassStat.TotalFileCount++ $fileclassStat.TotalFileSizeMB += ($request.Filesize/1024/1024) $batchId = [Math]::Max($fileclassStat.BatchCount, 1) $refId = $reqId.ToString("0000000") if (-not ([String]::IsNullOrEmpty($request.RefId))) { $refId = $request.RefId } $targetFilePath = [System.IO.Path]::Combine($pendingJobFolder, $fileclass + '-' + $fileclassStat.TotalBatchCount.ToString('00000') + ".csv") $request | Select-Object @{Label='RefId';Expression={$refId}}, @{Label='CategoryId';Expression={$requestCategory.Id}}, Filename, Filesize, @{Label="SourceUrl";Expression={$_.SasUrl}} | Export-Csv -Path $targetFilePath -Append -NoTypeInformation #Write-Host $request.filename $request.Filesize $fileclass $requestCategory = $null $reqId++ } $stats | ConvertTo-Json | Out-File $jobFolder\BatchJobMaster.json -Encoding utf8 return 0 } } function Start-MvDamUploadJob { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string] $JobFolder, [parameter(Mandatory=$false,Position=1)] [int] $MaxUploadRate = 200 ) PROCESS { $jobResult = 0 Write-Host "Starting batch asset submission...`n`n`n " try { $logPath = [System.IO.Path]::Combine($JobFolder,'Logs') $pendingJobFolder = [System.IO.Path]::Combine($JobFolder,'PendingBatches') $completedJobFolder = [System.IO.Path]::Combine($JobFolder,'CompletedBatches') $faultedJobFolder = [System.IO.Path]::Combine($JobFolder,'FaultedBatches') if (!(Test-Path $JobFolder)) { New-Item -Path $JobFolder -ItemType Directory > $null } if (!(Test-Path $pendingJobFolder)) { New-Item -Path $pendingJobFolder -ItemType Directory > $null } if (!(Test-Path $logPath)) { New-Item -Path $logPath -ItemType Directory > $null } if (!(Test-Path $completedJobFolder)) { New-Item -Path $completedJobFolder -ItemType Directory > $null } if (!(Test-Path $faultedJobFolder)) { New-Item -Path $faultedJobFolder -ItemType Directory > $null } $jobMasterFilePath = [System.IO.Path]::Combine($JobFolder,'BatchJobMaster.json') $jobSummaryFilePath = [System.IO.Path]::Combine($JobFolder,'BatchJobSummary.csv') $jobMaster = (Get-Content -Path $jobMasterFilePath -Encoding UTF8 | Out-String | ConvertFrom-Json) $currentBatchCount = ($jobMaster | Measure-Object -Property 'CurrentBatchCount' -Sum).Sum $totalBatchCount = ($jobMaster | Measure-Object -Property 'TotalBatchCount' -Sum).Sum $pullFromTop = $true $fileclassCode = $jobMaster | Where-Object {($_.TotalBatchCount -gt 0) -and ($_.CurrentBatchCount -le $_.TotalBatchCount)} | Sort-Object -Property TargetFrequencySecs | Select-Object FileClass -First 1 $maxUploadsPerMin = [Math]::Min(200, $MaxUploadRate) #Do not exceed 200 assets per minute $uploadLimitCounter = 0 $uploadLimitDuration = 0 $maxLinesPerLogFile = 10000 $linesLogged = 0 $logFiles = Get-ChildItem -Path $logPath -Filter "asset-import-log-*.csv" | Select @{Label="Id";Expression={[int]($_.Name.Substring(18,5))}}, FullName, Name $logFileCounter = [Math]::Max(($logFiles | Measure-Object -Property Id -Maximum).Maximum,1) if ($logFiles.Count -gt 0) { $lastLogFilePath = $logFiles | Where Id -eq $logFileCounter | Select FullName $linesLogged = (Import-Csv -Path $lastLogFilePath.FullName).Count } While($currentBatchCount -le $totalBatchCount) { Write-Progress -Activity 'Submitting upload batches...' -PercentComplete ($currentBatchCount/$totalBatchCount*100) if ($fileclassCode -eq $null) { break } $fileclass = $fileclassCode.FileClass $jobClass = $jobMaster | Where {$_.FileClass -eq $fileclass} if ($jobClass.CurrentBatchCount -le $jobClass.TotalBatchCount) { #Check if access token still valid even for a long running copy task if ((Get-Date).AddMinutes(45) -ge $MvDamContext.ClientRunspace.AccessTokenExpiry) { $region = $MvDamContext.ClientRunspace.Region.ToString().ToLower().Insert(2,'-') $password = $pass = ConvertTo-SecureString -String $MvDamContext.ClientRunspace.Password -AsPlainText -Force Connect-MvDamAccount -Username $MvDamContext.ClientRunspace.Username -Region $region -Password $pass } $batchId = $fileclass + '-' + ([Math]::Max($jobClass.CurrentBatchCount,1)).ToString('00000') $batchFilePath = [System.IO.Path]::Combine($pendingJobFolder, $batchId + ".csv") $jobClass.CurrentBatchCount++ if (!(Test-Path -Path $batchFilePath)) { Write-Host "$BatchId already processed." Continue } $batchStart = Get-Date $batch = Import-Csv -Path $batchFilePath -Encoding UTF8 #submit batch $jobClass.CurrentBatchFileSizeMB = [Math]::Round(($batch | Measure-Object -Property 'FileSize' -Sum).Sum/1024/1024, 4) Write-Host "Submitting $batchId with $($batch.Count) assets for ingestion - $($jobClass.CurrentBatchFileSizeMB)MB" $ingestionResult = Submit-MvDamIngestionBatch -IngestionRequests $batch #Get stats $batchEnd = Get-Date $elapsedSecs = ($batchEnd - $batchStart).TotalSeconds $uploadLimitCounter += $batch.Count $batchFileSizeMBCopied = [Math]::Round(($ingestionResult | Where {$_.Status.StartsWith('Success')} | Measure-Object -Property 'FileSize' -Sum).Sum/1024/1024, 4) $batchFileSuccessCount = ($ingestionResult | Where {$_.Status.StartsWith('Success')}).Count $batchFileFailCount = ($ingestionResult | Where {$_.Status.StartsWith('Fail')}).Count $jobClass.CurrentBatchFileSizeMB = $batchFileSizeMBCopied $jobClass.CurrentBatchFileCount = $batchFileSuccessCount $batchSummary = [PSCustomObject]@{BatchId=$batchId;TransferredMB=$batchFileSizeMBCopied;SuccessCount=$batchFileSuccessCount;FailCount=$batchFileFailCount;StartTime=$batchStart;EndTime=$batchEnd} $batchSummary | Export-Csv -Path $jobSummaryFilePath -Append -NoTypeInformation #Log the result if (($linesLogged + $ingestionResult.Count) -gt $maxLinesPerLogFile) { $linesLogged = 0 $logFileCounter++ } $logFilePath = [System.IO.Path]::Combine($logPath, "asset-import-log-$($logFileCounter.ToString("00000")).csv") $ingestionResult | Select-Object @{Label='BatchId';Expression={$batchId}}, RefId, AssetId, ` @{Label='CateoryIds';Expression={$_.CategoryIds -join ','}}, ` Filename, Filesize, Status, SrcSasUrl, DestSasUrl, ` @{Label='StartedOn';Expression={$_.PerfStats.StartTime}}, ` @{Label='CreatedOn';Expression={$_.PerfStats.CreatedTime}}, ` @{Label='BlobCopiedOn';Expression={$_.PerfStats.CopiedTime}}, ` @{Label='TitledOn';Expression={$_.PerfStats.TitledTime}}, ` @{Label='CategorizedOn';Expression={$_.PerfStats.ClassifiedTime}}, ` @{Label='EndedOn';Expression={$_.PerfStats.EndTime}} |` Export-Csv -Path $logFilePath -Append -NoTypeInformation $linesLogged =+ $ingestionResult.Count #update job master $jobmaster | ConvertTo-Json | Out-File $JobFolder\BatchJobMaster.json -Encoding utf8 #move pending file to completed or faulted $destinationPath = [System.IO.Path]::Combine($completedJobFolder, $batchId + ".csv") if ($batchFileFailCount -gt 0) { $destinationPath = [System.IO.Path]::Combine($faultedJobFolder, $batchId + ".csv") } Move-Item -Path $batchFilePath -Destination $destinationPath Write-Host "$batchId completed in $($elapsedSecs)s. $batchFileSuccessCount successfully submitted. $batchFileFailCount failed. `n" $uploadLimitDuration += $elapsedSecs if ($uploadLimitCounter -ge $maxUploadsPerMin) { if ($uploadLimitDuration -lt 60){ Write-Host "Throttling upload requests for $(60-$uploadLimitDuration)s...." Start-Sleep -Seconds (60-$uploadLimitDuration) } $uploadLimitCounter = 0 $uploadLimitDuration = 0 } $currentBatchCount++ } #Distribute ingestion workload $pullFromTop = -not $pullFromTop if ($pullFromTop) { $fileclassCode = $jobMaster | Where-Object {($_.TotalBatchCount -gt 0) -and ($_.CurrentBatchCount -le $_.TotalBatchCount)} | Sort-Object TargetFrequencySecs, {($_.CurrentBatchCount/$_.TotalBatchCount)} | Select-Object FileClass -First 1 } else { $fileclassCode = $jobMaster | Where-Object {($_.TotalBatchCount -gt 0) -and ($_.CurrentBatchCount -le $_.TotalBatchCount)} | Sort-Object TargetFrequencySecs, {($_.CurrentBatchCount/$_.TotalBatchCount)} | Select-Object FileClass -Last 1 } } #endwhile } catch { $jobResult = -1 } return $jobResult } } function Test-MvDamManifestImport { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string] $SrcSasUri, [parameter(Mandatory=$false,Position=1)] [string] $ParentCategoryPath = '\Root', [parameter(Mandatory=$true,Position=2)] [string] $JobFolder ) PROCESS { #Valid container SAS url $uriResult = $null $result = [Uri]::TryCreate($SrcSasUri, [UriKind]::Absolute, [ref] $uriResult) if ($result) { #check if signature is present if ([string]::IsNullOrEmpty($uriResult.Query)) { Write-Warning "Invalid blob container SAS URI specified. Missing SAS Signature." $result = $false } } else { Write-Warning "Invalid blob container SAS URI specified." } #Validate that the Az.Storage module is installed if ($null -eq (Get-Module -Name Az.Storage -ListAvailable)) { Write-Warning "Az.Storage Powershell Module is not installed. Please run 'Install-Module -Name Az -AllowClobber' as Administrator." $result = $false } #Validate MvDamContext if ($null -eq $MvDamContext) { Write-Warning "You are not connected to a MediaValet DAM Account. Please run 'Connect-MvDamAccount' to connect to an account." $result = $false } if ((Get-MvDamContext).FeatureFlags['FileNameVersioning'] -eq 'Active') { Write-Warning "Pre-requisite not met: FilenNameVersioning should be inactive when performing a batch upload from a container" $result = $false } #validate ParentCategoryPath $parentCategory = Get-MvDamCategory -CategoryPath $ParentCategoryPath if ($null -eq $parentCategory) { Write-Warning "ParentCategoryPath does not exist. Please create before proceeding." $result = $false } #ensure that we have a clean jobfolder if ((Test-Path $JobFolder) -and ((Get-ChildItem -Path $JobFolder).Count -gt 0)) { Write-Warning "JobFolder $JobFolder is not empty. Please use a clean job folder for every job." $result = $false } return $result; } } function Import-MvDamAssetsFromContainer { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string] $SrcSasUri, [parameter(Mandatory=$false,Position=1)] [string] $ParentCategoryPath = '\Root', [parameter(Mandatory=$true,Position=2)] [string] $JobFolder ) PROCESS { if (Test-MvDamManifestImport -SrcSasUri $SrcSasUri -ParentCategoryPath $ParentCategoryPath -JobFolder $JobFolder) { $result = Export-MvDamContainerManifest -SrcSasUri $SrcSasUri -ParentCategoryPath $ParentCategoryPath -JobFolder $JobFolder if ($result -ne 0) { Write-Warning "Error encountered exporting container manifest. Stopping job execution." return } $result = Import-MvDamCategoryTreeFromManifest -JobFolder $JobFolder if ($result -ne 0) { Write-Warning "Error encountered creating categories. Stopping job execution." return } $result = Initialize-MvDamUploadJob -JobFolder $JobFolder if ($result -ne 0) { Write-Warning "Error preparing DAM ingestion plan. Stopping job execution." return } $result = Start-MvDamUploadJob -JobFolder $JobFolder if ($result -ne 0) { Write-Warning "Error(s) encountered executing the ingestion plan. Please review the logs at $JobFolder\Logs." return } } else { Write-Warning "Import-MvDamAssetsFromContainer pre-flight check failed. Stopping job execution." return } } } function Reset-MvDamUploadJob { [CmdletBinding()] param ( [parameter(Mandatory=$true,Position=0)] [string] $JobFolder ) PROCESS { $pendingJobFolder = [System.IO.Path]::Combine($JobFolder,'PendingBatches') $completedJobFolder = [System.IO.Path]::Combine($JobFolder,'CompletedBatches') $faultedJobFolder = [System.IO.Path]::Combine($JobFolder,'FaultedBatches') $jobMasterFilePath = [System.IO.Path]::Combine($JobFolder,'BatchJobMaster.json') $jobMaster = (Get-Content -Path $jobMasterFilePath -Encoding UTF8 | Out-String | ConvertFrom-Json) foreach($fileclass in $jobMaster) { $firstPendingBatchId = $lastPendingBatchId = $lastCompletedBatchId =$lastFailedBatchId = 0 $fileClassCode = $fileclass.FileClass $pendingBatches = Get-ChildItem -Path $pendingJobFolder | Where {$_.Name -like "$fileClassCode-*.csv"} | Sort-Object -Property Name $firstPendingBatch = $pendingBatches | Select Name -First 1 if (-not [string]::IsNullOrEmpty($firstPendingBatch)) { $firstPendingBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($firstPendingBatch.Name)).Substring(2)) } $lastPendingBatch = $pendingBatches | Select Name -Last 1 if (-not [string]::IsNullOrEmpty($lastPendingBatch)) { $lastPendingBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($lastPendingBatch.Name)).Substring(2)) } $lastCompletedBatch = Get-ChildItem -Path $completedJobFolder | Where {$_.Name -like "$fileClassCode-*.csv"} | Sort-Object -Property Name | Select Name -Last 1 if (-not [string]::IsNullOrEmpty($lastCompletedBatch)) { $lastCompletedBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($lastCompletedBatch.Name)).Substring(2)) } $lastFailedBatch = Get-ChildItem -Path $faultedJobFolder | Where {$_.Name -like "$fileClassCode-*.csv"} | Sort-Object -Property Name | Select Name -Last 1 if (-not [string]::IsNullOrEmpty($lastFailedBatch)) { $lastFailedBatchId = [Convert]::ToInt32(([System.IO.Path]::GetFileNameWithoutExtension($lastFailedBatch.Name)).Substring(2)) } Write-Host "Resetting FileClass $fileClassCode First Pending: $firstPendingBatchId Last Pending: $lastPendingBatchId Completed: $lastCompletedBatchId Faulted: $lastFailedBatchId" $fileclass.CurrentBatchCount = [Math]::Max($lastCompletedBatchId, $lastFailedBatchId) $fileclass.CurrentBatchFileCount= 0 $fileclass.CurrentBatchFileSizeMB = 0 $fileclass.TotalBatchCount = [Math]::Max($lastPendingBatchId, [Math]::Max($lastCompletedBatchId, $lastFailedBatchId)) } Write-Host "Updating job master file at $jobMasterFilePath" $jobmaster | ConvertTo-Json | Out-File $JobFolder\BatchJobMaster.json -Encoding utf8 } } |