PwSh.Fw.Web.psm1
|
<#
#> <# .SYNOPSIS Download a file from the Internet. .Description Download a file using the System.Net.WebClient class or the Invoke-WebRequest method, then optionally check its MD5, SHA1 or SHA256 hash. The existence of the web resource is tested before downloading (HEAD request). .PARAMETER Url URL from where to fetch file .PARAMETER To Destination folder to write file. Defaults to the current location. .PARAMETER Filename Optional filename to rename downloaded file on-the-fly. Defaults to the last segment of the URL. .PARAMETER UseWebClient Instruct Download-File to use the .Net webclient object .PARAMETER UseInvokeWebRequest Instruct Download-File to use the slower but safer Invoke-WebRequest method .PARAMETER Headers Optional hashtable of HTTP headers to send with the request .PARAMETER checkMD5 If specified, the MD5 hash of the downloaded file must match this value .PARAMETER checkSHA1 If specified, the SHA1 hash of the downloaded file must match this value .PARAMETER checkSHA256 If specified, the SHA256 hash of the downloaded file must match this value .PARAMETER PassThru Return the full path of the downloaded file instead of the boolean return code .PARAMETER Force If specified, download the file even if it already exists at the destination. Hash checks are still performed. .OUTPUTS [bool] True when the file is present at the destination after the call and passed all requested hash checks (downloaded now, or already present without -Force). False otherwise : missing destination folder, non-200 HTTP status, failed download or hash mismatch. The function never returns $null. [String] Full path of the downloaded file when -PassThru is specified. .EXAMPLE Download-File -Url https://example.com/file.zip -To C:\downloads Downloads file.zip into C:\downloads and returns $true on success. .EXAMPLE $fullPath = Download-File -Url https://example.com/file.zip -To C:\downloads -PassThru Downloads file.zip and returns its full path. .EXAMPLE Download-File -Url https://example.com/file.zip -To C:\downloads -checkSHA256 "abc123..." Downloads file.zip and verifies its SHA256 hash. Returns $false on mismatch. #> function Download-File { [CmdletBinding()] [System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseApprovedVerbs", "", Justification="Download is a more intuitive verb for this function because Get- is too generic.")] [OutputType([Boolean])]Param( [Alias("Uri")] [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][System.Uri]$Url, [Alias("Destination")] [string]$To = (Get-Location).providerpath, [string]$Filename = [system.uri]::UnEscapeDataString($URL.Segments[-1]), [switch]$UseWebClient, [switch]$UseInvokeWebRequest, [hashtable]$Headers, [string]$checkMD5, [string]$checkSHA1, [string]$checkSHA256, [switch]$PassThru, [switch]$Force ) Begin { Write-EnterFunction # compute default download method if ($UseWebClient -eq $UseInvokeWebRequest) { $UseWebClient = $false $UseInvokeWebRequest = $true } } Process { $rc = $false # note : the check is done here and not in Begin because a "return" # statement in a Begin block does not stop the Process block if (-not (dirExist $To)) { Write-Error "Folder '$To' does not exist." return $false } $fileExist = Test-Path -Path "$To/$Filename" $webrc = Get-UriWebRc -Url $Url.AbsoluteUri -AsInt Write-Devel "[$webrc] $URL" if ($fileExist -and -not $Force) { # File already present and no force : nothing to download (idempotent success) $rc = $true } elseif ($webrc -ne 200) { Write-Warning "HTTP [$webrc] for $Url : download skipped." $rc = $false } else { if ($Force) { Write-Warning "File exist, but user asked to force download of $Url" } if ($UseWebClient) { # edevel "Download using WebClient .Net object." if (Test-IsUNCPath -Path $To) { Write-Warning "$To is a UNC Path. WebClient does not support UNC Path. Falling back to InvokeWebRequest method." $UseInvokeWebRequest = $true } else { $webClient = New-Object System.Net.WebClient if ($Headers) { $webClient.Headers = $Headers } $webClient.DownloadFile($Url, "$To/$Filename") } } if ($UseInvokeWebRequest) { # edevel "Download using Invoke-WebRequest method." $null = Invoke-WebRequest -Uri $Url.AbsoluteUri -OutFile "$To/$Filename" -UseBasicParsing -Headers $Headers } $rc = $true } $fileExist = Test-Path -Path "$To/$Filename" if ($fileExist) { if ($checkMD5) { $md5 = Get-FileHash -Algorithm MD5 -Path "$To/$Filename" if ($md5.Hash -eq $checkMD5) { Write-Success "MD5 hash match" $rc = $true } else { Write-Error "MD5 hash mismatch. Try downloading the file again." $rc = $false } } if ($checkSHA1) { $sha1 = Get-FileHash -Algorithm SHA1 -Path "$To/$Filename" if ($sha1.Hash -eq $checkSHA1) { Write-Success "SHA1 hash match" $rc = $true } else { Write-Error "SHA1 hash mismatch. Try downloading the file again." $rc = $false } } if ($checkSHA256) { $sha256 = Get-FileHash -Algorithm SHA256 -Path "$To/$Filename" if ($sha256.Hash -eq $checkSHA256) { Write-Success "SHA256 hash match" $rc = $true } else { Write-Error "SHA256 hash mismatch. Try downloading the file again." $rc = $false } } } if ($PassThru) { return (Resolve-Path "$To/$Filename").Path } else { return $rc } } End { Write-LeaveFunction } } <# .SYNOPSIS Download a file from the Internet and return the HTTP response code. .Description Download a file using the Invoke-WebRequest method and return the HTTP response code of the transfer. .PARAMETER Url URL from where to fetch file .PARAMETER To Destination folder to write file. Defaults to the current location. .PARAMETER Filename Optional filename to rename downloaded file on-the-fly. Defaults to the last segment of the URL. .OUTPUTS [Int] The HTTP status code of the successful download (usually 200), the HTTP status code when the resource is not available (e.g. 404), or -1 when the download cannot be performed locally (missing destination folder) or the request fails at network level (DNS, connection refused...). .EXAMPLE $rc = Download-FileWithRC -Url https://example.com/file.zip -To C:\downloads Downloads file.zip and returns 200 on success. #> function Download-FileWithRC { [System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseApprovedVerbs", "", Justification="Download is a more intuitive verb for this function because Get- is too generic.")] [CmdletBinding()][OutputType([Int])]Param ( [Parameter(Mandatory = $true)][System.Uri]$Url, [Alias("Destination")] [string]$To = (Get-Location).providerpath, [string]$Filename ) Begin { Write-EnterFunction if (!$Filename) { $Filename = [System.Uri]::UnEscapeDataString($Url.Segments[-1]) } } Process { # note : the check is done here and not in Begin because a "return" # statement in a Begin block does not stop the Process block if (-not (dirExist $To)) { Write-Error "Folder '$To' does not exist." return -1 } $outFile = "$To/$Filename" $webrc = Get-UriWebRc -Url $Url.AbsoluteUri -AsInt if ($webrc -eq 200) { $webRequest = Invoke-WebRequest -Uri $Url.AbsoluteUri -OutFile $outFile -PassThru -UseBasicParsing return [int]$webRequest.StatusCode } else { return $webrc } } End { Write-LeaveFunction } } <# .SYNOPSIS Get the HTTP response (or status) code from a web URI .DESCRIPTION Only get the HTTP response code, do not download any content (HEAD request). The request is performed with [System.Net.HttpWebRequest] so that the behaviour is identical under Windows PowerShell 5.1 and PowerShell 7. .PARAMETER Url Full URL to test .PARAMETER AsText If true, output the status message text (e.g. OK) .PARAMETER AsInt If true, output the status message code (e.g. 200) .OUTPUTS Valid HTTP status code or status message. If the request cannot be completed (hostname cannot be resolved, connection refused, timeout...), then an error message is returned accordingly (-AsText) or -1 (-AsInt). .EXAMPLE $httpCode = Get-UriWebRc -Url "https://www.google.com" -AsText Will output 'OK' (I hope so !) .EXAMPLE $httpCode = Get-UriWebRc -Url "http://www.google.com/nonexistentfile" -AsInt Will output 404 .NOTES General notes .LINK https://stackoverflow.com/questions/795751/can-i-get-detailed-exception-stacktrace-in-powershell https://www.reddit.com/r/PowerShell/comments/30r1e9/any_way_to_get_just_the_response_code_using/ #> function Get-UriWebRc { [CmdletBinding()]Param ( [Parameter(Mandatory = $true)][System.Uri]$Url, [switch]$AsText, [switch]$AsInt ) Begin { Write-EnterFunction if ($AsText -eq $false -and $AsInt -eq $false) { $AsInt = $true } } Process { $response = @{} $response.StatusCode = -1 $response.StatusMsg = $null try { $request = [System.Net.HttpWebRequest]::Create($Url.AbsoluteUri) $request.Method = "HEAD" $webResponse = $request.GetResponse() try { $response.StatusCode = [int]$webResponse.StatusCode $response.StatusMsg = $webResponse.StatusCode.ToString() } finally { $webResponse.Close() } } catch [System.Net.WebException] { if ($null -ne $_.Exception.Response) { # The server answered : report its HTTP status code (404, 500, ...) $code = [int]$_.Exception.Response.StatusCode $response.StatusCode = $code $response.StatusMsg = ([System.Net.HttpStatusCode]$code).ToString() } else { # Network-level failure (DNS, connection refused, timeout, ...) $response.StatusCode = -1 $response.StatusMsg = $_.Exception.Message } } catch { # Unexpected error (invalid URI, cancellation, ...) $response.StatusCode = -1 $response.StatusMsg = $_.Exception.Message } if ($AsText) { $response.StatusMsg } # This is a System.Net.HttpStatusCode enum value (@see https://msdn.microsoft.com/fr-fr/library/system.net.httpstatuscode(v=vs.110).aspx ) if ($AsInt) { $response.StatusCode } # This is the numeric version. } End { Write-LeaveFunction } } <# .SYNOPSIS Test if a web resource exist .DESCRIPTION Wrapper of Get-UriWebRc to quickly know if a web resource is available or not. .PARAMETER URL The URL to test. Better use it to check for a file .EXAMPLE Test-WebFileExist -URL http://www.example.com/file.txt #> function Test-WebFileExist { [CmdletBinding()] [OutputType([Boolean])] Param ( [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][System.Uri]$URL ) Begin { Write-EnterFunction } Process { $int = Get-UriWebRc -Url $URL -AsInt switch ($int) { 200 { $bool = $true } default { $bool = $false } } return $bool } End { Write-LeaveFunction } } <# .SYNOPSIS Convert an object to a query string. .DESCRIPTION Convert an object to a single string to be used with HTTP GET method. Arrays, dictionaries (hashtable, ordered dictionary) and custom objects are converted recursively. Keys and values are URL-encoded, scalar values are formatted with the invariant culture. .PARAMETER InputObject Object to convert .PARAMETER variableName Main variable name to use .PARAMETER CurrentDepth Current depth into InputObject. This parameter is used internally for recursivity. Do not assign. .PARAMETER MaxDepth Maximum depth of object to convert into a string .PARAMETER CurrentPath Parameter used internally for recursivity. Do not assign. .EXAMPLE $q = ConvertTo-QueryString -InputObject @{"one" = 1, "two" = 2} -variableName "query" Invoke-RestMethod -Method Get -ContentType "application/json" -Uri $("https://www.example.com/search?" + $q) The example above will convert the hashtable into the query string : "query[one]=1&query[two]=2" and then invoke a REST method using this query. .NOTES Booleans are rendered as lowercase 'true' / 'false'. #> function ConvertTo-QueryString { [CmdletBinding()] [OutputType([String])]Param ( [AllowNull()] [Parameter(Mandatory = $true, ValueFromPipeLine = $true)]$InputObject, [Parameter()][string]$variableName, [int16]$CurrentDepth = -1, [uint16]$MaxDepth = 16, [string]$CurrentPath ) Begin { Write-EnterFunction $queryString = "" } Process { if ($null -eq $InputObject) { return "" } if ($CurrentDepth -gt $MaxDepth) { # Maximum depth reached : append the value as-is $leafValue = [System.Convert]::ToString($InputObject, [System.Globalization.CultureInfo]::InvariantCulture) if ($InputObject -is [bool]) { $leafValue = $leafValue.ToLowerInvariant() } return $queryString + ("&" + $CurrentPath + "=" + [System.Uri]::EscapeDataString($leafValue)) } $CurrentDepth++ if ($CurrentDepth -eq 0) { $CurrentPath = [System.Uri]::EscapeDataString($variableName) } if ($InputObject -is [System.Collections.IList]) { for ($index = 0; $index -lt $InputObject.Count; $index++) { $queryString += ConvertTo-QueryString -CurrentDepth $CurrentDepth -InputObject $InputObject[$index] -CurrentPath ($CurrentPath + "[$index]") } } elseif ($InputObject -is [System.Collections.IDictionary]) { foreach ($key in $InputObject.Keys) { $escapedKey = [System.Uri]::EscapeDataString([string]$key) $queryString += ConvertTo-QueryString -CurrentDepth $CurrentDepth -InputObject $InputObject[$key] -CurrentPath ($CurrentPath + "[" + $escapedKey + "]") } } elseif ($InputObject -is [System.Management.Automation.PSCustomObject]) { foreach ($prop in $InputObject.PSObject.Properties) { $escapedKey = [System.Uri]::EscapeDataString($prop.Name) $queryString += ConvertTo-QueryString -CurrentDepth $CurrentDepth -InputObject $prop.Value -CurrentPath ($CurrentPath + "[" + $escapedKey + "]") } } else { # Scalar leaf (bool, int, long, double, decimal, string, char, guid, datetime, ...) $leafValue = [System.Convert]::ToString($InputObject, [System.Globalization.CultureInfo]::InvariantCulture) if ($InputObject -is [bool]) { $leafValue = $leafValue.ToLowerInvariant() } $queryString += ("&" + $CurrentPath + "=" + [System.Uri]::EscapeDataString($leafValue)) } $CurrentDepth-- if ($CurrentDepth -eq -1) { return $queryString.Trim("&") } else { return $queryString } } End { Write-LeaveFunction } } <# .SYNOPSIS Trim leading and trailing characters .DESCRIPTION Trim leading and trailing additional non-wanted characters like - slashes '/' .EXAMPLE Trim-Url -URL "https://localhost:8000/" .NOTES General notes #> function Trim-Url { [System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseApprovedVerbs", "", Justification="Trim is a more intuitive verb for this function because Get- or Edit- is too generic.")] [CmdletBinding()] [OutputType([String])] Param ( [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][string]$URL ) Begin { Write-EnterFunction } Process { return $URL.Trim('/') } End { Write-LeaveFunction } } function Upload-File { [CmdletBinding()] param ( [Alias('URL')] [Parameter(Mandatory = $true, ValueFromPipeLine = $true)][System.Uri]$To, [string]$Filename ) begin { Write-EnterFunction } process { } end { Write-LeaveFunction } } |