Private/Get-TrouwArticle.ps1
|
function Get-TrouwArticle { param ( [Parameter(Mandatory, Position = 0)] [string]$Uri, [Parameter()] [ScriptBlock]$UriFilter ) $DutchCulture = New-Object -TypeName System.Globalization.CultureInfo -ArgumentList 'nl-NL' curl --silent --location $Uri | Join-String -Separator ' ' | ConvertTo-HtmlDocument | ForEach-Object { $_.DocumentNode.SelectNodes("//a") } | Where-Object { $_.HasAttributes } | ForEach-Object { $_.GetAttributeValue('href', '')} | Where-Object -FilterScript $UriFilter | ForEach-Object { $Url = $_ -replace '^/', 'https://trouw.nl/' $Document = curl --silent --location $Url | Join-String -Separator ' ' | ConvertTo-HtmlDocument $DateTimeText = $PublishedTimeAttribute = $Document | Select-HtmlNode -XPath '//time' | Where-Object { $_.HasAttributes } | ForEach-Object { $_.GetAttributeValue('datetime', '') } $DateText = ($DateTimeText -split 'T')[0] $Date = [DateTime]::ParseExact($DateText, 'yyyy\-MM\-dd', $DutchCulture) $Title = (($Document | Select-HtmlNode -CssSelector 'h1' -All).InnerText | ForEach-Object { $_.Trim() } | Where-Object { $_ }) -join ' ' $Paragraphs = $Document | Select-HtmlNode -CssSelector 'section' | Select-HtmlNode -CssSelector 'p' -All | Where-Object { $_.GetAttributeValue('class', '') -ne 'artstyle__container__text' } $Body = (($Paragraphs).InnerText | ForEach-Object { $_.Trim() } | Where-Object { $_ }) -join ' ' [PSCustomObject]@{ PSTypeName = 'UncommonSense.Trouw.Article' Url = $Url Date = $Date Title = $Title Body = $Body } Start-Sleep -Seconds 1 # Prevent nginx 429 error (too many requests) } } |