Private/ConvertTo-GraphSearchWords.ps1
|
function ConvertTo-GraphSearchWords { <# .SYNOPSIS Normalizes free text into a lowercase, diacritic-free, word-boundary-aware token array. .DESCRIPTION Shared normalization used by the -Query ranking engine for both the user's query and catalog fields (command, endpoint, module, aliases, variants, permission names). Splits camelCase boundaries (so "SignIn" becomes "sign in") before splitting on non-alphanumeric characters, which is what lets a query token match a real word inside a cmdlet/endpoint name (e.g. "sign" matching "SignIn") without also matching an unrelated word that merely contains the same letters as a substring (e.g. "sign" inside "Assignment", which is a single camelCase word with no internal word boundary at "sign"). #> [CmdletBinding()] [OutputType([string[]])] param( [Parameter(Mandatory)] [AllowEmptyString()] [string]$Text ) if ([string]::IsNullOrWhiteSpace($Text)) { return @() } # The catalog's own fields (command, uri, module, aliases, ...) are always plain ASCII, so # the comparatively expensive FormD normalization + per-char Unicode-category scan below is # only needed for user-typed queries that may contain accented characters (e.g. Portuguese # input). Skipping it for ASCII-only text is what makes tokenizing the ~31k-record catalog # (called once per session to build the -Query inverted index) fast. if ([System.Text.RegularExpressions.Regex]::IsMatch($Text, '[^\x00-\x7F]')) { $normalized = $Text.Normalize([System.Text.NormalizationForm]::FormD) $sb = [System.Text.StringBuilder]::new() foreach ($ch in $normalized.ToCharArray()) { $category = [System.Globalization.CharUnicodeInfo]::GetUnicodeCategory($ch) if ($category -ne [System.Globalization.UnicodeCategory]::NonSpacingMark) { [void]$sb.Append($ch) } } $withoutDiacritics = $sb.ToString() } else { $withoutDiacritics = $Text } # Insert a boundary between a lowercase/digit and a following uppercase letter (camelCase). $withCamelBoundaries = [System.Text.RegularExpressions.Regex]::Replace($withoutDiacritics, '([a-z0-9])([A-Z])', '$1 $2') $lower = $withCamelBoundaries.ToLowerInvariant() # Deliberately avoid a `| Where-Object` pipeline here: at catalog-tokenization scale (tens of # thousands of calls to build the -Query inverted index) the pipeline's per-item overhead # dwarfs the cost of the regex work itself, so filter empty entries with a plain loop instead. $parts = [System.Text.RegularExpressions.Regex]::Split($lower, '[^a-z0-9]+') $words = [System.Collections.Generic.List[string]]::new($parts.Length) foreach ($part in $parts) { if ($part) { $words.Add($part) } } return , [string[]]$words } function Test-GraphWordMatch { <# .SYNOPSIS Compares a query token against a single catalog word and returns a match weight. .DESCRIPTION Returns 1.0 for an exact match, 0.7 for a simple singular/plural variant (trailing "s"), and 0 otherwise. Deliberately does not fall back to raw substring matching: that is what previously produced false positives such as "sign" matching inside "Assignment". Morphological variants beyond plural/singular (e.g. "eligible" vs "eligibility") are handled by data/graph-search-synonyms.json instead of ad hoc string rules here. #> [CmdletBinding()] [OutputType([double])] param( [Parameter(Mandatory)] [string]$Token, [Parameter(Mandatory)] [string]$Word ) if ($Token -eq $Word) { return 1.0 } if ($Token.Length -ge 3 -and $Word.Length -ge 3) { if ("$Token" + 's' -eq $Word -or "$Word" + 's' -eq $Token) { return 0.7 } } return 0.0 } |