CsvSql.psm1

Function Invoke-CsvSql {

    <#
     .SYNOPSIS
        Natively query CSV files using SQL syntax
         
     .DESCRIPTION
     This module will enable you to query a CSV files using SQL syntax using Microsoft's Text Drivers.
         
     If you are running this module on a 64-bit system, and the 64-bit Text Driver is not installed, the module will automatically switch to a 32-bit shell and execute the query.
     It will then communicate the data results to the 64-bit shell using Export-Clixml/Import-Clixml. While the shell switch process is rather quick, you can avoid this step by
     running the module within a 32-bit PowerShell shell ("$env:windir\syswow64\windowspowershell\v1.0\powershell.exe")
         
     The module returns datarows. See the examples for more details.
         
     .PARAMETER CSV
      The location of the CSV files to be queried. Multiple files are allowed, so long as they all support the same SQL query, and delimiter.
         
     .PARAMETER FirstRowColumnNames
      This parameter specifies whether the first row contains column names. If the first row does not contain column names, the query engine automatically names the columns or "fields", F1, F2, F3 and so on.
       
     .PARAMETER Delimiter
      Optional. If you do not pass a Delimiter, then a comma will be used. Valid Delimiters include: tab "`t", pipe "|", semicolon ";", space " " and maybe a couple other things.
       
      When this parameter is used, a schema.ini must be created. In the event that one already exists, it will be moved to TEMP, then moved back once the module is finished executing.
       
     .PARAMETER SQL
      The SQL statement to be executed. To make command line queries easier, this module will convert the word "csv" to the actual CSV formatted table name.
      If the FirstRowColumnNames switch is not used, the query engine automatically names the columns or "fields", F1, F2, F3 and so on.
       
      Example: select F1, F2, F3, F4 from csv where F1 > 5. See EXAMPLES for more example syntax.
      
     .PARAMETER shellswitch
      Internal parameter.
         
     .NOTES
        Author : Chrissy LeMaire
        Requires: PowerShell 3.0
        Version: 1.0.7
        DateUpdated: 2021-09-20 bg Chris Martinez
 
     .LINK
        https://gallery.technet.microsoft.com/scriptcenter/Query-CSV-with-SQL-c6c3c7e5
         
     .EXAMPLE
        Invoke-CsvSql -csv C:\temp\housingmarket.csv -sql "select address from csv where price < 250000" -FirstRowColumnNames
         
        This example return all rows with a price less than 250000 to the screen. The first row of the CSV file, C:\temp\housingmarket.csv, contains column names.
         
     .EXAMPLE
        Invoke-CsvSql -csv C:\temp\unstructured.csv -sql "select F1, F2, F3 from csv"
         
        This example will return the first three columns of all rows within the CSV file C:\temp\unstructured.csv to the screen.
        Since the -FirstRowColumnNames switch was not used, the query engine automatically names the columns or "fields", F1, F2, F3 and so on.
      
     .EXAMPLE
        $datatable = Invoke-CsvSql -csv C:\temp\unstructured.csv -sql "select F1, F2, F3 from csv"
        $datatable.rows.count
      
        The script returns rows of a datatable, and in this case, we create a datatable by assigning the output of the module to a variable, instead of to the screen.
      
    #>
 
    #Requires -Version 3.0
    [CmdletBinding()] 
    Param(
        [Parameter(Mandatory=$true)] 
        [string[]]$Csv,
        [switch]$FirstRowColumnNames,
        [string]$Delimiter = ",",
        [Parameter(Mandatory=$true)] 
        [string]$Sql,
        [switch]$shellswitch
        )
        
    BEGIN {
        # In order to ensure consistent results, a schema.ini file must be created.
        # If a schema.ini currently exists, it will be moved to TEMP temporarily.
        
        if (!$shellswitch) {
        $resolvedcsv = @()
        foreach ($file in $csv) { $resolvedcsv += (Resolve-Path $file).Path }
        $csv = $resolvedcsv
        
        $movedschemaini = @{}
        foreach ($file in $csv) {
            $directory = Split-Path $file
            $schemaexists = Test-Path "$directory\schema.ini"
            if ($schemaexists -eq $true) {
                $newschemaname = "$env:TEMP\$(Split-Path $file -leaf)-schema.ini"
                $movedschemaini.Add($newschemaname,"$directory\schema.ini")
                Move-Item "$directory\schema.ini" $newschemaname -Force
            }
        }
        }
        
        # Check for drivers.
        $provider = (New-Object System.Data.OleDb.OleDbEnumerator).GetElements() | Where-Object { $_.SOURCES_NAME -like "Microsoft.ACE.OLEDB.*" }
        
        if ($null -eq $provider) {
            $provider = (New-Object System.Data.OleDb.OleDbEnumerator).GetElements() | Where-Object { $_.SOURCES_NAME -like "Microsoft.Jet.OLEDB.*" }    
        }
        
        if ($null -eq $provider) { 
            Write-Host -ForegroundColor Yellow "No x64 OLEDB driver detected, using x86 OLEDB driver."
        } else { 
            if ($provider -is [system.array]) { $provider = $provider[$provider.GetUpperBound(0)].SOURCES_NAME } else {  $provider = $provider.SOURCES_NAME }
        }
    }

    PROCESS {
        
        # Try hard to find a suitable provider; switch to x86 if necessary.
        # Encode the SQL string, since some characters
        if ($null -eq $provider) {
            $bytes  = [System.Text.Encoding]::UTF8.GetBytes($sql)
            $sql = [System.Convert]::ToBase64String($bytes)
            
            if ($firstRowColumnNames) { $frcn = "-FirstRowColumnNames" }
                $csv = $csv -join ","
                &"$env:windir\syswow64\windowspowershell\v1.0\powershell.exe" "Set-ExecutionPolicy RemoteSigned -Scope CurrentUser -confirm:0 -Force ; Invoke-CsvSql -csv $csv $frcn -Delimiter '$Delimiter' -SQL $sql -shellswitch" 
                return
        }
        # If the shell has switched, decode the $sql string.
        if ($shellswitch) {
            $bytes  = [System.Convert]::FromBase64String($sql)
            $sql = [System.Text.Encoding]::UTF8.GetString($bytes)
            $csv = $csv -Split ","
        }
        
        # Check for proper SQL syntax, which for the purposes of this module must include the word "csv"
        if ($sql.ToLower() -notmatch "\bcsv\b") {
            throw "SQL statement must contain the word 'csv'. Please see this module's documentation for more details."
        }
        
        switch ($FirstRowColumnNames) {
                $true { $frcn = "Yes" }
                $false { $frcn = "No" }
        }
        
        # Does first line contain the specified delimiter?
        foreach ($file in $csv) {
            $firstline = Get-Content $file -First 1
            if (($firstline -match $Delimiter) -eq $false) {  throw "Delimiter $Delimiter not found in first row of $file." }
        }
        
        # If more than one csv specified, check to ensure number of columns match
        if ($csv -is [system.array]){ 
            $numberofcolumns = ((Get-Content $csv[0] -First 1) -Split $delimiter).Count 
            foreach ($file in $csv) {
                $firstline = Get-Content $file -First 1
                $newnumcolumns = ($firstline -Split $Delimiter).Count
                if ($newnumcolumns -ne $numberofcolumns) { throw "Multiple csv file mismatch. Do both use the same delimiter and have the same number of columns?" }
            }
        }
        
        # Create the resulting datatable
        $dt = New-Object System.Data.DataTable
        
        # Go through each file
        foreach ($file in $csv) {
        
            # Unfortunately, passing delimiter within the connection string is unreliable, so we'll use schema.ini instead
            # The default delimiter in Windows changes depending on country, so we'll do this for every delimiter, even commas.
            $filename = Split-Path $file -leaf; $directory = Split-Path $file
            Add-Content -Path "$directory\schema.ini" -Value "[$filename]"
            Add-Content -Path "$directory\schema.ini" -Value "Format=Delimited($Delimiter)"
            Add-Content -Path "$directory\schema.ini" -Value "ColNameHeader=$FirstRowColumnNames"
            
            # Setup the connection string. Data Source is the directory that contains the csv.
            # The file name is also the table name, but with a "#" instead of a "."
            $datasource = Split-Path $file
            $tablename = (Split-Path $file -leaf).Replace(".","#")
            
            $connstring = "Provider=$provider;Data Source=$datasource;Extended Properties='text;HDR=$frcn;';"

            # To make command line queries easier, let the user just specify "table" instead of the
            # OleDbconnection formatted name (file.csv -> file#csv)
            $sql = $sql -replace "\bcsv\b"," [$tablename]"
            
            # Setup the OleDbconnection
            $conn = New-Object System.Data.OleDb.OleDbconnection
            $conn.ConnectionString = $connstring
            try { $conn.Open() } catch { throw "Could not open OLEDB connection." }
            
            # Setup the OleDBCommand
            try {
                $cmd = New-Object System.Data.OleDB.OleDBCommand
                $cmd.Connection = $conn
                $cmd.CommandText = $sql
            } catch { throw "Could not create OLEDB command." }
            
            # Execute the query, then load it into a datatable
            
            try {
                $null = $dt.Load($cmd.ExecuteReader([System.Data.CommandBehavior]::CloseConnection))
            } catch { 
                $errormessage = $_.Exception.Message.ToString()
                if ($errormessage -like "*for one or more required parameters*") {
                    throw "Looks like your SQL syntax may be invalid. `nCheck the documentation for more information or start with a simple -sql 'select top 10 * from table'"
                } else { Write-Error "Execute failed: $errormessage" }
            }
            
            # This should automatically close, but just in case...
            try {
                if ($conn.State -eq "Open") { $null = $conn.close }
                $null = $cmd.Dispose; $null = $conn.Dispose
            } catch { Write-Warning "Could not close connection. This is just an informational message." }
        }
        
        # Use a file to facilitate the passing of a datatable from x86 to x64 if necessary
        if ($shellswitch) {
            try { $dt | Export-Clixml "$env:TEMP\invoke-csvsql-dt.xml" -Force } catch { throw "Could not export datatable to file." }
        }
    }

    END {
        # Delete new schema files
        foreach ($file in $csv) {
            $directory = Split-Path $file
            $null = Remove-Item "$directory\schema.ini" -Force -ErrorAction SilentlyContinue
        }
        
        # Move original schema.ini's back if they existed
        if ($movedschemaini.count -gt 0) {
            foreach ($item in $movedschemaini) {
                Move-Item $item.keys $item.values -Force -ErrorAction SilentlyContinue    
            }
        }

        # If going between shell architectures, import a properly structured datatable.
        if ($null -eq $dt -and (Test-Path "$env:TEMP\invoke-csvsql-dt.xml")) {
            $dt = Import-Clixml "$env:TEMP\invoke-csvsql-dt.xml" 
            $null = Remove-Item "$env:TEMP\invoke-csvsql-dt.xml" -ErrorAction SilentlyContinue
        }
        
        if ($shellswitch -eq $false) { return $dt }    
    }
}
# SIG # Begin signature block
# MIIL8AYJKoZIhvcNAQcCoIIL4TCCC90CAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB
# gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR
# AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQU2zsjGjpBGNo9kb+G8XY3j5Qs
# haaggglnMIIEkjCCA3qgAwIBAgIHPQAAAAAGPzANBgkqhkiG9w0BAQsFADBKMRMw
# EQYKCZImiZPyLGQBGRYDZ292MRIwEAYKCZImiZPyLGQBGRYCdmExHzAdBgNVBAMT
# FlZBLUludGVybmFsLVMyLVJDQTEtdjEwHhcNMjAwMjI3MjA0ODU2WhcNMzAwMjI0
# MjA0ODU2WjBHMRMwEQYKCZImiZPyLGQBGRYDZ292MRIwEAYKCZImiZPyLGQBGRYC
# dmExHDAaBgNVBAMTE1ZBLUludGVybmFsLVMyLUlDQTQwggEiMA0GCSqGSIb3DQEB
# AQUAA4IBDwAwggEKAoIBAQCZZShLfVw6k5JK9mS83RaEwYfVIufoVnV4gWNUtqOV
# xJXqPH01aa1t3BHEvchk/BHeDjr0vGlmOJ0w8Mt7A/zP03wRoc8y7vjTJj6Iuvpg
# QA+dQLZdNJMxFGF0L+ZRmRMOAie8AXD5I7G5R/lQkBpilwm7KYRqNxpxW6zB4DSK
# KxuLVCbhHa83SaeI6zI0IU+B96ChEiDBnKqJWVvcxe4waFBqjn6gdw3GbsgUw0OE
# uPRNgI+ZS5Bt6OpFYFVC/5RK4P7g3h0yfJ3Jn7M28ytBAzsj+lJVDfBKy2XHCMrd
# tBgLtSJZu+r+Li6VPpsHP+E2NLHUkXUZ8oTj8gtLx9QRAgMBAAGjggF+MIIBejAQ
# BgkrBgEEAYI3FQEEAwIBADAdBgNVHQ4EFgQUaLdyiZ5C6uRRRR9YogonD3NgQKMw
# PQYJKwYBBAGCNxUHBDAwLgYmKwYBBAGCNxUIgcjDM4H58AaBpZ8NhOCBCIXCqksG
# g6+BUIHVsx0CAWQCARswCwYDVR0PBAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
# HwYDVR0jBBgwFoAUAS2LYWYbjNDg00w+TuxbGbARxmEwSQYDVR0fBEIwQDA+oDyg
# OoY4aHR0cDovL2NybC5wa2kudmEuZ292L3BraS9jcmwvVkEtSW50ZXJuYWwtUzIt
# UkNBMS12MS5jcmwwewYIKwYBBQUHAQEEbzBtMEcGCCsGAQUFBzAChjtodHRwOi8v
# YWlhLnBraS52YS5nb3YvcGtpL2FpYS9WQS9WQS1JbnRlcm5hbC1TMi1SQ0ExLXYx
# LmNlcjAiBggrBgEFBQcwAYYWaHR0cDovL29jc3AucGtpLnZhLmdvdjANBgkqhkiG
# 9w0BAQsFAAOCAQEAr4E5n96q069qvjjcZ0jgTj6mCvR8XLUGjm5Guwkj3EWZ3tU9
# ou+TDTSgGJW0dqWTttMbdNQ2eFcAqXonF84yxEI5cieKuty876R1ixQfD2CAuR48
# Ln6WlnszRuAyRZ/BVClsO5LFoy8Bn9hasjf42sxvzgdJpm5Nqi6mrdwpzWusKJxn
# Q/Amft/A2S4KGmXgEACImSndsns0lbDskkDD2qtY1bDcvfQi+n+rVsoQJYDwpsfq
# /6wBB77aHfyt7QNUOTq285C7z4PA4ZMqkIhv0L30RmoAGMAyR6r5Id5qNFJUzspw
# Eq/7XCPCKjR2TDWOgmgWzf6mpZW3yJx65N7UVjCCBM0wggO1oAMCAQICBz0AAAAA
# C4QwDQYJKoZIhvcNAQELBQAwRzETMBEGCgmSJomT8ixkARkWA2dvdjESMBAGCgmS
# JomT8ixkARkWAnZhMRwwGgYDVQQDExNWQS1JbnRlcm5hbC1TMi1JQ0E0MB4XDTIw
# MTIwMzEyNDg0M1oXDTIzMTIwMzEyNDg0M1owTDETMBEGCgmSJomT8ixkARkWA0dP
# VjESMBAGCgmSJomT8ixkARkWAlZBMSEwHwYDVQQDExhWQSBJTlRFUk5BTCBDT0RF
# IFNJR05JTkcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDNFUag6iW0
# iKw7pV27EGh0CBdhm1SMdI50tPwxAAWTA+lAADyKw6C5XsylQs1Ix6GrkWTwZy/W
# CepKsXUZ4rTQa+NlFftE28i2V/L6PMOvwAGatj9nPaKM3bXK5Y3LQ3gAMypxxvQd
# Q5Sj1lCw/gfSlxLMCz1daVhNWHMQ69xyob41LqIhQJMfE8FIqcn+618QCXesxKN/
# Wozy/RFTVOcGoFsTIv+sih2Y1sbXo3qUltaXiuKDkAEtpos0qz6SaG4NmsFTnIci
# DQTUe8cg3MSY4o4asvjgjwdHk9x2YAGDbHjuOXEeQhOTKB8ZtAXa97XLPm9hKzIA
# Usu2hiaIQrkNAgMBAAGjggG3MIIBszA8BgkrBgEEAYI3FQcELzAtBiUrBgEEAYI3
# FQiByMMzgfnwBoGlnw2E4IEIhcKqSwaCxeZ57dsiAgFkAgEfMBMGA1UdJQQMMAoG
# CCsGAQUFBwMDMA4GA1UdDwEB/wQEAwIGwDAbBgkrBgEEAYI3FQoEDjAMMAoGCCsG
# AQUFBwMDMB0GA1UdDgQWBBSVtBo8/RBNAiE5eHBFuJNHov/aIzAvBgNVHREEKDAm
# oCQGCisGAQQBgjcUAgOgFgwUVkFJTlRDT0RFU0lHTkB2YS5nb3YwHwYDVR0jBBgw
# FoAUaLdyiZ5C6uRRRR9YogonD3NgQKMwRgYDVR0fBD8wPTA7oDmgN4Y1aHR0cDov
# L2NybC5wa2kudmEuZ292L3BraS9jcmwvVkEtSW50ZXJuYWwtUzItSUNBNC5jcmww
# eAYIKwYBBQUHAQEEbDBqMEQGCCsGAQUFBzAChjhodHRwOi8vYWlhLnBraS52YS5n
# b3YvcGtpL2FpYS92YS9WQS1JbnRlcm5hbC1TMi1JQ0E0LmNlcjAiBggrBgEFBQcw
# AYYWaHR0cDovL29jc3AucGtpLnZhLmdvdjANBgkqhkiG9w0BAQsFAAOCAQEAkfsO
# rV8Z70ZIVkNE6HFkQ2uqrtbPUT9rxOPUszNbDnxNf51pZeie7Vj7hiwldg9ybe4Y
# Z6KMEeVwUlsOzuRNPOD6TWVGHLUg3LvrRkbOEihjNMQdIB1DkzRnPh7lNCxCGEN2
# mvJsyN7Sfqv3vXRexh3AY/19C7drZdVzG7o135mrNY6I4gOJwbDtya+hsPXONDXY
# b/59s5J+0NzNoAR+b+6FgJShZ+z05cnfVwPH3W3xIe9SgawA+FaUfSvGfVHHRR4+
# +zqDAB7FVzyf3X30S9E1NZLTaAGQ2XiCco41nnUXJKS91u+FfJcwnDgSwqYsYgws
# v1+q3nno5nLmYAlmYTGCAfMwggHvAgEBMFIwRzETMBEGCgmSJomT8ixkARkWA2dv
# djESMBAGCgmSJomT8ixkARkWAnZhMRwwGgYDVQQDExNWQS1JbnRlcm5hbC1TMi1J
# Q0E0Agc9AAAAAAuEMAkGBSsOAwIaBQCgeDAYBgorBgEEAYI3AgEMMQowCKACgACh
# AoAAMBkGCSqGSIb3DQEJAzEMBgorBgEEAYI3AgEEMBwGCisGAQQBgjcCAQsxDjAM
# BgorBgEEAYI3AgEVMCMGCSqGSIb3DQEJBDEWBBS2jT2Odp6FALCxdcR0HDrZUMBh
# OzANBgkqhkiG9w0BAQEFAASCAQBAGu6xgNPG4Ct+AS6W1kn5NJS/XV+tABYcvMs5
# VqgqiRVAnISamTt8s149B/osP0nAmqRmLNKoZ/4plNglycaH8UEKdg0Bg5g6WGVU
# D7f4a82XHcXN+2R11jJLMMzneFDgu0MM5yhzfUaqRGfXkS856dL12kaZc+4TEC7m
# 6cszCjLomfQbPx3XlZcnD1pw1mH0MoeAoUbeLL6+pkR/Gj+fJggFWgQnxSXbX1Ik
# XYvZPMRekzMu9t9378s9YhmYauYxKYjGT60vCN6A1UVbbWEHQmmPHzsbxf47chMF
# xzeXLqXhTlpiRAmeQrzvMlZf5hnV6299te/GD7YuPbVwk3wa
# SIG # End signature block