Experimental PowerShell discover extension

data.build.json

@@ -19,6 +19,8 @@
       "NOTICE.txt",
       "osinfo",
       "osinfo.dsc.resource.json",
+      "powershell.dsc.extension.json",
+      "powershell.discover.ps1",
       "powershell.dsc.resource.json",
       "psDscAdapter/",
       "psscript.ps1",
@@ -44,6 +46,8 @@
       "NOTICE.txt",
       "osinfo",
       "osinfo.dsc.resource.json",
+      "powershell.dsc.extension.json",
+      "powershell.discover.ps1",
       "powershell.dsc.resource.json",
       "psDscAdapter/",
       "psscript.ps1",
@@ -69,6 +73,8 @@
       "NOTICE.txt",
       "osinfo.exe",
       "osinfo.dsc.resource.json",
+      "powershell.dsc.extension.json",
+      "powershell.discover.ps1",
       "powershell.dsc.resource.json",
       "psDscAdapter/",
       "psscript.ps1",
@@ -191,6 +197,17 @@
         ]
       }
     },
+    {
+      "Name": "extensions/powershell",
+      "Kind": "Extension",
+      "RelativePath": "extensions/powershell",
+      "CopyFiles": {
+        "All": [
+          "powershell.discover.ps1",
+          "powershell.dsc.extension.json"
+        ]
+      }
+    },
     {
       "Name": "tree-sitter-dscexpression",
       "Kind": "Grammar",

docs/reference/schemas/extension/stdout/discover.md

@@ -22,13 +22,16 @@ Type:          object
 
 ## Description
 
-Represents the actual state of a resource instance in DSCpath to a discovered DSC resource or
+Represents the actual state of a resource instance in DSC path to a discovered DSC resource or
 extension manifest on the system. DSC expects every JSON Line emitted to stdout for the
 **Discover** operation to adhere to this schema.
 
 The output must be a JSON object. The object must define the full path to the discovered manifest.
 If an extension returns JSON that is invalid against this schema, DSC raises an error.
 
+If the extension doesn't discover any manifests, it must return nothing to stdout. An empty output
+indicates no resources were found.
+
 ## Required Properties
 
 The output for the `discover` operation must include these properties:
@@ -43,6 +46,9 @@ The value for this property must be the absolute path to a manifest file on the
 manifest can be for a DSC resource or extension. If the returned path doesn't exist, DSC raises an
 error.
 
+Each discovered manifest must be emitted as a separate JSON Line to stdout. If no manifests are
+discovered, the extension must not emit any output to stdout.
+
 ```yaml
 Type:     string
 Required: true

dsc/tests/dsc_extension_discover.tests.ps1

@@ -24,21 +24,29 @@ Describe 'Discover extension tests' {
         $out = dsc extension list | ConvertFrom-Json
         $LASTEXITCODE | Should -Be 0
         if ($IsWindows) {
-            $out.Count | Should -Be 2 -Because ($out | Out-String)
-            $out[0].type | Should -Be 'Microsoft.Windows.Appx/Discover'
-            $out[0].version | Should -Be '0.1.0'
+            $out.Count | Should -Be 3 -Because ($out | Out-String)
+            $out[0].type | Should -BeExactly 'Microsoft.PowerShell/Discover'
+            $out[0].version | Should -BeExactly '0.1.0'
             $out[0].capabilities | Should -BeExactly @('discover')
             $out[0].manifest | Should -Not -BeNullOrEmpty
-            $out[1].type | Should -BeExactly 'Test/Discover'
+            $out[1].type | Should -BeExactly 'Microsoft.Windows.Appx/Discover'
             $out[1].version | Should -BeExactly '0.1.0'
             $out[1].capabilities | Should -BeExactly @('discover')
             $out[1].manifest | Should -Not -BeNullOrEmpty
+            $out[2].type | Should -BeExactly 'Test/Discover'
+            $out[2].version | Should -BeExactly '0.1.0'
+            $out[2].capabilities | Should -BeExactly @('discover')
+            $out[2].manifest | Should -Not -BeNullOrEmpty
         } else {
-            $out.Count | Should -Be 1 -Because ($out | Out-String)
-            $out[0].type | Should -BeExactly 'Test/Discover'
+            $out.Count | Should -Be 2 -Because ($out | Out-String)
+            $out[0].type | Should -BeExactly 'Microsoft.PowerShell/Discover'
             $out[0].version | Should -BeExactly '0.1.0'
             $out[0].capabilities | Should -BeExactly @('discover')
             $out[0].manifest | Should -Not -BeNullOrEmpty
+            $out[1].type | Should -BeExactly 'Test/Discover'
+            $out[1].version | Should -BeExactly '0.1.0'
+            $out[1].capabilities | Should -BeExactly @('discover')
+            $out[1].manifest | Should -Not -BeNullOrEmpty
         }
     }
 
@@ -148,4 +156,4 @@ Describe 'Discover extension tests' {
             $env:DSC_RESOURCE_PATH = $null
         }
     }
-}
+}

extensions/powershell/.project.data.json

@@ -0,0 +1,10 @@
+{
+    "Name": "extensions/powershell",
+    "Kind": "Extension",
+    "CopyFiles": {
+        "All": [
+            "powershell.discover.ps1",
+            "powershell.dsc.extension.json"
+        ]
+    }
+}

extensions/powershell/copy_files.txt

@@ -0,0 +1,2 @@
+powershell.discover.ps1
+powershell.dsc.extension.json

extensions/powershell/powershell.discover.ps1

@@ -0,0 +1,122 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+[CmdletBinding()]
+param ()
+
+function Get-CacheFilePath {
+    if ($IsWindows) {
+        Join-Path $env:LocalAppData "dsc\PowerShellDiscoverCache.json"
+    } else {
+        Join-Path $env:HOME ".dsc" "PowerShellDiscoverCache.json"
+    }
+}
+
+function Test-CacheValid {
+    param([string]$CacheFilePath, [string[]]$PSPaths)
+
+    if (-not (Test-Path $CacheFilePath)) {
+        return $false
+    }
+
+    try {
+        $cache = Get-Content -Raw $CacheFilePath | ConvertFrom-Json
+
+        foreach ($entry in $cache.PathInfo.PSObject.Properties) {
+            $path = $entry.Name
+            if (-not (Test-Path $path)) {
+                return $false
+            }
+
+            $currentLastWrite = (Get-Item $path).LastWriteTimeUtc
+            $cachedLastWrite = [DateTime]$entry.Value
+
+            if ($currentLastWrite -ne $cachedLastWrite) {
+                return $false
+            }
+        }
+
+        $cachedPaths = [string[]]$cache.PSModulePaths
+        if ($cachedPaths.Count -ne $PSPaths.Count) {
+            return $false
+        }
+
+        $diff = Compare-Object $cachedPaths $PSPaths
+        if ($null -ne $diff) {
+            return $false
+        }
+
+        foreach ($manifest in $cache.Manifests) {
+            if (-not (Test-Path -LiteralPath $manifest.manifestPath)) {
+                return $false
+            }
+        }
+
+        return $true
+    } catch {
+        return $false
+    }
+}
+
+function Invoke-DscResourceDiscovery {
+    [CmdletBinding()]
+    param()
+
+    begin {
+        $psPaths = $env:PSModulePath -split [System.IO.Path]::PathSeparator | Where-Object { $_ -notmatch 'WindowsPowerShell' }
+
+        $cacheFilePath = Get-CacheFilePath
+        $useCache = Test-CacheValid -CacheFilePath $cacheFilePath -PSPaths $psPaths
+    }
+    process {
+        if ($useCache) {
+            $cache = Get-Content -Raw $cacheFilePath | ConvertFrom-Json
+            $manifests = $cache.Manifests
+        } else {
+            $manifests = $psPaths | ForEach-Object -Parallel {
+                $searchPatterns = @('*.dsc.resource.json', '*.dsc.resource.yaml', '*.dsc.resource.yml')
+                $enumOptions = [System.IO.EnumerationOptions]@{ IgnoreInaccessible = $true; RecurseSubdirectories = $true }
+                foreach ($pattern in $searchPatterns) {
+                    try {
+                        [System.IO.Directory]::EnumerateFiles($_, $pattern, $enumOptions) | ForEach-Object {
+                            @{ manifestPath = $_ }
+                        }
+                    } catch { }
+                }
+            } -ThrottleLimit 10
+
+            $pathInfo = @{}
+            foreach ($path in $psPaths) {
+                $item = Get-Item -LiteralPath $path -ErrorAction Ignore
+                if ($item) {
+                    $pathInfo[$path] = $item.LastWriteTimeUtc
+                    # Track each module subdirectory so that manifest changes inside an
+                    # already-known module are detected, even if the parent directory timestamp isn't updated.
+                    Get-ChildItem -LiteralPath $path -Directory -ErrorAction Ignore | ForEach-Object {
+                        $pathInfo[$_.FullName] = $_.LastWriteTimeUtc
+                    }
+                }
+            }
+
+            $cacheObject = @{
+                PSModulePaths = $psPaths
+                PathInfo      = $pathInfo
+                Manifests     = $manifests
+            }
+
+            $cacheDir = Split-Path $cacheFilePath -Parent
+            if (-not (Test-Path $cacheDir)) {
+                New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null
+            }
+
+            $cacheObject | ConvertTo-Json -Depth 10 | Set-Content -Path $cacheFilePath -Force
+        }
+    }
+    end {
+        if ($manifests.Count -gt 0) {
+            $manifests | ForEach-Object { $_ | ConvertTo-Json -Compress }
+        }
+    }
+}
+
+Invoke-DscResourceDiscovery

extensions/powershell/powershell.discover.tests.ps1

@@ -0,0 +1,130 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+BeforeAll {
+    $fakeManifest = @{
+        '$schema' = "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json"
+        type      = "Test/FakeResource"
+        version   = "0.1.0"
+        get       = @{
+            executable = "fakeResource"
+            args       = @(
+                "get",
+                @{
+                    jsonInputArg = "--input"
+                    mandatory    = $true
+                }
+            )
+        }
+    }
+
+    $manifestPath = Join-Path $TestDrive "fake.dsc.resource.json"
+    $fakeManifest | ConvertTo-Json -Depth 10 | Set-Content -Path $manifestPath
+    $script:OldPSModulePath = $env:PSModulePath
+    $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestDrive
+
+    $script:discoverScript = Join-Path $PSScriptRoot "powershell.discover.ps1"
+
+    $cacheFilePath = if ($IsWindows) {
+        Join-Path $env:LocalAppData "dsc\PowerShellDiscoverCache.json"
+    } else {
+        Join-Path $env:HOME ".dsc" "PowerShellDiscoverCache.json"
+    }
+    $script:cacheFilePath = $cacheFilePath
+
+    Remove-Item -Force -ErrorAction SilentlyContinue -Path $script:cacheFilePath
+}
+
+AfterAll {
+    $env:PSModulePath = $script:OldPSModulePath
+}
+
+Describe 'Tests for PowerShell resource discovery' {    
+    It 'Should create cache file on first run' {
+        $script:cacheFilePath | Should -Not -Exist
+        $null = & $script:discoverScript 2>&1
+        $script:cacheFilePath | Should -Exist
+
+        $cache = Get-Content -Raw $script:cacheFilePath | ConvertFrom-Json
+        $cache.PSModulePaths | Should -Not -BeNullOrEmpty
+        $cache.PathInfo | Should -Not -BeNullOrEmpty
+        $cache.Manifests | Should -Not -BeNullOrEmpty
+    }
+
+    It 'Should find DSC PowerShell resources' {
+        $out = dsc resource list | ConvertFrom-Json
+        $LASTEXITCODE | Should -Be 0
+        $out.manifest.type | Should -Contain 'Test/FakeResource'
+    }
+
+    It 'Should use cache on subsequent runs' {
+        $null = & $script:discoverScript 2>&1
+        $script:cacheFilePath | Should -Exist
+
+        $cacheLastWriteTime = (Get-Item $script:cacheFilePath).LastWriteTimeUtc
+
+        Start-Sleep -Milliseconds 100
+
+        $null = & $script:discoverScript 2>&1
+
+        $newLastWriteTime = (Get-Item $script:cacheFilePath).LastWriteTimeUtc
+        $newLastWriteTime | Should -Be $cacheLastWriteTime
+    }
+
+    It 'Should invalidate cache when PSModulePath changes' {
+        $null = & $script:discoverScript 2>&1
+        $script:cacheFilePath | Should -Exist
+
+        $cache = Get-Content -Raw $script:cacheFilePath | ConvertFrom-Json
+        $originalPaths = $cache.PSModulePaths
+        $cache.PSModulePaths = @($originalPaths[0])  # Remove some paths
+        $cache | ConvertTo-Json -Depth 10 | Set-Content -Path $script:cacheFilePath -Force
+
+        $cacheLastWriteTime = (Get-Item $script:cacheFilePath).LastWriteTimeUtc
+        Start-Sleep -Milliseconds 100
+
+        $null = & $script:discoverScript 2>&1
+
+        $newLastWriteTime = (Get-Item $script:cacheFilePath).LastWriteTimeUtc
+        $newLastWriteTime | Should -Not -Be $cacheLastWriteTime
+    }
+
+    It 'Should invalidate cache when module directory is modified' {
+        $null = & $script:discoverScript 2>&1
+        $script:cacheFilePath | Should -Exist
+
+        $cache = Get-Content -Raw $script:cacheFilePath | ConvertFrom-Json
+
+        $firstPath = $cache.PathInfo.PSObject.Properties | Select-Object -First 1
+        if ($firstPath) {
+            $oldTimestamp = [DateTime]$firstPath.Value
+            $newTimestamp = $oldTimestamp.AddDays(-1)
+            $cache.PathInfo.($firstPath.Name) = $newTimestamp
+            $cache | ConvertTo-Json -Depth 10 | Set-Content -Path $script:cacheFilePath -Force
+
+            $cacheLastWriteTime = (Get-Item $script:cacheFilePath).LastWriteTimeUtc
+            Start-Sleep -Milliseconds 100
+
+            $null = & $script:discoverScript 2>&1
+
+            $newLastWriteTime = (Get-Item $script:cacheFilePath).LastWriteTimeUtc
+            $newLastWriteTime | Should -Not -Be $cacheLastWriteTime
+        }
+    }
+
+    It 'Should rebuild cache if cache file is corrupted' {
+        "{ invalid json }" | Set-Content -Path $script:cacheFilePath -Force
+        $script:cacheFilePath | Should -Exist
+
+        $null = & $script:discoverScript 2>&1
+
+        $cache = Get-Content -Raw $script:cacheFilePath | ConvertFrom-Json
+        $cache.PSModulePaths | Should -Not -BeNullOrEmpty
+        $cache.PathInfo | Should -Not -BeNullOrEmpty
+    }
+
+    It 'Should include test manifest in discovery results' {
+        $out = & $script:discoverScript | ConvertFrom-Json
+        $out.manifestPath | Should -Contain $manifestPath
+    }
+}