Add dry_run option to the accessibility scanner

.github/actions/file/action.yml

@@ -24,6 +24,10 @@ inputs:
     description: "In the 'file' step, also open grouped issues which link to all issues with the same root cause"
     required: false
     default: "false"
+  dry_run:
+    description: "When true, log the issues that would be filed without opening, closing, or reopening any issues."
+    required: false
+    default: "false"
 
 outputs:
   filings_file:

.github/actions/file/src/index.ts

@@ -29,12 +29,14 @@ export default async function () {
     ? JSON.parse(fs.readFileSync(cachedFilingsFile, 'utf8'))
     : []
   const shouldOpenGroupedIssues = core.getBooleanInput('open_grouped_issues')
+  const dryRun = core.getBooleanInput('dry_run')
   core.debug(`Input: 'findings_file: ${findingsFile}'`)
   core.debug(`Input: 'repository: ${repoWithOwner}'`)
   core.debug(`Input: 'base_url: ${baseUrl ?? '(default)'}'`)
   core.debug(`Input: 'screenshot_repository: ${screenshotRepo}'`)
   core.debug(`Input: 'cached_filings_file: ${cachedFilingsFile}'`)
   core.debug(`Input: 'open_grouped_issues: ${shouldOpenGroupedIssues}'`)
+  core.debug(`Input: 'dry_run: ${dryRun}'`)
 
   const octokit = new OctokitWithThrottling({
     auth: token,
@@ -61,10 +63,26 @@ export default async function () {
   // Track new issues for grouping
   const newIssuesByProblemShort: Record<string, FindingGroupIssue[]> = {}
   const trackingIssueUrls: Record<string, string> = {}
+  const dryRunCounts = {open: 0, reopen: 0, close: 0}
 
   for (const filing of filings) {
     let response: OctokitResponse<IssueResponse> | undefined
     try {
+      if (dryRun) {
+        if (isResolvedFiling(filing)) {
+          dryRunCounts.close++
+          core.info(`[dry run] Would CLOSE issue: ${filing.issue.url}`)
+        } else if (isNewFiling(filing)) {
+          dryRunCounts.open++
+          core.info(
+            `[dry run] Would OPEN a new issue for: ${filing.findings[0].problemShort} (${filing.findings[0].url})`,
+          )
+        } else if (isRepeatedFiling(filing)) {
+          dryRunCounts.reopen++
+          core.info(`[dry run] Would REOPEN issue: ${filing.issue.url}`)
+        }
+        continue
+      }
       if (isResolvedFiling(filing)) {
         // Close the filing’s issue (if necessary)
         response = await closeIssue(octokit, new Issue(filing.issue))
@@ -114,7 +132,7 @@ export default async function () {
 
   // Open tracking issues for groups with >1 new issue and link back from each
   // new issue
-  if (shouldOpenGroupedIssues) {
+  if (shouldOpenGroupedIssues && !dryRun) {
     for (const [problemShort, issues] of Object.entries(newIssuesByProblemShort)) {
       if (issues.length > 1) {
         const capitalizedProblemShort = problemShort[0].toUpperCase() + problemShort.slice(1)
@@ -138,6 +156,16 @@ export default async function () {
     }
   }
 
+  if (dryRun) {
+    core.info('[dry run] Summary of actions that would be taken:')
+    console.table({
+      open: dryRunCounts.open,
+      reopen: dryRunCounts.reopen,
+      close: dryRunCounts.close,
+      total: dryRunCounts.open + dryRunCounts.reopen + dryRunCounts.close,
+    })
+  }
+
   const filingsPath = path.join(process.env.RUNNER_TEMP || '/tmp', `filings-${crypto.randomUUID()}.json`)
   fs.writeFileSync(filingsPath, JSON.stringify(filings))
   core.setOutput('filings_file', filingsPath)

.github/actions/file/tests/dryRun.test.ts

@@ -0,0 +1,163 @@
+import {describe, it, expect, vi, beforeEach, afterEach} from 'vitest'
+
+// --- Mock the issue-mutating helpers so we can assert they are NEVER called in dry run ---
+const openIssue = vi.fn()
+const reopenIssue = vi.fn()
+const closeIssue = vi.fn()
+vi.mock('../src/openIssue.js', () => ({openIssue: (...args: unknown[]) => openIssue(...args)}))
+vi.mock('../src/reopenIssue.js', () => ({reopenIssue: (...args: unknown[]) => reopenIssue(...args)}))
+vi.mock('../src/closeIssue.js', () => ({closeIssue: (...args: unknown[]) => closeIssue(...args)}))
+
+// --- Mock @actions/core: control inputs, capture logs/outputs ---
+const inputs: Record<string, string> = {}
+const infoLines: string[] = []
+const outputs: Record<string, string> = {}
+vi.mock('@actions/core', () => ({
+  getInput: (name: string) => inputs[name] ?? '',
+  getBooleanInput: (name: string) => (inputs[name] ?? 'false') === 'true',
+  setOutput: (name: string, value: string) => {
+    outputs[name] = value
+  },
+  info: (msg: string) => {
+    infoLines.push(msg)
+  },
+  debug: () => {},
+  warning: () => {},
+  setFailed: () => {},
+}))
+
+// --- Mock fs: feed findings/cached filings in, swallow the output write ---
+const files: Record<string, string> = {}
+vi.mock('node:fs', () => ({
+  default: {
+    readFileSync: (p: string) => files[p],
+    writeFileSync: (p: string, data: string) => {
+      files[p] = data
+    },
+  },
+}))
+
+// --- Stub Octokit so constructing it in index.ts doesn't do anything real ---
+const octokitRequest = vi.fn()
+vi.mock('@octokit/core', () => ({
+  Octokit: {
+    plugin: () =>
+      class {
+        request = octokitRequest
+      },
+  },
+}))
+vi.mock('@octokit/plugin-throttling', () => ({throttling: {}}))
+
+import runFileAction from '../src/index.ts'
+
+const finding = {
+  scannerType: 'axe',
+  ruleId: 'color-contrast',
+  url: 'https://example.com/page',
+  html: '<span>Low contrast</span>',
+  problemShort: 'elements must meet minimum color contrast ratio thresholds',
+  problemUrl: 'https://dequeuniversity.com/rules/axe/4.10/color-contrast',
+  solutionShort: 'ensure sufficient contrast',
+}
+
+// A second finding with no matching cached filing -> NEW (open)
+const newFinding = {...finding, ruleId: 'heading-order', html: '<h3>Skipped</h3>'}
+
+// A cached filing whose finding matches `finding` -> REPEATED (reopen)
+const repeatedCached = {
+  issue: {id: 1, nodeId: 'N1', url: 'https://github.com/org/repo/issues/1', title: 'repeat'},
+  findings: [finding],
+}
+
+// A cached filing with NO matching finding this run -> RESOLVED (close)
+const resolvedCached = {
+  issue: {id: 2, nodeId: 'N2', url: 'https://github.com/org/repo/issues/2', title: 'resolved'},
+  findings: [{...finding, ruleId: 'landmark-one-main', html: '<div>old</div>'}],
+}
+
+function setup() {
+  // findings file: includes `finding` (matches repeatedCached) and `newFinding` (brand new)
+  files['/tmp/findings.json'] = JSON.stringify([finding, newFinding])
+  // cached filings: one repeated, one resolved (its finding is absent from findings file)
+  files['/tmp/cached.json'] = JSON.stringify([repeatedCached, resolvedCached])
+  inputs.findings_file = '/tmp/findings.json'
+  inputs.cached_filings_file = '/tmp/cached.json'
+  inputs.repository = 'org/repo'
+  inputs.token = 'fake-token'
+}
+
+describe('file action — dry_run', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    infoLines.length = 0
+    for (const k of Object.keys(inputs)) delete inputs[k]
+    for (const k of Object.keys(outputs)) delete outputs[k]
+    vi.spyOn(console, 'table').mockImplementation(() => {})
+  })
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('does not open, reopen, or close any issues when dry_run is true', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    expect(openIssue).not.toHaveBeenCalled()
+    expect(reopenIssue).not.toHaveBeenCalled()
+    expect(closeIssue).not.toHaveBeenCalled()
+    expect(octokitRequest).not.toHaveBeenCalled()
+  })
+
+  it('logs the intended action for each filing type', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    const log = infoLines.join('\n')
+    expect(log).toContain(
+      '[dry run] Would OPEN a new issue for: elements must meet minimum color contrast ratio thresholds (https://example.com/page)',
+    )
+    expect(log).toContain('[dry run] Would REOPEN issue: https://github.com/org/repo/issues/1')
+    expect(log).toContain('[dry run] Would CLOSE issue: https://github.com/org/repo/issues/2')
+  })
+
+  it('logs a summary table with counts', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    expect(vi.mocked(console.table)).toHaveBeenCalledWith(
+      expect.objectContaining({open: 1, reopen: 1, close: 1, total: 3}),
+    )
+  })
+
+  it('still writes the filings_file output in dry run', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    expect(outputs.filings_file).toBeDefined()
+  })
+
+  it('does call the mutating helpers when dry_run is false (regression guard)', async () => {
+    setup()
+    inputs.dry_run = 'false'
+    // helpers return a minimal Octokit-style response so index.ts can read response.data
+    const resp = {data: {id: 9, node_id: 'N', number: 9, html_url: 'https://github.com/org/repo/issues/9', title: 't'}}
+    openIssue.mockResolvedValue(resp)
+    reopenIssue.mockResolvedValue(resp)
+    closeIssue.mockResolvedValue(resp)
+
+    await runFileAction()
+
+    expect(openIssue).toHaveBeenCalled()
+    expect(reopenIssue).toHaveBeenCalled()
+    expect(closeIssue).toHaveBeenCalled()
+  })
+})

FAQ.md

@@ -60,6 +60,17 @@ Just keep in mind that resetting the cache means the Action will "forget" what
 it's already seen, so it may reopen issues that were previously tracked or
 closed.
 
+### How can I preview what the scanner would do without filing issues?
+
+Set the `dry_run` input to `true`. The scanner will run a normal scan and log the
+issues it _would_ open, reopen, or close — but it won't create, close, reopen, or
+assign any issues, and it won't write to the `gh-cache` branch.
+
+This is handy for trying out a new configuration or seeing how many issues a scan
+would file, without making any changes to your repository. Because dry runs don't
+update the cache, your next real run behaves exactly as if the dry run never
+happened.
+
 ### Does this work with private repositories?
 
 Yes! The Action works with both public and private repositories. Since it runs

README.md

@@ -55,6 +55,7 @@ jobs:
           # skip_copilot_assignment: false # Optional: Set to true to skip assigning issues to GitHub Copilot (or if you don't have GitHub Copilot)
           # include_screenshots: false # Optional: Set to true to capture screenshots and include links to them in filed issues
           # open_grouped_issues: false # Optional: Set to true to open an issue grouping individual issues per violation
+          # dry_run: false # Optional: Set to true to scan and log what would be filed without creating/closing issues or writing the cache
           # reduced_motion: no-preference # Optional: Playwright reduced motion configuration option
           # color_scheme: light # Optional: Playwright color scheme configuration option
           # scans: '["axe","reflow-scan"]' # Optional: An array of scans (or plugins) to be performed. If not provided, only Axe will be performed.
@@ -131,6 +132,7 @@ Trigger the workflow manually or automatically based on your configuration. The
 | `reduced_motion`          | No       | Playwright `reducedMotion` setting for scan contexts. Allowed values: `reduce`, `no-preference`                                                                                                                                                                  | `reduce`                                                                    |
 | `color_scheme`            | No       | Playwright `colorScheme` setting for scan contexts. Allowed values: `light`, `dark`, `no-preference`                                                                                                                                                             | `dark`                                                                      |
 | `scans`                   | No       | An array of scans (or plugins) to be performed. If not provided, only Axe will be performed.                                                                                                                                                                     | `'["axe", "reflow-scan", ...other plugins]'`                                |
+| `dry_run`                 | No       | When `true`, scan and log the issues that _would_ be filed without opening, closing, reopening, or assigning any issues — and without writing to the `gh-cache` branch. Useful for safely previewing results. Default: `false`                                   | `true`                                                                      |
 | `url_configs`             | No       | A stringified JSON array of URL config objects. Each object must have a `url` field and may have an optional `excludeSelectors` field (array of CSS selectors to exclude from the Axe scan for that URL). When provided, takes precedence over the `urls` input. | `'[{"url":"https://example.com","excludeSelectors":["iframe","#widget"]}]'` |
 
 ---

action.yml

@@ -54,6 +54,10 @@ inputs:
   scans:
     description: 'Stringified JSON array of scans to perform. If not provided, only Axe will be performed'
     required: false
+  dry_run:
+    description: 'When true, scan and log the issues that would be filed without opening, closing, reopening, or assigning any issues, and without writing to the cache.'
+    required: false
+    default: 'false'
 
 outputs:
   results:
@@ -129,6 +133,7 @@ runs:
         cached_filings_file: ${{ steps.normalize_cache.outputs.cached_filings_file }}
         screenshot_repository: ${{ github.repository }}
         open_grouped_issues: ${{ inputs.open_grouped_issues }}
+        dry_run: ${{ inputs.dry_run }}
     - if: ${{ steps.file.outputs.filings_file }}
       name: Get issues from filings
       id: get_issues_from_filings
@@ -137,7 +142,7 @@ runs:
         # Extract open issues from Filing objects and write to a file
         jq -c '[.[] | select(.issue.state == "open") | .issue]' "${{ steps.file.outputs.filings_file }}" > "$RUNNER_TEMP/issues.json"
         echo "issues_file=$RUNNER_TEMP/issues.json" >> "$GITHUB_OUTPUT"
-    - if: ${{ inputs.skip_copilot_assignment != 'true' }}
+    - if: ${{ inputs.skip_copilot_assignment != 'true' && inputs.dry_run != 'true' }}
       name: Fix
       id: fix
       uses: ./../../_actions/github/accessibility-scanner/current/.github/actions/fix
@@ -185,19 +190,20 @@ runs:
 
         # Set results_file output
         echo "results_file=$RESULTS_FILE" >> "$GITHUB_OUTPUT"
-    - if: ${{ inputs.include_screenshots == 'true' }}
+    - if: ${{ inputs.include_screenshots == 'true' && inputs.dry_run != 'true' }}
       name: Save screenshots
       uses: ./../../_actions/github/accessibility-scanner/current/.github/actions/gh-cache/save
       with:
         path: .screenshots
         token: ${{ inputs.token }}
     - name: Copy results to cache path
+      if: ${{ inputs.dry_run != 'true' }}
       shell: bash
       run: |
         mkdir -p "$(dirname '${{ inputs.cache_key }}')"
         cp "$GITHUB_WORKSPACE/scanner-results.json" "${{ inputs.cache_key }}"
-
     - name: Save cached results
+      if: ${{ inputs.dry_run != 'true' }}
       uses: ./../../_actions/github/accessibility-scanner/current/.github/actions/gh-cache/save
       with:
         path: ${{ inputs.cache_key }}