diff --git a/.github/actions/file/action.yml b/.github/actions/file/action.yml index 836e125d..7d76af7a 100644 --- a/.github/actions/file/action.yml +++ b/.github/actions/file/action.yml @@ -24,6 +24,10 @@ inputs: description: "In the 'file' step, also open grouped issues which link to all issues with the same root cause" required: false default: "false" + dry_run: + description: "When true, log the issues that would be filed without opening, closing, or reopening any issues." + required: false + default: "false" outputs: filings_file: diff --git a/.github/actions/file/src/index.ts b/.github/actions/file/src/index.ts index 86d14ec8..506fdff7 100644 --- a/.github/actions/file/src/index.ts +++ b/.github/actions/file/src/index.ts @@ -29,12 +29,14 @@ export default async function () { ? JSON.parse(fs.readFileSync(cachedFilingsFile, 'utf8')) : [] const shouldOpenGroupedIssues = core.getBooleanInput('open_grouped_issues') + const dryRun = core.getBooleanInput('dry_run') core.debug(`Input: 'findings_file: ${findingsFile}'`) core.debug(`Input: 'repository: ${repoWithOwner}'`) core.debug(`Input: 'base_url: ${baseUrl ?? '(default)'}'`) core.debug(`Input: 'screenshot_repository: ${screenshotRepo}'`) core.debug(`Input: 'cached_filings_file: ${cachedFilingsFile}'`) core.debug(`Input: 'open_grouped_issues: ${shouldOpenGroupedIssues}'`) + core.debug(`Input: 'dry_run: ${dryRun}'`) const octokit = new OctokitWithThrottling({ auth: token, @@ -61,10 +63,26 @@ export default async function () { // Track new issues for grouping const newIssuesByProblemShort: Record = {} const trackingIssueUrls: Record = {} + const dryRunCounts = {open: 0, reopen: 0, close: 0} for (const filing of filings) { let response: OctokitResponse | undefined try { + if (dryRun) { + if (isResolvedFiling(filing)) { + dryRunCounts.close++ + core.info(`[dry run] Would CLOSE issue: ${filing.issue.url}`) + } else if (isNewFiling(filing)) { + dryRunCounts.open++ + core.info( + `[dry run] Would OPEN a new issue for: ${filing.findings[0].problemShort} (${filing.findings[0].url})`, + ) + } else if (isRepeatedFiling(filing)) { + dryRunCounts.reopen++ + core.info(`[dry run] Would REOPEN issue: ${filing.issue.url}`) + } + continue + } if (isResolvedFiling(filing)) { // Close the filing’s issue (if necessary) response = await closeIssue(octokit, new Issue(filing.issue)) @@ -114,7 +132,7 @@ export default async function () { // Open tracking issues for groups with >1 new issue and link back from each // new issue - if (shouldOpenGroupedIssues) { + if (shouldOpenGroupedIssues && !dryRun) { for (const [problemShort, issues] of Object.entries(newIssuesByProblemShort)) { if (issues.length > 1) { const capitalizedProblemShort = problemShort[0].toUpperCase() + problemShort.slice(1) @@ -138,6 +156,16 @@ export default async function () { } } + if (dryRun) { + core.info('[dry run] Summary of actions that would be taken:') + console.table({ + open: dryRunCounts.open, + reopen: dryRunCounts.reopen, + close: dryRunCounts.close, + total: dryRunCounts.open + dryRunCounts.reopen + dryRunCounts.close, + }) + } + const filingsPath = path.join(process.env.RUNNER_TEMP || '/tmp', `filings-${crypto.randomUUID()}.json`) fs.writeFileSync(filingsPath, JSON.stringify(filings)) core.setOutput('filings_file', filingsPath) diff --git a/.github/actions/file/tests/dryRun.test.ts b/.github/actions/file/tests/dryRun.test.ts new file mode 100644 index 00000000..53f231f5 --- /dev/null +++ b/.github/actions/file/tests/dryRun.test.ts @@ -0,0 +1,163 @@ +import {describe, it, expect, vi, beforeEach, afterEach} from 'vitest' + +// --- Mock the issue-mutating helpers so we can assert they are NEVER called in dry run --- +const openIssue = vi.fn() +const reopenIssue = vi.fn() +const closeIssue = vi.fn() +vi.mock('../src/openIssue.js', () => ({openIssue: (...args: unknown[]) => openIssue(...args)})) +vi.mock('../src/reopenIssue.js', () => ({reopenIssue: (...args: unknown[]) => reopenIssue(...args)})) +vi.mock('../src/closeIssue.js', () => ({closeIssue: (...args: unknown[]) => closeIssue(...args)})) + +// --- Mock @actions/core: control inputs, capture logs/outputs --- +const inputs: Record = {} +const infoLines: string[] = [] +const outputs: Record = {} +vi.mock('@actions/core', () => ({ + getInput: (name: string) => inputs[name] ?? '', + getBooleanInput: (name: string) => (inputs[name] ?? 'false') === 'true', + setOutput: (name: string, value: string) => { + outputs[name] = value + }, + info: (msg: string) => { + infoLines.push(msg) + }, + debug: () => {}, + warning: () => {}, + setFailed: () => {}, +})) + +// --- Mock fs: feed findings/cached filings in, swallow the output write --- +const files: Record = {} +vi.mock('node:fs', () => ({ + default: { + readFileSync: (p: string) => files[p], + writeFileSync: (p: string, data: string) => { + files[p] = data + }, + }, +})) + +// --- Stub Octokit so constructing it in index.ts doesn't do anything real --- +const octokitRequest = vi.fn() +vi.mock('@octokit/core', () => ({ + Octokit: { + plugin: () => + class { + request = octokitRequest + }, + }, +})) +vi.mock('@octokit/plugin-throttling', () => ({throttling: {}})) + +import runFileAction from '../src/index.ts' + +const finding = { + scannerType: 'axe', + ruleId: 'color-contrast', + url: 'https://example.com/page', + html: 'Low contrast', + problemShort: 'elements must meet minimum color contrast ratio thresholds', + problemUrl: 'https://dequeuniversity.com/rules/axe/4.10/color-contrast', + solutionShort: 'ensure sufficient contrast', +} + +// A second finding with no matching cached filing -> NEW (open) +const newFinding = {...finding, ruleId: 'heading-order', html: '

Skipped

'} + +// A cached filing whose finding matches `finding` -> REPEATED (reopen) +const repeatedCached = { + issue: {id: 1, nodeId: 'N1', url: 'https://github.com/org/repo/issues/1', title: 'repeat'}, + findings: [finding], +} + +// A cached filing with NO matching finding this run -> RESOLVED (close) +const resolvedCached = { + issue: {id: 2, nodeId: 'N2', url: 'https://github.com/org/repo/issues/2', title: 'resolved'}, + findings: [{...finding, ruleId: 'landmark-one-main', html: '
old
'}], +} + +function setup() { + // findings file: includes `finding` (matches repeatedCached) and `newFinding` (brand new) + files['/tmp/findings.json'] = JSON.stringify([finding, newFinding]) + // cached filings: one repeated, one resolved (its finding is absent from findings file) + files['/tmp/cached.json'] = JSON.stringify([repeatedCached, resolvedCached]) + inputs.findings_file = '/tmp/findings.json' + inputs.cached_filings_file = '/tmp/cached.json' + inputs.repository = 'org/repo' + inputs.token = 'fake-token' +} + +describe('file action — dry_run', () => { + beforeEach(() => { + vi.clearAllMocks() + infoLines.length = 0 + for (const k of Object.keys(inputs)) delete inputs[k] + for (const k of Object.keys(outputs)) delete outputs[k] + vi.spyOn(console, 'table').mockImplementation(() => {}) + }) + afterEach(() => { + vi.restoreAllMocks() + }) + + it('does not open, reopen, or close any issues when dry_run is true', async () => { + setup() + inputs.dry_run = 'true' + + await runFileAction() + + expect(openIssue).not.toHaveBeenCalled() + expect(reopenIssue).not.toHaveBeenCalled() + expect(closeIssue).not.toHaveBeenCalled() + expect(octokitRequest).not.toHaveBeenCalled() + }) + + it('logs the intended action for each filing type', async () => { + setup() + inputs.dry_run = 'true' + + await runFileAction() + + const log = infoLines.join('\n') + expect(log).toContain( + '[dry run] Would OPEN a new issue for: elements must meet minimum color contrast ratio thresholds (https://example.com/page)', + ) + expect(log).toContain('[dry run] Would REOPEN issue: https://github.com/org/repo/issues/1') + expect(log).toContain('[dry run] Would CLOSE issue: https://github.com/org/repo/issues/2') + }) + + it('logs a summary table with counts', async () => { + setup() + inputs.dry_run = 'true' + + await runFileAction() + + expect(vi.mocked(console.table)).toHaveBeenCalledWith( + expect.objectContaining({open: 1, reopen: 1, close: 1, total: 3}), + ) + }) + + it('still writes the filings_file output in dry run', async () => { + setup() + inputs.dry_run = 'true' + + await runFileAction() + + expect(outputs.filings_file).toBeDefined() + }) + + it('does call the mutating helpers when dry_run is false (regression guard)', async () => { + setup() + inputs.dry_run = 'false' + // helpers return a minimal Octokit-style response so index.ts can read response.data + const resp = {data: {id: 9, node_id: 'N', number: 9, html_url: 'https://github.com/org/repo/issues/9', title: 't'}} + openIssue.mockResolvedValue(resp) + reopenIssue.mockResolvedValue(resp) + closeIssue.mockResolvedValue(resp) + + await runFileAction() + + expect(openIssue).toHaveBeenCalled() + expect(reopenIssue).toHaveBeenCalled() + expect(closeIssue).toHaveBeenCalled() + }) +}) diff --git a/FAQ.md b/FAQ.md index 2fc40164..2b26ab99 100644 --- a/FAQ.md +++ b/FAQ.md @@ -60,6 +60,17 @@ Just keep in mind that resetting the cache means the Action will "forget" what it's already seen, so it may reopen issues that were previously tracked or closed. +### How can I preview what the scanner would do without filing issues? + +Set the `dry_run` input to `true`. The scanner will run a normal scan and log the +issues it _would_ open, reopen, or close — but it won't create, close, reopen, or +assign any issues, and it won't write to the `gh-cache` branch. + +This is handy for trying out a new configuration or seeing how many issues a scan +would file, without making any changes to your repository. Because dry runs don't +update the cache, your next real run behaves exactly as if the dry run never +happened. + ### Does this work with private repositories? Yes! The Action works with both public and private repositories. Since it runs diff --git a/README.md b/README.md index 88b68c11..a20261e7 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ jobs: # skip_copilot_assignment: false # Optional: Set to true to skip assigning issues to GitHub Copilot (or if you don't have GitHub Copilot) # include_screenshots: false # Optional: Set to true to capture screenshots and include links to them in filed issues # open_grouped_issues: false # Optional: Set to true to open an issue grouping individual issues per violation + # dry_run: false # Optional: Set to true to scan and log what would be filed without creating/closing issues or writing the cache # reduced_motion: no-preference # Optional: Playwright reduced motion configuration option # color_scheme: light # Optional: Playwright color scheme configuration option # scans: '["axe","reflow-scan"]' # Optional: An array of scans (or plugins) to be performed. If not provided, only Axe will be performed. @@ -131,6 +132,7 @@ Trigger the workflow manually or automatically based on your configuration. The | `reduced_motion` | No | Playwright `reducedMotion` setting for scan contexts. Allowed values: `reduce`, `no-preference` | `reduce` | | `color_scheme` | No | Playwright `colorScheme` setting for scan contexts. Allowed values: `light`, `dark`, `no-preference` | `dark` | | `scans` | No | An array of scans (or plugins) to be performed. If not provided, only Axe will be performed. | `'["axe", "reflow-scan", ...other plugins]'` | +| `dry_run` | No | When `true`, scan and log the issues that _would_ be filed without opening, closing, reopening, or assigning any issues — and without writing to the `gh-cache` branch. Useful for safely previewing results. Default: `false` | `true` | | `url_configs` | No | A stringified JSON array of URL config objects. Each object must have a `url` field and may have an optional `excludeSelectors` field (array of CSS selectors to exclude from the Axe scan for that URL). When provided, takes precedence over the `urls` input. | `'[{"url":"https://example.com","excludeSelectors":["iframe","#widget"]}]'` | --- diff --git a/action.yml b/action.yml index b86a45e5..1b852a5c 100644 --- a/action.yml +++ b/action.yml @@ -54,6 +54,10 @@ inputs: scans: description: 'Stringified JSON array of scans to perform. If not provided, only Axe will be performed' required: false + dry_run: + description: 'When true, scan and log the issues that would be filed without opening, closing, reopening, or assigning any issues, and without writing to the cache.' + required: false + default: 'false' outputs: results: @@ -129,6 +133,7 @@ runs: cached_filings_file: ${{ steps.normalize_cache.outputs.cached_filings_file }} screenshot_repository: ${{ github.repository }} open_grouped_issues: ${{ inputs.open_grouped_issues }} + dry_run: ${{ inputs.dry_run }} - if: ${{ steps.file.outputs.filings_file }} name: Get issues from filings id: get_issues_from_filings @@ -137,7 +142,7 @@ runs: # Extract open issues from Filing objects and write to a file jq -c '[.[] | select(.issue.state == "open") | .issue]' "${{ steps.file.outputs.filings_file }}" > "$RUNNER_TEMP/issues.json" echo "issues_file=$RUNNER_TEMP/issues.json" >> "$GITHUB_OUTPUT" - - if: ${{ inputs.skip_copilot_assignment != 'true' }} + - if: ${{ inputs.skip_copilot_assignment != 'true' && inputs.dry_run != 'true' }} name: Fix id: fix uses: ./../../_actions/github/accessibility-scanner/current/.github/actions/fix @@ -185,19 +190,20 @@ runs: # Set results_file output echo "results_file=$RESULTS_FILE" >> "$GITHUB_OUTPUT" - - if: ${{ inputs.include_screenshots == 'true' }} + - if: ${{ inputs.include_screenshots == 'true' && inputs.dry_run != 'true' }} name: Save screenshots uses: ./../../_actions/github/accessibility-scanner/current/.github/actions/gh-cache/save with: path: .screenshots token: ${{ inputs.token }} - name: Copy results to cache path + if: ${{ inputs.dry_run != 'true' }} shell: bash run: | mkdir -p "$(dirname '${{ inputs.cache_key }}')" cp "$GITHUB_WORKSPACE/scanner-results.json" "${{ inputs.cache_key }}" - - name: Save cached results + if: ${{ inputs.dry_run != 'true' }} uses: ./../../_actions/github/accessibility-scanner/current/.github/actions/gh-cache/save with: path: ${{ inputs.cache_key }}