diff --git a/.github/actions/file/action.yml b/.github/actions/file/action.yml index 7d76af7a..6d229250 100644 --- a/.github/actions/file/action.yml +++ b/.github/actions/file/action.yml @@ -28,6 +28,14 @@ inputs: description: "When true, log the issues that would be filed without opening, closing, or reopening any issues." required: false default: "false" + file_best_practice_issues: + description: "File issues for best-practice findings (accessibility recommendations that are not hard WCAG failures). Disabling only suppresses new issues; existing ones are left untouched." + required: false + default: "true" + file_experimental_issues: + description: "File issues for experimental findings (checks that are not yet stable). Disabling only suppresses new issues; existing ones are left untouched." + required: false + default: "true" outputs: filings_file: diff --git a/.github/actions/file/src/generateIssueBody.ts b/.github/actions/file/src/generateIssueBody.ts index 18e25d31..f0167ffb 100644 --- a/.github/actions/file/src/generateIssueBody.ts +++ b/.github/actions/file/src/generateIssueBody.ts @@ -18,13 +18,25 @@ export function generateIssueBody(finding: Finding, screenshotRepo: string): str ` } + const categoryNotice = + finding.category && finding.category !== 'wcag' + ? `> [!NOTE]\n> This is ${ + finding.category === 'experimental' ? 'an experimental check' : 'a best-practice recommendation' + }, not a definite WCAG failure.\n\n` + : '' + + const standardsLine = + finding.category && finding.category !== 'wcag' + ? '- [ ] The fix MUST meet the accessibility standards specified by the repository or organization (WCAG 2.2 if applicable).' + : '- [ ] The fix MUST meet WCAG 2.2 guidelines OR the accessibility standards specified by the repository or organization.' + const acceptanceCriteria = `## Acceptance Criteria - [ ] The specific violation reported in this issue is no longer reproducible. -- [ ] The fix MUST meet WCAG 2.1 guidelines OR the accessibility standards specified by the repository or organization. +${standardsLine} - [ ] A test SHOULD be added to ensure this specific violation does not regress. - [ ] This PR MUST NOT introduce any new accessibility issues or regressions.` - const body = `## What + const body = `${categoryNotice}## What An accessibility scan ${finding.html ? `flagged the element \`${finding.html}\`` : `found an issue on ${finding.url}`} because ${finding.problemShort}. Learn more about why this was flagged by visiting ${finding.problemUrl}. ${screenshotSection ?? ''} diff --git a/.github/actions/file/src/index.ts b/.github/actions/file/src/index.ts index 832a701a..13a3306c 100644 --- a/.github/actions/file/src/index.ts +++ b/.github/actions/file/src/index.ts @@ -17,6 +17,12 @@ import {updateFilingsWithNewFindings} from './updateFilingsWithNewFindings.js' import {OctokitResponse} from '@octokit/types' const OctokitWithThrottling = Octokit.plugin(throttling) +// core.getBooleanInput throws on unset inputs, so apply the default first. +function getBooleanInputWithDefault(name: string, defaultValue: boolean): boolean { + if (!core.getInput(name)) return defaultValue + return core.getBooleanInput(name) +} + export default async function () { core.info("Started 'file' action") const findingsFile = core.getInput('findings_file', {required: true}) @@ -31,6 +37,8 @@ export default async function () { : [] const shouldOpenGroupedIssues = core.getBooleanInput('open_grouped_issues') const dryRun = core.getBooleanInput('dry_run') + const fileBestPracticeIssues = getBooleanInputWithDefault('file_best_practice_issues', true) + const fileExperimentalIssues = getBooleanInputWithDefault('file_experimental_issues', true) core.debug(`Input: 'findings_file: ${findingsFile}'`) core.debug(`Input: 'repository: ${repoWithOwner}'`) core.debug(`Input: 'base_url: ${baseUrl ?? '(default)'}'`) @@ -38,6 +46,8 @@ export default async function () { core.debug(`Input: 'cached_filings_file: ${cachedFilingsFile}'`) core.debug(`Input: 'open_grouped_issues: ${shouldOpenGroupedIssues}'`) core.debug(`Input: 'dry_run: ${dryRun}'`) + core.debug(`Input: 'file_best_practice_issues: ${fileBestPracticeIssues}'`) + core.debug(`Input: 'file_experimental_issues: ${fileExperimentalIssues}'`) const octokit = new OctokitWithThrottling({ auth: token, @@ -61,6 +71,9 @@ export default async function () { }) const filings = updateFilingsWithNewFindings(cachedFilings, findings) + // Suppressed new filings are kept out of the cache + const suppressedFilings = new Set() + // Fetch closed wontfix issues once up front; a failed fetch reopens as usual let wontfixIssueNumbers = new Set() if (!dryRun) { @@ -80,6 +93,21 @@ export default async function () { for (const filing of filings) { let response: OctokitResponse | undefined try { + // Category switches gate only new issues + if (isNewFiling(filing)) { + const category = filing.findings[0].category ?? 'wcag' + if ( + (category === 'best-practice' && !fileBestPracticeIssues) || + (category === 'experimental' && !fileExperimentalIssues) + ) { + core.info( + `Skipping new ${category} issue (filing disabled for this category): ${filing.findings[0].problemShort}`, + ) + suppressedFilings.add(filing) + continue + } + } + if (dryRun) { if (isResolvedFiling(filing)) { dryRunCounts.close++ @@ -182,7 +210,8 @@ export default async function () { } const filingsPath = path.join(process.env.RUNNER_TEMP || '/tmp', `filings-${crypto.randomUUID()}.json`) - fs.writeFileSync(filingsPath, JSON.stringify(filings)) + const outputFilings = suppressedFilings.size > 0 ? filings.filter(f => !suppressedFilings.has(f)) : filings + fs.writeFileSync(filingsPath, JSON.stringify(outputFilings)) core.setOutput('filings_file', filingsPath) core.debug(`Output: 'filings_file: ${filingsPath}'`) diff --git a/.github/actions/file/src/openIssue.ts b/.github/actions/file/src/openIssue.ts index 937f06cf..03161c2c 100644 --- a/.github/actions/file/src/openIssue.ts +++ b/.github/actions/file/src/openIssue.ts @@ -26,6 +26,10 @@ export async function openIssue(octokit: Octokit, repoWithOwner: string, finding if (finding.ruleId) { labels.push(`${finding.scannerType} rule: ${finding.ruleId}`) } + // Flag non-WCAG findings so they can be filtered or triaged separately + if (finding.category && finding.category !== 'wcag') { + labels.push(finding.category) + } const title = truncateWithEllipsis( `Accessibility issue: ${finding.problemShort[0].toUpperCase() + finding.problemShort.slice(1)} on ${new URL(finding.url).pathname}`, diff --git a/.github/actions/file/src/types.d.ts b/.github/actions/file/src/types.d.ts index ee91bc67..ba36ecc3 100644 --- a/.github/actions/file/src/types.d.ts +++ b/.github/actions/file/src/types.d.ts @@ -1,5 +1,8 @@ +export type FindingCategory = 'wcag' | 'best-practice' | 'experimental' + export type Finding = { scannerType: string + category?: FindingCategory ruleId?: string url: string html?: string diff --git a/.github/actions/file/tests/generateIssueBody.test.ts b/.github/actions/file/tests/generateIssueBody.test.ts index 167ee5f8..7fbabe5d 100644 --- a/.github/actions/file/tests/generateIssueBody.test.ts +++ b/.github/actions/file/tests/generateIssueBody.test.ts @@ -26,6 +26,7 @@ describe('generateIssueBody', () => { expect(body).toContain('## What') expect(body).toContain('## Acceptance Criteria') expect(body).toContain('The specific violation reported in this issue is no longer reproducible.') + expect(body).toContain('The fix MUST meet WCAG 2.2 guidelines OR') expect(body).not.toContain('Specifically:') }) @@ -76,4 +77,31 @@ describe('generateIssueBody', () => { expect(body).toContain(`found an issue on ${findingWithEmptyOptionalFields.url}`) expect(body).not.toContain('flagged the element') }) + + it('omits the category notice for WCAG findings', () => { + expect(generateIssueBody(baseFinding, 'github/accessibility-scanner')).not.toContain('> [!NOTE]') + expect(generateIssueBody({...baseFinding, category: 'wcag'}, 'github/accessibility-scanner')).not.toContain( + '> [!NOTE]', + ) + }) + + it('includes a best-practice notice for best-practice findings', () => { + const body = generateIssueBody({...baseFinding, category: 'best-practice'}, 'github/accessibility-scanner') + + expect(body).toContain('> [!NOTE]') + expect(body).toContain('best-practice recommendation') + expect(body).toContain('not a definite WCAG failure') + expect(body).toContain('WCAG 2.2 if applicable') + expect(body).not.toContain('The fix MUST meet WCAG 2.2 guidelines OR') + }) + + it('includes an experimental notice for experimental findings', () => { + const body = generateIssueBody({...baseFinding, category: 'experimental'}, 'github/accessibility-scanner') + + expect(body).toContain('> [!NOTE]') + expect(body).toContain('an experimental check') + expect(body).toContain('not a definite WCAG failure') + expect(body).toContain('WCAG 2.2 if applicable') + expect(body).not.toContain('The fix MUST meet WCAG 2.2 guidelines OR') + }) }) diff --git a/.github/actions/file/tests/openIssue.test.ts b/.github/actions/file/tests/openIssue.test.ts index 77a184c3..e9cb46ff 100644 --- a/.github/actions/file/tests/openIssue.test.ts +++ b/.github/actions/file/tests/openIssue.test.ts @@ -65,6 +65,28 @@ describe('openIssue', () => { ) }) + it('adds a category label for non-WCAG findings', async () => { + const octokit = mockOctokit() + await openIssue(octokit, 'org/repo', {...baseFinding, category: 'best-practice'}) + + expect(octokit.request).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + labels: ['axe-scanning-issue', 'axe rule: color-contrast', 'best-practice'], + }), + ) + }) + + it('does not add a category label for WCAG findings', async () => { + const octokit = mockOctokit() + await openIssue(octokit, 'org/repo', {...baseFinding, category: 'wcag'}) + + const labels = octokit.request.mock.calls[0][1].labels + expect(labels).not.toContain('wcag') + expect(labels).not.toContain('best-practice') + expect(labels).not.toContain('experimental') + }) + it('truncates long titles with ellipsis', async () => { const octokit = mockOctokit() const longFinding = { diff --git a/.github/actions/find/src/findForUrl.ts b/.github/actions/find/src/findForUrl.ts index d9f1ea87..ea4f1e18 100644 --- a/.github/actions/find/src/findForUrl.ts +++ b/.github/actions/find/src/findForUrl.ts @@ -1,4 +1,4 @@ -import type {ColorSchemePreference, Finding, ReducedMotionPreference, UrlConfig} from './types.d.js' +import type {ColorSchemePreference, Finding, FindingCategory, ReducedMotionPreference, UrlConfig} from './types.d.js' import {AxeBuilder} from '@axe-core/playwright' import playwright from 'playwright' import {AuthContext} from './AuthContext.js' @@ -87,6 +87,7 @@ async function runAxeScan({ for (const violation of rawFindings.violations) { await addFinding({ scannerType: 'axe', + category: categorizeAxeViolation(violation.tags), url, html: violation.nodes[0].html.replace(/'/g, '''), problemShort: violation.help.toLowerCase().replace(/'/g, '''), @@ -98,3 +99,11 @@ async function runAxeScan({ } } } + +// Maps an Axe violation's tags to a conformance tier. Experimental is checked +// first because some experimental rules also carry a wcag* tag. +function categorizeAxeViolation(tags: string[]): FindingCategory { + if (tags.includes('experimental')) return 'experimental' + if (tags.includes('best-practice')) return 'best-practice' + return 'wcag' +} diff --git a/.github/actions/find/src/types.d.ts b/.github/actions/find/src/types.d.ts index dcbc8600..a2f4a534 100644 --- a/.github/actions/find/src/types.d.ts +++ b/.github/actions/find/src/types.d.ts @@ -1,5 +1,8 @@ +export type FindingCategory = 'wcag' | 'best-practice' | 'experimental' + export type Finding = { scannerType: string + category?: FindingCategory url: string html?: string problemShort: string diff --git a/.github/actions/find/tests/findForUrl.test.ts b/.github/actions/find/tests/findForUrl.test.ts index 85299c5c..54244885 100644 --- a/.github/actions/find/tests/findForUrl.test.ts +++ b/.github/actions/find/tests/findForUrl.test.ts @@ -117,4 +117,40 @@ describe('findForUrl', () => { expect(loadedPlugins[1].default).toHaveBeenCalledTimes(0) }) }) + + describe('axe finding categorization', () => { + function axeViolation(tags: string[]) { + return { + id: 'some-rule', + help: 'Help', + helpUrl: 'https://example.com', + description: 'Description', + tags, + nodes: [{html: '
', failureSummary: 'summary'}], + } + } + + async function categoryFor(tags: string[]) { + clearAll() + actionInput = JSON.stringify(['axe']) + vi.mocked(AxeBuilder.prototype.analyze).mockResolvedValueOnce({ + violations: [axeViolation(tags)], + } as unknown as axe.AxeResults) + + const findings = await findForUrl('test.com') + return findings[0].category + } + + it('categorizes a violation with only wcag tags as wcag', async () => { + expect(await categoryFor(['wcag2a', 'wcag111'])).toBe('wcag') + }) + + it('categorizes a violation with a best-practice tag as best-practice', async () => { + expect(await categoryFor(['cat.semantics', 'best-practice'])).toBe('best-practice') + }) + + it('categorizes a violation with an experimental tag as experimental, even alongside wcag tags', async () => { + expect(await categoryFor(['wcag2a', 'experimental'])).toBe('experimental') + }) + }) }) diff --git a/README.md b/README.md index 28145177..5c57745d 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ jobs: # skip_copilot_assignment: false # Optional: Set to true to skip assigning issues to GitHub Copilot (or if you don't have GitHub Copilot) # include_screenshots: false # Optional: Set to true to capture screenshots and include links to them in filed issues # open_grouped_issues: false # Optional: Set to true to open an issue grouping individual issues per violation + # file_best_practice_issues: true # Optional: Set to false to stop filing new issues for best-practice findings (recommendations that are not hard WCAG failures) + # file_experimental_issues: true # Optional: Set to false to stop filing new issues for experimental findings (checks that are not yet stable) # dry_run: false # Optional: Set to true to scan and log what would be filed without creating/closing issues or writing the cache # reduced_motion: no-preference # Optional: Playwright reduced motion configuration option # color_scheme: light # Optional: Playwright color scheme configuration option @@ -115,25 +117,27 @@ Trigger the workflow manually or automatically based on your configuration. The ## Action inputs -| Input | Required | Description | Example | -| ------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | -| `urls` | No\* | Newline-delimited list of URLs to scan. Required unless `url_configs` is provided. | `https://primer.style`
`https://primer.style/octicons` | -| `repository` | Yes | Repository (with owner) for issues and PRs | `primer/primer-docs` | -| `token` | Yes | PAT with write permissions (see above) | `${{ secrets.GH_TOKEN }}` | -| `cache_key` | Yes | Key for caching results across runs
Allowed: `A-Za-z0-9._/-` | `cached_results-primer.style-main.json` | -| `base_url` | No | GitHub API base URL used by Octokit. Set this for GitHub Enterprise Server (format: `https://HOSTNAME/api/v3`). Defaults to `https://api.github.com` | `https://ghe.example.com/api/v3` | -| `login_url` | No | If scanned pages require authentication, the URL of the login page | `https://github.com/login` | -| `username` | No | If scanned pages require authentication, the username to use for login | `some-user` | -| `password` | No | If scanned pages require authentication, the password to use for login | `${{ secrets.PASSWORD }}` | -| `auth_context` | No | If scanned pages require authentication, a stringified JSON object containing username, password, cookies, and/or localStorage from an authenticated session | `{"username":"some-user","password":"***","cookies":[...]}` | -| `skip_copilot_assignment` | No | Whether to skip assigning filed issues to GitHub Copilot. Set to `true` if you don't have GitHub Copilot or prefer to handle issues manually | `true` | -| `include_screenshots` | No | Whether to capture screenshots of scanned pages and include links to them in filed issues. Screenshots are stored on the `gh-cache` branch of the repository running the workflow. Default: `false` | `true` | -| `open_grouped_issues` | No | Whether to create a tracking issue which groups filed issues together by violation type. Default: `false` | `true` | -| `reduced_motion` | No | Playwright `reducedMotion` setting for scan contexts. Allowed values: `reduce`, `no-preference` | `reduce` | -| `color_scheme` | No | Playwright `colorScheme` setting for scan contexts. Allowed values: `light`, `dark`, `no-preference` | `dark` | -| `scans` | No | An array of scans (or plugins) to be performed. If not provided, only Axe will be performed. | `'["axe", "reflow-scan", ...other plugins]'` | -| `dry_run` | No | When `true`, scan and log the issues that _would_ be filed without opening, closing, reopening, or assigning any issues — and without writing to the `gh-cache` branch. Useful for safely previewing results. Default: `false` | `true` | -| `url_configs` | No | A stringified JSON array of URL config objects. Each object must have a `url` field and may have an optional `excludeSelectors` field (array of CSS selectors to exclude from the Axe scan for that URL). When provided, takes precedence over the `urls` input. | `'[{"url":"https://example.com","excludeSelectors":["iframe","#widget"]}]'` | +| Input | Required | Description | Example | +| --------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| `urls` | No | Newline-delimited list of URLs to scan. Required unless `url_configs` is provided. | `https://primer.style`
`https://primer.style/octicons` | +| `repository` | Yes | Repository (with owner) for issues and PRs | `primer/primer-docs` | +| `token` | Yes | PAT with write permissions (see above) | `${{ secrets.GH_TOKEN }}` | +| `cache_key` | Yes | Key for caching results across runs
Allowed: `A-Za-z0-9._/-` | `cached_results-primer.style-main.json` | +| `base_url` | No | GitHub API base URL used by Octokit. Set this for GitHub Enterprise Server (format: `https://HOSTNAME/api/v3`). Defaults to `https://api.github.com` | `https://ghe.example.com/api/v3` | +| `login_url` | No | If scanned pages require authentication, the URL of the login page | `https://github.com/login` | +| `username` | No | If scanned pages require authentication, the username to use for login | `some-user` | +| `password` | No | If scanned pages require authentication, the password to use for login | `${{ secrets.PASSWORD }}` | +| `auth_context` | No | If scanned pages require authentication, a stringified JSON object containing username, password, cookies, and/or localStorage from an authenticated session | `{"username":"some-user","password":"***","cookies":[...]}` | +| `skip_copilot_assignment` | No | Whether to skip assigning filed issues to GitHub Copilot. Set to `true` if you don't have GitHub Copilot or prefer to handle issues manually | `true` | +| `include_screenshots` | No | Whether to capture screenshots of scanned pages and include links to them in filed issues. Screenshots are stored on the `gh-cache` branch of the repository running the workflow. Default: `false` | `true` | +| `open_grouped_issues` | No | Whether to create a tracking issue which groups filed issues together by violation type. Default: `false` | `true` | +| `file_best_practice_issues` | No | Whether to file issues for best-practice findings (accessibility recommendations that are not hard WCAG failures). Set to `false` to suppress new best-practice issues; existing ones are left untouched. Default: `true` | `false` | +| `file_experimental_issues` | No | Whether to file issues for experimental findings (checks that are not yet stable). Set to `false` to suppress new experimental issues; existing ones are left untouched. Default: `true` | `false` | +| `reduced_motion` | No | Playwright `reducedMotion` setting for scan contexts. Allowed values: `reduce`, `no-preference` | `reduce` | +| `color_scheme` | No | Playwright `colorScheme` setting for scan contexts. Allowed values: `light`, `dark`, `no-preference` | `dark` | +| `scans` | No | An array of scans (or plugins) to be performed. If not provided, only Axe will be performed. | `'["axe", "reflow-scan", ...other plugins]'` | +| `dry_run` | No | When `true`, scan and log the issues that _would_ be filed without opening, closing, reopening, or assigning any issues — and without writing to the `gh-cache` branch. Useful for safely previewing results. Default: `false` | `true` | +| `url_configs` | No | A stringified JSON array of URL config objects. Each object must have a `url` field and may have an optional `excludeSelectors` field (array of CSS selectors to exclude from the Axe scan for that URL). When provided, takes precedence over the `urls` input. | `'[{"url":"https://example.com","excludeSelectors":["iframe","#widget"]}]'` | --- diff --git a/action.yml b/action.yml index 1b852a5c..9c3052b0 100644 --- a/action.yml +++ b/action.yml @@ -45,6 +45,14 @@ inputs: description: "In the 'file' step, also open grouped issues which link to all issues with the same problem" required: false default: 'false' + file_best_practice_issues: + description: 'File issues for best-practice findings (accessibility recommendations that are not hard WCAG failures). Disabling suppresses new issues while existing ones are left untouched.' + required: false + default: 'true' + file_experimental_issues: + description: 'File issues for experimental findings (checks that are not yet stable). Disabling suppresses new issues while existing ones are left untouched.' + required: false + default: 'true' reduced_motion: description: 'Playwright reducedMotion setting: https://playwright.dev/docs/api/class-browser#browser-new-page-option-reduced-motion' required: false @@ -134,6 +142,8 @@ runs: screenshot_repository: ${{ github.repository }} open_grouped_issues: ${{ inputs.open_grouped_issues }} dry_run: ${{ inputs.dry_run }} + file_best_practice_issues: ${{ inputs.file_best_practice_issues }} + file_experimental_issues: ${{ inputs.file_experimental_issues }} - if: ${{ steps.file.outputs.filings_file }} name: Get issues from filings id: get_issues_from_filings diff --git a/tests/site-with-errors.test.ts b/tests/site-with-errors.test.ts index 8c7981bd..cf216880 100644 --- a/tests/site-with-errors.test.ts +++ b/tests/site-with-errors.test.ts @@ -58,6 +58,7 @@ describe('site-with-errors', () => { const expected = [ { scannerType: 'axe', + category: 'wcag', url: 'http://127.0.0.1:4000/', html: 'Jul 30, 2025', problemShort: 'elements must meet minimum color contrast ratio thresholds', @@ -67,6 +68,7 @@ describe('site-with-errors', () => { }, { scannerType: 'axe', + category: 'best-practice', url: 'http://127.0.0.1:4000/', html: '', problemShort: 'page should contain a level-one heading', @@ -75,6 +77,7 @@ describe('site-with-errors', () => { }, { scannerType: 'axe', + category: 'wcag', url: 'http://127.0.0.1:4000/jekyll/update/2025/07/30/welcome-to-jekyll.html', html: ``, @@ -85,6 +88,7 @@ describe('site-with-errors', () => { }, { scannerType: 'axe', + category: 'wcag', url: 'http://127.0.0.1:4000/about/', html: 'jekyllrb.com', problemShort: 'elements must meet minimum color contrast ratio thresholds', @@ -94,6 +98,7 @@ describe('site-with-errors', () => { }, { scannerType: 'axe', + category: 'wcag', url: 'http://127.0.0.1:4000/404.html', html: '
  • Accessibility Scanner Demo
    • ', problemShort: 'elements must meet minimum color contrast ratio thresholds', @@ -103,6 +108,7 @@ describe('site-with-errors', () => { }, { scannerType: 'axe', + category: 'best-practice', url: 'http://127.0.0.1:4000/404.html', html: '

    ', problemShort: 'headings should not be empty',