diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index e34997998..3cbdcaa35 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -16,6 +16,9 @@ on: permissions: contents: write pull-requests: write + # Read this run's jobs so the Slack alert can deep-link the failing job log + # rather than the run summary. + actions: read concurrency: group: generate-toolkit-docs @@ -28,6 +31,11 @@ jobs: # switch on 2026-06-02. Harmless today; unblocks the cutover. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + outputs: + # Distinguishes "generation itself failed" from "a later step failed". + # Without it, a broken notification would make the alert job announce a + # generation failure that never happened. + generation-succeeded: ${{ steps.generate-docs.outputs.succeeded }} steps: - name: Checkout repository @@ -48,6 +56,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Generate toolkit docs + id: generate-docs # Invoked by path rather than through `pnpm exec`, which would reset the # working directory to the repo root and break the relative paths below. run: | @@ -72,6 +81,8 @@ jobs: --ignore-file ./skip-toolkits.txt \ --custom-sections ./curation \ --output data/toolkits + + echo "succeeded=true" >>"$GITHUB_OUTPUT" working-directory: toolkit-docs-generator env: ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }} @@ -108,42 +119,81 @@ jobs: branch: automation/toolkit-docs delete-branch: true + # A token without reviewer permission is a real gap, but it must not fail + # the run: the generated PR is already open and useful without a reviewer + # attached. An annotation keeps it visible instead of silent. - name: Request team review if: steps.cpr.outputs.pull-request-number != '' - continue-on-error: true - run: gh pr edit ${{ steps.cpr.outputs.pull-request-number }} --add-reviewer ArcadeAI/engineering-tools-and-dx + run: | + if ! gh pr edit ${{ steps.cpr.outputs.pull-request-number }} \ + --add-reviewer ArcadeAI/engineering-tools-and-dx 2>review-error.log; then + echo "::warning::Could not request review on PR #${{ steps.cpr.outputs.pull-request-number }}: $(cat review-error.log)" + fi env: GH_TOKEN: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }} - - name: Warn #proj-docs about preserved or omitted toolkit docs - continue-on-error: true + - name: Upload generation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: failed-tools + path: toolkit-docs-generator-verification/logs/failed-tools.json + if-no-files-found: ignore + + # No `continue-on-error` here on purpose. A Slack step that dies quietly + # is how a wrong webhook secret went unnoticed: the alert it was meant to + # deliver is exactly what nobody was watching for. + # + # Gated on the generate step rather than on the default "everything so far + # succeeded". Missing or stale pages are worth reporting even when a later + # step broke — under the default gate a failed sidebar sync or PR creation + # would skip this step, while `generation-succeeded` suppressed the alert + # job below, and nobody heard about the toolkits at all. `!cancelled()` + # rather than `always()` because `cancel-in-progress` cancels superseded + # runs, and those have nothing to report. + - name: Report preserved or omitted toolkits to Slack + if: ${{ !cancelled() && steps.generate-docs.outputs.succeeded == 'true' }} run: | - report=toolkit-docs-generator-verification/logs/failed-tools.json - if [ ! -f "$report" ]; then + # The deep link is a nicety; the alert is the point. Actions runs this + # with `-e -o pipefail`, so a failed `gh api` — or a SIGPIPE from + # trimming its output — would abort the step before curl and lose the + # alert entirely. Select the first match in jq instead of piping to + # head, and fall back to the run URL on any failure. + job_url=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \ + --jq '[.jobs[] | select(.name == "generate") | .html_url] | .[0] // empty') || job_url="" + + payload=$(../node_modules/.bin/tsx src/cli/index.ts alert \ + --report ../toolkit-docs-generator-verification/logs/failed-tools.json \ + --log-url "${job_url:-$RUN_URL}") + + if [ -z "$payload" ]; then + echo "No preserved or omitted toolkits to report." exit 0 fi - preserved=$(jq -r '(.preservedToolkits // []) | join(", ")' "$report") - omitted=$(jq -r '(.omittedToolkits // []) | join(", ")' "$report") - if [ -z "$preserved" ] && [ -z "$omitted" ]; then - exit 0 + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "::error::SLACK_PROJ_DOCS_WEBHOOK_URL is not configured" + exit 1 fi - payload=$(jq -n \ - --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ - --arg preserved "$preserved" \ - --arg omitted "$omitted" \ - '{text: (":warning: Toolkit docs generation completed with recoverable failures\n\n*Workflow run:* <" + $run_url + "|Open run>" + (if $preserved != "" then "\n*Continuing to serve previous docs:* " + $preserved else "" end) + (if $omitted != "" then "\n*No docs are being served (no prior output):* " + $omitted else "" end))}') - curl --fail-with-body --silent --show-error \ -X POST \ -H "Content-Type: application/json" \ --data "$payload" \ - "${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }}" + "$SLACK_WEBHOOK_URL" + working-directory: toolkit-docs-generator + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }} alert: - name: Alert on generation failure - if: ${{ always() && needs.generate.result == 'failure' }} + name: Alert on failure + # Every red run gets a message. Which message depends on whether generation + # itself broke or something after it did: those need different people to do + # different things, and calling a post-generation failure a "generation + # failure" sends someone hunting for a validation error that doesn't exist. + if: ${{ !cancelled() && needs.generate.result == 'failure' }} needs: generate runs-on: ubuntu-latest permissions: {} @@ -154,17 +204,28 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} TRIGGER: ${{ github.event_name }} COMMIT: ${{ github.sha }} + GENERATION_SUCCEEDED: ${{ needs.generate.outputs.generation-succeeded }} run: | if [ -z "$SLACK_WEBHOOK_URL" ]; then echo "::error::SLACK_PROJ_DOCS_WEBHOOK_URL is not configured" exit 1 fi + if [ "$GENERATION_SUCCEEDED" = "true" ]; then + headline=":warning: Toolkit docs generated, but the workflow failed afterward" + detail="Generation succeeded, so the toolkit JSON is fine. A later workflow step failed — inspect the run to determine whether sidebar sync, PR creation, artifact upload, or Slack notification needs attention." + else + headline=":rotating_light: Toolkit docs generation failed" + detail="The failed run contains the exact file path and validation error." + fi + payload=$(jq -n \ + --arg headline "$headline" \ + --arg detail "$detail" \ --arg run_url "$RUN_URL" \ --arg trigger "$TRIGGER" \ --arg commit "$COMMIT" \ - '{text: (":rotating_light: Toolkit docs generation failed\n\n*Workflow run:* <" + $run_url + "|Open failed run>\n*Trigger:* " + $trigger + "\n*Commit:* " + $commit + "\n\nThe failed run contains the exact file path and validation error.")}') + '{text: ($headline + "\n\n*Workflow run:* <" + $run_url + "|Open failed run>\n*Trigger:* " + $trigger + "\n*Commit:* " + $commit + "\n\n" + $detail)}') curl --fail-with-body --silent --show-error \ -X POST \ diff --git a/toolkit-docs-generator/CURATION.md b/toolkit-docs-generator/CURATION.md index 3fa6b0319..3b14d09f6 100644 --- a/toolkit-docs-generator/CURATION.md +++ b/toolkit-docs-generator/CURATION.md @@ -241,9 +241,8 @@ toolkit with its message, and exits non-zero on failure. It runs the same compiler generation uses, so a pass means kinds, frontmatter, and MDX are all valid. It cannot check `tool:` targets. -Invoke `tsx` by path, not through `pnpm exec`, which resets the working -directory to the repo root and breaks the relative path. `pnpm dlx tsx` also -works. +Invoke `tsx` by path because `toolkit-docs-generator` has no package for +`pnpm exec`. Seeing a chunk on a page needs generated JSON. Either wait for the automated generation PR, or generate that one toolkit with Engine credentials and run diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index b6ddf7593..bc971cb0f 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -129,7 +129,9 @@ pnpm dlx tsx src/cli/index.ts generate \ ## Local usage -Run these commands from the `toolkit-docs-generator` directory. +Run these commands from the `toolkit-docs-generator` directory. Invoke `tsx` by +path because this directory has no package for `pnpm exec`; the sidebar sync +command below runs from the repo root instead. Generate a single toolkit: @@ -263,6 +265,36 @@ pnpm dlx tsx src/cli/index.ts validate-curation --toolkit GoogleFlights That compiles the directory with the same code generation uses and needs no credentials. Omit `--toolkit` to check everything. +## Run alerts + +The nightly workflow posts to Slack when a toolkit fails and the run recovers +from it. Two outcomes, and the difference matters: + +- **Missing entirely**: the toolkit failed and had no previous artifact to fall + back on, so it has no page on the docs site at all. +- **Still serving the previous docs**: the toolkit failed but still publishes + its last good output, so readers see yesterday's page. + +Each entry names the reason and the fix. Every run attaches the full report as +its `failed-tools` artifact. Build the message locally against any report with +`pnpm dlx tsx src/cli/index.ts alert --report `. It prints nothing when a +run has nothing to report. + +No red run stays quiet. Three messages can arrive, and they answer different +questions: + +| Message | Means | Who acts | +| --- | --- | --- | +| Toolkits missing or stale | Generation recovered from a per-toolkit failure | Follow the fix named in the entry | +| Toolkit docs generation failed | The generate step itself broke | Open the run for the path and validation error | +| Generated, but workflow failed afterward | Generation was fine, a later step broke | Open the run to determine whether sidebar sync, PR creation, artifact upload, or Slack notification needs attention | + +The first is reported from inside the generate job, gated on the generate step +rather than on the steps after it, so a broken sidebar sync or PR creation does +not hide missing pages. The other two come from the alert job. A run that both +recovers from a toolkit failure and then fails to publish sends the first +message and the third, because those are two separate things to fix. + ## Troubleshooting - **Nothing regenerated**: `--skip-unchanged` exits early when tool definitions did not change. diff --git a/toolkit-docs-generator/src/alerts/docs-alert.ts b/toolkit-docs-generator/src/alerts/docs-alert.ts new file mode 100644 index 000000000..cb6a7b1fc --- /dev/null +++ b/toolkit-docs-generator/src/alerts/docs-alert.ts @@ -0,0 +1,96 @@ +/** + * Slack message for a generation run that recovered from toolkit failures. + * + * The text is built here rather than in workflow YAML so that it is testable, + * and so it can carry the two things the report already knows but a bare list + * of toolkit names cannot say: why the toolkit failed, and what to do about it. + * + * `omitted` outranks `preserved` throughout. A preserved toolkit still serves + * yesterday's page and nobody notices; an omitted toolkit has no page at all. + */ +import type { + FailedToolsReport, + RecoveredToolkitEntry, +} from "../utils/run-logs"; + +export interface SlackMessage { + readonly text: string; +} + +export interface DocsAlertLinks { + /** Deep link to the job log, so the reader lands on the error, not a summary. */ + readonly logUrl?: string; +} + +const MISSING_METADATA_REASON = "missing design-system metadata"; +const CURATION_REASON_PREFIX = "Curation"; + +const suggestFix = (entry: RecoveredToolkitEntry): string => { + if (entry.reason.includes(MISSING_METADATA_REASON)) { + return `Add a \`${entry.id}\` entry to \`@arcadeai/design-system\` and bump the pin in this repo, or add it to \`skip-toolkits.txt\` if it is not meant to be public yet.`; + } + if (entry.reason.startsWith(CURATION_REASON_PREFIX)) { + return `Fix the curation source under \`toolkit-docs-generator/curation/${entry.id.toLowerCase()}/\`.`; + } + return "Open the failing step for the full error."; +}; + +const formatEntry = (entry: RecoveredToolkitEntry): string => + [`• *${entry.id}* — ${entry.reason}`, ` Fix: ${suggestFix(entry)}`].join( + "\n" + ); + +const formatSection = ( + heading: string, + entries: readonly RecoveredToolkitEntry[] +): string[] => + entries.length === 0 + ? [] + : [`*${heading}*`, entries.map(formatEntry).join("\n")]; + +const formatHeadline = ( + omitted: readonly RecoveredToolkitEntry[], + preserved: readonly RecoveredToolkitEntry[] +): string => { + if (omitted.length === 1) { + return `:no_entry: ${omitted[0]?.id} is missing from the docs site`; + } + if (omitted.length > 1) { + return `:no_entry: ${omitted.length} toolkits are missing from the docs site`; + } + if (preserved.length === 1) { + return `:warning: ${preserved[0]?.id} kept its previous docs`; + } + return `:warning: ${preserved.length} toolkits kept their previous docs`; +}; + +/** + * Build the Slack payload for a finished run, or null when the run had nothing + * worth interrupting anyone about. + */ +export const buildDocsAlert = ( + report: Pick, + links: DocsAlertLinks = {} +): SlackMessage | null => { + const recovered = report.recoveredToolkits ?? []; + if (recovered.length === 0) { + return null; + } + + const omitted = recovered.filter((entry) => entry.recovery === "omitted"); + const preserved = recovered.filter((entry) => entry.recovery === "preserved"); + + const blocks = [ + formatHeadline(omitted, preserved), + ...formatSection("Missing entirely (no previous output)", omitted), + ...formatSection("Still serving the previous docs", preserved), + ]; + + if (links.logUrl) { + blocks.push( + `<${links.logUrl}|Open the failing step> · full detail in the \`failed-tools\` artifact on the run` + ); + } + + return { text: blocks.join("\n\n") }; +}; diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 0070e42f1..97269e35d 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -13,9 +13,11 @@ import chalk from "chalk"; import { Command } from "commander"; +import { existsSync } from "fs"; import { readdir, readFile } from "fs/promises"; import ora from "ora"; import { join, resolve } from "path"; +import { buildDocsAlert } from "../alerts/docs-alert"; import { detectChanges, formatChangeSummary, @@ -1830,12 +1832,21 @@ program 0 ); const failedToolkits = mergeFailures.map((result) => result.toolkit.id); - const preservedToolkits = mergeFailures - .filter((result) => result.recovery === "preserved") - .map((result) => result.toolkit.id); - const omittedToolkits = mergeFailures - .filter((result) => result.recovery === "omitted") - .map((result) => result.toolkit.id); + // The reason travels with the toolkit id: it is what turns "MicrosoftUsers + // has no page" into something a reader can act on without opening the log. + const recoveredToolkits = mergeFailures + .filter((result) => result.recovery !== undefined) + .map((result) => ({ + id: result.toolkit.id, + recovery: result.recovery as "preserved" | "omitted", + reason: result.error ?? "unknown error", + })); + const preservedToolkits = recoveredToolkits + .filter((entry) => entry.recovery === "preserved") + .map((entry) => entry.id); + const omittedToolkits = recoveredToolkits + .filter((entry) => entry.recovery === "omitted") + .map((entry) => entry.id); const failedTools = allResults.flatMap((result) => result.failedTools); const failedToolkitsFromTools = Array.from( new Set(failedTools.map((tool) => tool.toolkitId)) @@ -1905,8 +1916,7 @@ program generatedAt: new Date().toISOString(), toolkits: failedToolkitsFromTools, failedToolkits, - preservedToolkits, - omittedToolkits, + recoveredToolkits, tools: failedTools, }); @@ -3120,5 +3130,36 @@ program } ); +program + .command("alert") + .description( + "Print the Slack payload for a finished run, or nothing when the run had no recoverable failures" + ) + .option( + "--report ", + "Failed tools report to summarize", + buildLogPaths(getDefaultLogDir()).failedToolsPath + ) + .option( + "--log-url ", + "Link the message should point at, ideally the job log rather than the run summary" + ) + .action(async (options: { report: string; logUrl?: string }) => { + // Silence rather than failure when there is no report: a clean run never + // writes one, and CI should not need to know the difference. + if (!existsSync(options.report)) { + return; + } + + const report = await readFailedToolsReport(options.report); + const message = buildDocsAlert( + report, + options.logUrl ? { logUrl: options.logUrl } : {} + ); + if (message) { + console.log(JSON.stringify(message)); + } + }); + // Parse command line arguments program.parse(); diff --git a/toolkit-docs-generator/src/utils/run-logs.ts b/toolkit-docs-generator/src/utils/run-logs.ts index ecf4045d5..156eda297 100644 --- a/toolkit-docs-generator/src/utils/run-logs.ts +++ b/toolkit-docs-generator/src/utils/run-logs.ts @@ -22,12 +22,24 @@ export interface FailedToolEntry { readonly reason: string; } +/** + * A toolkit whose merge failed and was recovered rather than aborting the run. + * + * `preserved` keeps serving the previous artifact; `omitted` means there was no + * previous artifact, so the toolkit has no page at all. The reason travels with + * the entry because it is the only thing that tells a reader what to fix. + */ +export interface RecoveredToolkitEntry { + readonly id: string; + readonly recovery: "preserved" | "omitted"; + readonly reason: string; +} + export interface FailedToolsReport { readonly generatedAt: string; readonly toolkits: readonly string[]; readonly failedToolkits: readonly string[]; - readonly preservedToolkits?: readonly string[]; - readonly omittedToolkits?: readonly string[]; + readonly recoveredToolkits?: readonly RecoveredToolkitEntry[]; readonly tools: readonly FailedToolEntry[]; } diff --git a/toolkit-docs-generator/tests/alerts/docs-alert.test.ts b/toolkit-docs-generator/tests/alerts/docs-alert.test.ts new file mode 100644 index 000000000..55296fa54 --- /dev/null +++ b/toolkit-docs-generator/tests/alerts/docs-alert.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; + +import { buildDocsAlert } from "../../src/alerts/docs-alert"; + +const omittedMicrosoftUsers = { + id: "MicrosoftUsers", + recovery: "omitted" as const, + reason: "missing design-system metadata", +}; + +const preservedGithub = { + id: "Github", + recovery: "preserved" as const, + reason: "Curation for Github targets unknown tool(s): CreateIsue", +}; + +describe("buildDocsAlert", () => { + it("stays silent when nothing was preserved or omitted", () => { + expect(buildDocsAlert({ recoveredToolkits: [] })).toBeNull(); + expect(buildDocsAlert({})).toBeNull(); + }); + + it("names the toolkit in the headline when only one is missing", () => { + const message = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(message?.text).toContain( + ":no_entry: MicrosoftUsers is missing from the docs site" + ); + }); + + it("reports the reason, not just the toolkit name", () => { + const message = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(message?.text).toContain("missing design-system metadata"); + }); + + it("tells the reader how to fix missing design-system metadata", () => { + const message = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(message?.text).toContain("`MicrosoftUsers` entry to"); + expect(message?.text).toContain("@arcadeai/design-system"); + expect(message?.text).toContain("skip-toolkits.txt"); + }); + + it("points curation failures at the curation directory", () => { + const message = buildDocsAlert({ recoveredToolkits: [preservedGithub] }); + + expect(message?.text).toContain("curation/github/"); + }); + + it("falls back to the log for an unrecognized reason", () => { + const message = buildDocsAlert({ + recoveredToolkits: [ + { id: "Slack", recovery: "preserved", reason: "socket hang up" }, + ], + }); + + expect(message?.text).toContain( + "Open the failing step for the full error." + ); + }); + + // An omitted toolkit has no page at all; a preserved one still serves + // yesterday's. Leading with the softer of the two is how the real alert + // read as "handled, ignore me". + it("leads with the omitted toolkits when both kinds are present", () => { + const message = buildDocsAlert({ + recoveredToolkits: [preservedGithub, omittedMicrosoftUsers], + }); + const text = message?.text ?? ""; + + expect(text.startsWith(":no_entry:")).toBe(true); + expect(text.indexOf("Missing entirely")).toBeLessThan( + text.indexOf("Still serving the previous docs") + ); + }); + + it("counts toolkits when more than one is missing", () => { + const message = buildDocsAlert({ + recoveredToolkits: [ + omittedMicrosoftUsers, + { ...omittedMicrosoftUsers, id: "MicrosoftDynamics" }, + ], + }); + + expect(message?.text).toContain( + ":no_entry: 2 toolkits are missing from the docs site" + ); + }); + + it("links the job log when one is given, and omits the line when not", () => { + const withLink = buildDocsAlert( + { recoveredToolkits: [omittedMicrosoftUsers] }, + { logUrl: "https://github.com/ArcadeAI/docs/actions/runs/1/job/2" } + ); + const withoutLink = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(withLink?.text).toContain( + "" + ); + expect(withoutLink?.text).not.toContain("Open the failing step>"); + }); +}); diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index 2e739f501..bad7b93cf 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -70,21 +70,55 @@ test("porter workflow opts JS actions into Node 24 to unblock the 2026-06-02 dep ); }); -test("porter workflow alerts Slack when generation fails", () => { +test("porter workflow alerts Slack on a red run", () => { expect(workflowContents).toContain("needs.generate.result == 'failure'"); expect(workflowContents).toContain("SLACK_PROJ_DOCS_WEBHOOK_URL"); // The jq program is single-quoted, so the shell passes backslashes through // untouched. `\n` reaches jq as a newline escape; `\\n` would reach it as an // escaped backslash followed by "n" and Slack would print a literal "\n". - expect(workflowContents).toContain("generation failed\\n\\n*Workflow run:*"); + expect(workflowContents).toContain('$headline + "\\n\\n*Workflow run:* <"'); expect(workflowContents).not.toContain("\\\\n"); }); -test("porter workflow warns when it preserves or omits a broken toolkit", () => { - expect(workflowContents).toContain("preservedToolkits"); - expect(workflowContents).toContain("omittedToolkits"); - expect(workflowContents).toContain("Continuing to serve previous docs"); - expect(workflowContents).toContain("No docs are being served"); +test("porter workflow builds the preserved/omitted message in tested code", () => { + // The message text lives in src/alerts/docs-alert.ts, not in this YAML. A jq + // program embedded in a workflow is untestable and drifts from what the + // generator actually knows about each failure. + expect(workflowContents).toContain("src/cli/index.ts alert"); + expect(workflowContents).toContain("--log-url"); +}); + +test("porter workflow lets a failed Slack post fail the job", () => { + // A wrong webhook secret went unnoticed because this step swallowed its own + // failure. The alert nobody is watching for is the one that must be loud. + const slackStep = workflowContents.slice( + workflowContents.indexOf("Report preserved or omitted toolkits to Slack") + ); + expect(slackStep).not.toContain("continue-on-error"); +}); + +test("porter workflow only claims generation failed when generation failed", () => { + // Steps after generation can now fail the job. Calling that a generation + // failure sends someone hunting for a validation error that doesn't exist, + // so the alert picks its headline from whether generation itself finished. + expect(workflowContents).toContain("generation-succeeded:"); + expect(workflowContents).toContain("steps.generate-docs.outputs.succeeded"); + expect(workflowContents).toContain('echo "succeeded=true"'); + expect(workflowContents).toContain("GENERATION_SUCCEEDED:"); + expect(workflowContents).toContain( + "needs.generate.outputs.generation-succeeded" + ); + expect(workflowContents).toContain('if [ "$GENERATION_SUCCEEDED" = "true" ]'); + expect(workflowContents).toContain( + "Toolkit docs generated, but the workflow failed afterward" + ); + expect(workflowContents).toContain("Toolkit docs generation failed"); +}); + +test("porter workflow keeps the failure report as an artifact", () => { + expect(workflowContents).toContain("actions/upload-artifact"); + expect(workflowContents).toContain("failed-tools"); + expect(workflowContents).toContain("actions: read"); }); test("workflow dispatch keeps default full-run behavior", () => {