fix: make docs generation alerts say why and what to do

.github/workflows/generate-toolkit-docs.yml

@@ -16,6 +16,9 @@ on:
 permissions:
   contents: write
   pull-requests: write
+  # Read this run's jobs so the Slack alert can deep-link the failing job log
+  # rather than the run summary.
+  actions: read
 
 concurrency:
   group: generate-toolkit-docs
@@ -28,6 +31,11 @@ jobs:
     # switch on 2026-06-02. Harmless today; unblocks the cutover.
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    outputs:
+      # Distinguishes "generation itself failed" from "a later step failed".
+      # Without it, a broken notification would make the alert job announce a
+      # generation failure that never happened.
+      generation-succeeded: ${{ steps.generate-docs.outputs.succeeded }}
 
     steps:
       - name: Checkout repository
@@ -48,6 +56,7 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Generate toolkit docs
+        id: generate-docs
         # Invoked by path rather than through `pnpm exec`, which would reset the
         # working directory to the repo root and break the relative paths below.
         run: |
@@ -72,6 +81,8 @@ jobs:
             --ignore-file ./skip-toolkits.txt \
             --custom-sections ./curation \
             --output data/toolkits
+
+          echo "succeeded=true" >>"$GITHUB_OUTPUT"
         working-directory: toolkit-docs-generator
         env:
           ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }}
@@ -108,42 +119,81 @@ jobs:
           branch: automation/toolkit-docs
           delete-branch: true
 
+      # A token without reviewer permission is a real gap, but it must not fail
+      # the run: the generated PR is already open and useful without a reviewer
+      # attached. An annotation keeps it visible instead of silent.
       - name: Request team review
         if: steps.cpr.outputs.pull-request-number != ''
-        continue-on-error: true
-        run: gh pr edit ${{ steps.cpr.outputs.pull-request-number }} --add-reviewer ArcadeAI/engineering-tools-and-dx
+        run: |
+          if ! gh pr edit ${{ steps.cpr.outputs.pull-request-number }} \
+            --add-reviewer ArcadeAI/engineering-tools-and-dx 2>review-error.log; then
+            echo "::warning::Could not request review on PR #${{ steps.cpr.outputs.pull-request-number }}: $(cat review-error.log)"
+          fi
         env:
           GH_TOKEN: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
 
-      - name: Warn #proj-docs about preserved or omitted toolkit docs
-        continue-on-error: true
+      - name: Upload generation report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-tools
+          path: toolkit-docs-generator-verification/logs/failed-tools.json
+          if-no-files-found: ignore
+
+      # No `continue-on-error` here on purpose. A Slack step that dies quietly
+      # is how a wrong webhook secret went unnoticed: the alert it was meant to
+      # deliver is exactly what nobody was watching for.
+      #
+      # Gated on the generate step rather than on the default "everything so far
+      # succeeded". Missing or stale pages are worth reporting even when a later
+      # step broke — under the default gate a failed sidebar sync or PR creation
+      # would skip this step, while `generation-succeeded` suppressed the alert
+      # job below, and nobody heard about the toolkits at all. `!cancelled()`
+      # rather than `always()` because `cancel-in-progress` cancels superseded
+      # runs, and those have nothing to report.
+      - name: Report preserved or omitted toolkits to Slack
+        if: ${{ !cancelled() && steps.generate-docs.outputs.succeeded == 'true' }}
         run: |
-          report=toolkit-docs-generator-verification/logs/failed-tools.json
-          if [ ! -f "$report" ]; then
+          # The deep link is a nicety; the alert is the point. Actions runs this
+          # with `-e -o pipefail`, so a failed `gh api` — or a SIGPIPE from
+          # trimming its output — would abort the step before curl and lose the
+          # alert entirely. Select the first match in jq instead of piping to
+          # head, and fall back to the run URL on any failure.
+          job_url=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \
+            --jq '[.jobs[] | select(.name == "generate") | .html_url] | .[0] // empty') || job_url=""
+
+          payload=$(../node_modules/.bin/tsx src/cli/index.ts alert \
+            --report ../toolkit-docs-generator-verification/logs/failed-tools.json \
+            --log-url "${job_url:-$RUN_URL}")
+
+          if [ -z "$payload" ]; then
+            echo "No preserved or omitted toolkits to report."
             exit 0
           fi
 
-          preserved=$(jq -r '(.preservedToolkits // []) | join(", ")' "$report")
-          omitted=$(jq -r '(.omittedToolkits // []) | join(", ")' "$report")
-          if [ -z "$preserved" ] && [ -z "$omitted" ]; then
-            exit 0
+          if [ -z "$SLACK_WEBHOOK_URL" ]; then
+            echo "::error::SLACK_PROJ_DOCS_WEBHOOK_URL is not configured"
+            exit 1
           fi
 
-          payload=$(jq -n \
-            --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
-            --arg preserved "$preserved" \
-            --arg omitted "$omitted" \
-            '{text: (":warning: Toolkit docs generation completed with recoverable failures\n\n*Workflow run:* <" + $run_url + "|Open run>" + (if $preserved != "" then "\n*Continuing to serve previous docs:* " + $preserved else "" end) + (if $omitted != "" then "\n*No docs are being served (no prior output):* " + $omitted else "" end))}')
-
           curl --fail-with-body --silent --show-error \
             -X POST \
             -H "Content-Type: application/json" \
             --data "$payload" \
-            "${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }}"
+            "$SLACK_WEBHOOK_URL"
+        working-directory: toolkit-docs-generator
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }}
 
   alert:
-    name: Alert on generation failure
-    if: ${{ always() && needs.generate.result == 'failure' }}
+    name: Alert on failure
+    # Every red run gets a message. Which message depends on whether generation
+    # itself broke or something after it did: those need different people to do
+    # different things, and calling a failed sidebar sync a "generation failure"
+    # sends someone hunting for a validation error that doesn't exist.
+    if: ${{ !cancelled() && needs.generate.result == 'failure' }}
     needs: generate
     runs-on: ubuntu-latest
     permissions: {}
@@ -154,17 +204,28 @@ jobs:
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
           TRIGGER: ${{ github.event_name }}
           COMMIT: ${{ github.sha }}
+          GENERATION_SUCCEEDED: ${{ needs.generate.outputs.generation-succeeded }}
         run: |
           if [ -z "$SLACK_WEBHOOK_URL" ]; then
             echo "::error::SLACK_PROJ_DOCS_WEBHOOK_URL is not configured"
             exit 1
           fi
 
+          if [ "$GENERATION_SUCCEEDED" = "true" ]; then
+            headline=":warning: Toolkit docs were generated but never published"
+            detail="Generation succeeded, so the toolkit JSON is fine — a step after it failed (sidebar sync, PR creation, or the Slack report). Today's docs are not on their way to the site until this run's work lands, and there may be no auto-PR to merge."
+          else
+            headline=":rotating_light: Toolkit docs generation failed"
+            detail="The failed run contains the exact file path and validation error."
+          fi
+
           payload=$(jq -n \
+            --arg headline "$headline" \
+            --arg detail "$detail" \
             --arg run_url "$RUN_URL" \
             --arg trigger "$TRIGGER" \
             --arg commit "$COMMIT" \
-            '{text: (":rotating_light: Toolkit docs generation failed\n\n*Workflow run:* <" + $run_url + "|Open failed run>\n*Trigger:* " + $trigger + "\n*Commit:* " + $commit + "\n\nThe failed run contains the exact file path and validation error.")}')
+            '{text: ($headline + "\n\n*Workflow run:* <" + $run_url + "|Open failed run>\n*Trigger:* " + $trigger + "\n*Commit:* " + $commit + "\n\n" + $detail)}')
 
           curl --fail-with-body --silent --show-error \
             -X POST \

toolkit-docs-generator/CURATION.md

@@ -241,9 +241,8 @@ toolkit with its message, and exits non-zero on failure. It runs the same
 compiler generation uses, so a pass means kinds, frontmatter, and MDX are all
 valid. It cannot check `tool:` targets.
 
-Invoke `tsx` by path, not through `pnpm exec`, which resets the working
-directory to the repo root and breaks the relative path. `pnpm dlx tsx` also
-works.
+Invoke `tsx` by path because `toolkit-docs-generator` has no package for
+`pnpm exec`.
 
 Seeing a chunk on a page needs generated JSON. Either wait for the automated
 generation PR, or generate that one toolkit with Engine credentials and run

toolkit-docs-generator/README.md

@@ -129,7 +129,9 @@ pnpm dlx tsx src/cli/index.ts generate \
 
 ## Local usage
 
-Run these commands from the `toolkit-docs-generator` directory.
+Run these commands from the `toolkit-docs-generator` directory. Invoke `tsx` by
+path because this directory has no package for `pnpm exec`; the sidebar sync
+command below runs from the repo root instead.
 
 Generate a single toolkit:
 
@@ -263,6 +265,36 @@ pnpm dlx tsx src/cli/index.ts validate-curation --toolkit GoogleFlights
 That compiles the directory with the same code generation uses and needs no
 credentials. Omit `--toolkit` to check everything.
 
+## Run alerts
+
+The nightly workflow posts to Slack when a toolkit fails and the run recovers
+from it. Two outcomes, and the difference matters:
+
+- **Missing entirely**: the toolkit failed and had no previous artifact to fall
+  back on, so it has no page on the docs site at all.
+- **Still serving the previous docs**: the toolkit failed but still publishes
+  its last good output, so readers see yesterday's page.
+
+Each entry names the reason and the fix. Every run attaches the full report as
+its `failed-tools` artifact. Build the message locally against any report with
+`pnpm dlx tsx src/cli/index.ts alert --report <file>`. It prints nothing when a
+run has nothing to report.
+
+No red run stays quiet. Three messages can arrive, and they answer different
+questions:
+
+| Message | Means | Who acts |
+| --- | --- | --- |
+| Toolkits missing or stale | Generation recovered from a per-toolkit failure | Follow the fix named in the entry |
+| Toolkit docs generation failed | The generate step itself broke | Open the run for the path and validation error |
+| Generated but never published | Generation was fine, a later step broke | Check sidebar sync and PR creation — there may be no auto-PR to merge |
+
+The first is reported from inside the generate job, gated on the generate step
+rather than on the steps after it, so a broken sidebar sync or PR creation does
+not hide missing pages. The other two come from the alert job. A run that both
+recovers from a toolkit failure and then fails to publish sends the first
+message and the third, because those are two separate things to fix.
+
 ## Troubleshooting
 
 - **Nothing regenerated**: `--skip-unchanged` exits early when tool definitions did not change.

toolkit-docs-generator/src/alerts/docs-alert.ts

@@ -0,0 +1,96 @@
+/**
+ * Slack message for a generation run that recovered from toolkit failures.
+ *
+ * The text is built here rather than in workflow YAML so that it is testable,
+ * and so it can carry the two things the report already knows but a bare list
+ * of toolkit names cannot say: why the toolkit failed, and what to do about it.
+ *
+ * `omitted` outranks `preserved` throughout. A preserved toolkit still serves
+ * yesterday's page and nobody notices; an omitted toolkit has no page at all.
+ */
+import type {
+  FailedToolsReport,
+  RecoveredToolkitEntry,
+} from "../utils/run-logs";
+
+export interface SlackMessage {
+  readonly text: string;
+}
+
+export interface DocsAlertLinks {
+  /** Deep link to the job log, so the reader lands on the error, not a summary. */
+  readonly logUrl?: string;
+}
+
+const MISSING_METADATA_REASON = "missing design-system metadata";
+const CURATION_REASON_PREFIX = "Curation";
+
+const suggestFix = (entry: RecoveredToolkitEntry): string => {
+  if (entry.reason.includes(MISSING_METADATA_REASON)) {
+    return `Add a \`${entry.id}\` entry to \`@arcadeai/design-system\` and bump the pin in this repo, or add it to \`skip-toolkits.txt\` if it is not meant to be public yet.`;
+  }
+  if (entry.reason.startsWith(CURATION_REASON_PREFIX)) {
+    return `Fix the curation source under \`toolkit-docs-generator/curation/${entry.id.toLowerCase()}/\`.`;
+  }
+  return "Open the failing step for the full error.";
+};
+
+const formatEntry = (entry: RecoveredToolkitEntry): string =>
+  [`• *${entry.id}* — ${entry.reason}`, `    Fix: ${suggestFix(entry)}`].join(
+    "\n"
+  );
+
+const formatSection = (
+  heading: string,
+  entries: readonly RecoveredToolkitEntry[]
+): string[] =>
+  entries.length === 0
+    ? []
+    : [`*${heading}*`, entries.map(formatEntry).join("\n")];
+
+const formatHeadline = (
+  omitted: readonly RecoveredToolkitEntry[],
+  preserved: readonly RecoveredToolkitEntry[]
+): string => {
+  if (omitted.length === 1) {
+    return `:no_entry: ${omitted[0]?.id} is missing from the docs site`;
+  }
+  if (omitted.length > 1) {
+    return `:no_entry: ${omitted.length} toolkits are missing from the docs site`;
+  }
+  if (preserved.length === 1) {
+    return `:warning: ${preserved[0]?.id} kept its previous docs`;
+  }
+  return `:warning: ${preserved.length} toolkits kept their previous docs`;
+};
+
+/**
+ * Build the Slack payload for a finished run, or null when the run had nothing
+ * worth interrupting anyone about.
+ */
+export const buildDocsAlert = (
+  report: Pick<FailedToolsReport, "recoveredToolkits">,
+  links: DocsAlertLinks = {}
+): SlackMessage | null => {
+  const recovered = report.recoveredToolkits ?? [];
+  if (recovered.length === 0) {
+    return null;
+  }
+
+  const omitted = recovered.filter((entry) => entry.recovery === "omitted");
+  const preserved = recovered.filter((entry) => entry.recovery === "preserved");
+
+  const blocks = [
+    formatHeadline(omitted, preserved),
+    ...formatSection("Missing entirely (no previous output)", omitted),
+    ...formatSection("Still serving the previous docs", preserved),
+  ];
+
+  if (links.logUrl) {
+    blocks.push(
+      `<${links.logUrl}|Open the failing step> · full detail in the \`failed-tools\` artifact on the run`
+    );
+  }
+
+  return { text: blocks.join("\n\n") };
+};