Merge pull request #10 from coder/cat/scanner-llm-mode

feat(scanner): enable SkillSpector LLM semantic pass (Anthropic Claude)

Author: DevelopmentCats

.github/workflows/scan.yaml

@@ -88,22 +88,43 @@ jobs:
             echo "drift=true" >> "$GITHUB_OUTPUT"
             echo "Skill path source/${{ matrix.skill_path }} not present upstream; will report catalogue drift." >&2
           fi
+      - name: Determine LLM mode
+        id: llm
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          set -euo pipefail
+          if [[ -n "${ANTHROPIC_API_KEY:-}" ]]; then
+            echo "extra_flags=" >> "$GITHUB_OUTPUT"
+            echo "SkillSpector LLM mode: enabled (anthropic provider, api.anthropic.com)." >&2
+          else
+            echo "extra_flags=--no-llm" >> "$GITHUB_OUTPUT"
+            echo "::warning::ANTHROPIC_API_KEY secret not set; SkillSpector will run with --no-llm. Set the secret on this repo to enable the LLM semantic pass."
+          fi
       - name: SkillSpector (JSON)
         if: steps.path_check.outputs.drift == 'false'
         continue-on-error: true
+        env:
+          SKILLSPECTOR_PROVIDER: anthropic
+          SKILLSPECTOR_MODEL: claude-sonnet-4-6
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
         run: |
           mkdir -p out
           skillspector scan "source/${{ matrix.skill_path }}" \
-            --no-llm \
+            ${{ steps.llm.outputs.extra_flags }} \
             --format json \
             --output "out/skillspector.json" || true
       - name: SkillSpector (SARIF)
         if: steps.path_check.outputs.drift == 'false'
         continue-on-error: true
+        env:
+          SKILLSPECTOR_PROVIDER: anthropic
+          SKILLSPECTOR_MODEL: claude-sonnet-4-6
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
         run: |
           mkdir -p out
           skillspector scan "source/${{ matrix.skill_path }}" \
-            --no-llm \
+            ${{ steps.llm.outputs.extra_flags }} \
             --format sarif \
             --output "out/skillspector.sarif" || true
       - name: Combine

README.md

@@ -8,8 +8,10 @@ Every 6 hours, the scheduled workflow in this repo:
 1. Enumerates every skill in `coder/registry` (both the in-tree
    `.agents/skills/` format and the future external-sources format).
 2. Shallow-clones each source repo.
-3. Runs [NVIDIA SkillSpector](https://github.com/NVIDIA/SkillSpector) in
-   `--no-llm` static mode over the upstream content.
+3. Runs [NVIDIA SkillSpector](https://github.com/NVIDIA/SkillSpector) over
+   the upstream content. The scheduled scan runs SkillSpector's LLM
+   semantic pass when the workflow's LLM credential secret is
+   configured, and falls back to `--no-llm` static-only mode otherwise.
 4. Builds a per-skill verdict (`clean`, `suspicious`, `malicious`,
    `unknown`) from `risk_score` plus the thresholds in `config.yaml`.
 5. Builds the React SPA in `site/` and ships it together with
@@ -97,7 +99,13 @@ This scanner is data-driven. To run it against a different registry:
    "GitHub Actions").
 4. Set Actions workflow permissions to "Read and write" so the
    publish-release job can create releases.
-5. Enable Actions.
+5. To enable the LLM semantic pass, set the credential secret matching
+   `config.yaml`'s `scanners.skillspector.llm.provider` on your fork
+   (for the default `anthropic` provider, `ANTHROPIC_API_KEY`), AND
+   confirm `.github/workflows/scan.yaml` exports that secret into the
+   SkillSpector step. Static-only mode (without the secret) is the
+   default and works out of the box.
+6. Enable Actions.
 
 No source changes required for catalogue changes.
 
@@ -112,10 +120,7 @@ verdict:
 ```
 
 SkillSpector's `risk_score` (0-100) is the only input. The thresholds
-are aligned to SkillSpector's own `HIGH` and `CRITICAL` bands;
-[`docs/CALIBRATION.md`](./docs/CALIBRATION.md) walks through the
-evidence (SkillSpector source, the ClawHub paper, our in-tree
-catalogue) behind the chosen numbers.
+are aligned to SkillSpector's own `HIGH` and `CRITICAL` bands.
 
 The architecture keeps room for additional scanners (gitleaks, Semgrep,
 VirusTotal Premium, etc.); adding one is a new module under `scanner/`,

config.yaml

@@ -6,11 +6,13 @@
 config_version: 1
 
 catalogue:
-  # Where to enumerate skills from. Both the current production format
-  # (in-tree under .agents/skills/) and the future external-sources
-  # format (registry/<ns>/skills/README.md with sources[].repo) are
-  # supported. When both name the same slug, the external-sources entry
-  # wins.
+  # Skills are declared by per-namespace README.md files under
+  # registry/<ns>/skills/ in the catalogue repo. Each README's
+  # frontmatter lists sources[].repo plus per-skill overrides. This is
+  # the canonical declaration; the in-tree .agents/skills/ format is
+  # supported in scanner/enumerate.py for forks that need it but is
+  # not enabled here because coder/registry duplicates the same
+  # upstream skills across both layouts under different slugs.
   registry_repo:
     owner: coder
     repo: registry
@@ -22,13 +24,6 @@ catalogue:
       # has its frontmatter parsed for sources[].repo plus per-skill
       # overrides keyed by slug.
       readme_glob: registry/*/skills/README.md
-    in_tree:
-      enabled: true
-      # The namespace is fixed for in-tree skills today (coder).
-      namespace: coder
-      # Path glob inside the catalogue repo. Each <slug>/SKILL.md is one
-      # skill row in the matrix.
-      base_path: .agents/skills
 
 scanners:
   skillspector:
@@ -39,8 +34,12 @@ scanners:
     # so a bumper bot lives outside the loop until the upstream
     # publishes to PyPI and the pin can move into pyproject.toml.
     pin: "skillspector @ git+https://github.com/NVIDIA/SkillSpector.git@2eb844780ab163f01468ecf142c40a2ec0fcaec0"
-    flags:
-      - "--no-llm"
+    # Empty so .github/workflows/scan.yaml can append --no-llm
+    # dynamically based on whether the LLM credential secret is set.
+    flags: []
+    llm:
+      provider: anthropic
+      model: "claude-sonnet-4-6"
 
 # Per-skill verdict policy. v1 has one input (SkillSpector risk_score).
 # When more scanners join the pipeline we add new threshold fields here
@@ -54,13 +53,12 @@ scanners:
 #   51-80  HIGH      DO_NOT_INSTALL   -> verdict: suspicious
 #   81-100 CRITICAL  DO_NOT_INSTALL   -> verdict: malicious
 #
-# Rationale and source links live in docs/CALIBRATION.md. Short version:
-# SkillSpector's static-analysis layer is loud on real catalogues (the
-# ClawHub paper measured a ~49% positive rate on 67k skills) and is
-# advisory rather than authoritative, so we only escalate above its
-# HIGH cutoff. CAUTION-band findings still appear in the per-skill page
-# so reviewers can see them; we just do not flag the skill as suspicious
-# at the catalogue level.
+# Rationale: SkillSpector's static-analysis layer is loud on real
+# catalogues (the ClawHub paper measured a ~49% positive rate on 67k
+# skills) and is advisory rather than authoritative, so we only
+# escalate above its HIGH cutoff. CAUTION-band findings still appear
+# on the per-skill page so reviewers can see them; we just do not
+# flag the skill as suspicious at the catalogue level.
 verdict:
   malicious_risk_score: 81
   suspicious_risk_score: 51

docs/CALIBRATION.md

@@ -48,7 +48,7 @@ def evaluate(
 
     thresholds = config.get("verdict") or {}
     # Defaults match config.yaml. Keep these in sync with
-    # docs/CALIBRATION.md and VerdictExplanation.tsx's defaults.
+    # VerdictExplanation.tsx's defaults.
     malicious_at = int(thresholds.get("malicious_risk_score", 81))
     suspicious_at = int(thresholds.get("suspicious_risk_score", 51))
 

scanner/verdict.py

@@ -19,7 +19,7 @@ interface RiskBarProps {
    * Optional cutoffs (0..100) for the suspicious and malicious bands.
    * When supplied, the bar renders thin tick marks at those positions so
    * the user can see how close a score is to escalating. Defaults match
-   * the policy in config.yaml and docs/CALIBRATION.md.
+   * the policy in config.yaml.
    */
   suspicious_at?: number;
   malicious_at?: number;

site/src/components/RiskBar/RiskBar.tsx

@@ -374,8 +374,7 @@ const CategoryCard: FC<CategoryCardProps> = ({ group }) => {
 export const VerdictExplanation: FC<VerdictExplanationProps> = ({
   skill,
   // Defaults match config.yaml and scanner/verdict.py. They are also
-  // SkillSpector's own HIGH and CRITICAL band edges; see
-  // docs/CALIBRATION.md for the calibration write-up.
+  // SkillSpector's own HIGH and CRITICAL band edges.
   malicious_at = 81,
   suspicious_at = 51,
   className,