fix(claim-evidence): refuse a check that cannot fail

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 ## Unreleased
 
+### Changed
+
+- **`verdictFor` refuses a check that cannot fail.** At rung 4 and above, two evidence shapes that
+  used to grade `verified` now grade `uncheckable`: a check recorded without an `expect` value, and
+  a constant-emitter check. An exit code alone does not reproduce a value, and a command that
+  prints its own expectation cannot refute the claim it is attached to.
+- A constant emitter is defined narrowly and mechanically: the whole command is `true` or `:`, or
+  the whole command is one `echo` or `printf` whose arguments hold no command substitution, no
+  pipe, no command separator, no redirection from a file, and no variable reference. A check that
+  reads a value, such as `echo "n=$(grep -c x out.txt)"`, still verifies.
+- Behaviour below rung 4 does not change. An execution that decided the claim still outranks these
+  refusals: a missing input stays `unrunnable`, and a nonzero exit stays `contradicted`.
+
+### Added
+
+- `gradeFor(evidence, execution)` returns `{ verdict, note }`. The note names the refused shape and
+  tells the author what to record. `verdictFor` keeps its signature and returns the verdict alone.
+
 ## 7.2.6
 
 ### Changed

src/claim-evidence.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it } from 'vitest'
-import { assertGradeableEvidence, UncheckableClaimError, verdictFor } from './claim-evidence'
+import {
+  assertGradeableEvidence,
+  gradeFor,
+  UncheckableClaimError,
+  verdictFor,
+} from './claim-evidence'
 
 describe('assertGradeableEvidence', () => {
   it('refuses rung 4+ without a check — a self-grade must fail at record time', () => {
@@ -51,3 +56,117 @@ describe('verdictFor — the calibration cases that were graded wrong before thi
     expect(verdictFor({ rung: 4, check: 'x' }, null)).toBe('unrunnable')
   })
 })
+
+const PASSED = { exitCode: 0, stdout: '', stderr: '' }
+
+describe('verdictFor — a check that cannot fail is refused at the checkable rungs', () => {
+  it('an exit code with no expectation decides nothing', () => {
+    expect(verdictFor({ rung: 4, check: 'pnpm test' }, { ...PASSED, stdout: 'ok' })).toBe(
+      'uncheckable',
+    )
+    expect(
+      verdictFor({ rung: 4, check: 'pnpm test', expect: '  ' }, { ...PASSED, stdout: 'ok' }),
+    ).toBe('uncheckable')
+  })
+
+  it('`true` and `:` as the whole check are constant emitters', () => {
+    expect(verdictFor({ rung: 4, check: 'true', expect: '1' }, PASSED)).toBe('uncheckable')
+    expect(verdictFor({ rung: 5, check: ' : ', expect: '1' }, PASSED)).toBe('uncheckable')
+  })
+
+  it('an echo that prints its own expectation is a constant emitter', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: "echo 'roots_checked=2983'", expect: 'roots_checked=2983' },
+        { ...PASSED, stdout: 'roots_checked=2983' },
+      ),
+    ).toBe('uncheckable')
+  })
+
+  it('a printf that prints its own expectation is a constant emitter', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: "printf '%s\\n' 'ratio=0.91'", expect: 'ratio=0.91' },
+        { ...PASSED, stdout: 'ratio=0.91' },
+      ),
+    ).toBe('uncheckable')
+  })
+
+  it('an echo that reads a value through command substitution still verifies', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: 'echo "roots=$(grep -c root out.txt)"', expect: 'roots=2983' },
+        { ...PASSED, stdout: 'roots=2983' },
+      ),
+    ).toBe('verified')
+    expect(
+      verdictFor(
+        { rung: 4, check: 'echo "roots=`grep -c root out.txt`"', expect: 'roots=2983' },
+        { ...PASSED, stdout: 'roots=2983' },
+      ),
+    ).toBe('verified')
+    expect(
+      verdictFor(
+        { rung: 4, check: 'echo $ROOTS', expect: 'roots=2983' },
+        { ...PASSED, stdout: 'roots=2983' },
+      ),
+    ).toBe('verified')
+  })
+
+  it('a real command that prints the decisive value keeps verifying', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: 'grep -c root out.txt', expect: '2983' },
+        { ...PASSED, stdout: '2983' },
+      ),
+    ).toBe('verified')
+  })
+
+  it('below the threshold nothing tightens', () => {
+    expect(verdictFor({ rung: 3, check: 'true' }, PASSED)).toBe('verified')
+    expect(
+      verdictFor(
+        { rung: 3, check: "echo 'tests pass'", expect: 'tests pass' },
+        { ...PASSED, stdout: 'tests pass' },
+      ),
+    ).toBe('verified')
+    expect(verdictFor({ rung: 1 }, PASSED)).toBe('verified')
+  })
+
+  it('an unrunnable or contradicting execution still outranks a missing expectation', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: 'python k3.py' },
+        { exitCode: 1, stdout: '', stderr: 'FileNotFoundError: k3.json' },
+      ),
+    ).toBe('unrunnable')
+    expect(
+      verdictFor(
+        { rung: 4, check: 'python k3.py' },
+        { exitCode: 1, stdout: 'assert failed', stderr: '' },
+      ),
+    ).toBe('contradicted')
+  })
+})
+
+describe('gradeFor — the refusal says which shape it refused', () => {
+  it('names the missing expectation', () => {
+    const grade = gradeFor({ rung: 4, check: 'pnpm test' }, { ...PASSED, stdout: 'ok' })
+    expect(grade.verdict).toBe('uncheckable')
+    expect(grade.note).toContain('exit code alone')
+  })
+
+  it('names the constant-emitter shape', () => {
+    const grade = gradeFor({ rung: 4, check: "echo 'ratio=0.91'", expect: 'ratio=0.91' }, PASSED)
+    expect(grade.verdict).toBe('uncheckable')
+    expect(grade.note).toContain('constant emitter')
+  })
+
+  it('carries no note when the check decided the claim', () => {
+    const grade = gradeFor(
+      { rung: 4, check: 'grep -c root out.txt', expect: '2983' },
+      { ...PASSED, stdout: '2983' },
+    )
+    expect(grade).toEqual({ verdict: 'verified' })
+  })
+})

src/claim-evidence.ts

@@ -76,7 +76,8 @@ export function assertGradeableEvidence(evidence: ClaimEvidence): ClaimEvidence
  *                 lacks the expectation
  *   unrunnable    the check itself could not execute (missing input, missing module) — an
  *                 environment verdict, never a claim verdict
- *   uncheckable   rung demanded a check and none was recorded — a self-grade, counted against
+ *   uncheckable   the recorded evidence cannot decide the claim at this rung: no check, no
+ *                 expectation for the check to print, or a check that cannot fail
  */
 export type ClaimVerdict =
   | 'verified'
@@ -95,22 +96,97 @@ export interface CheckExecution {
   stderr: string
 }
 
+/** The commands that succeed unconditionally, so their exit code carries no information. */
+const ALWAYS_SUCCEEDS = new Set(['true', ':'])
+
+/** The commands that print their arguments back, so their output carries only their arguments. */
+const PRINTS_ITS_ARGUMENTS = new Set(['echo', 'printf'])
+
+/**
+ * A character that makes an argument depend on something outside the command line: a pipe, a
+ * command separator, a redirection from a file, a backtick or `$(` substitution, or a variable
+ * reference. `$'` and `$"` are quoting sigils, not variable references.
+ */
+const READS_SOMETHING_ELSE = /[|;&\n<`]|\$(?!['"])/
+
+/**
+ * Whether a check emits a constant, and therefore cannot fail whatever the claim's subject does.
+ *
+ * The test is deliberately narrow and mechanical. It recognizes exactly two shapes: the whole
+ * command is `true` or `:`, or the whole command is one `echo` or `printf` whose arguments read
+ * nothing outside the command line. Anything else is treated as a real check.
+ *
+ * This CANNOT catch every check that cannot fail. A script that prints a hard-coded number, a
+ * command whose output an author copied into `expect`, and an `echo` behind a shell alias all look
+ * identical to a real check from here. Catching those is not this function's job: it needs an
+ * independent party to re-derive the value, which is what rung 5 means. This function refuses only
+ * the shapes that carry zero information on their face, where refusing costs the author nothing
+ * but a rewrite of the command.
+ */
+function isConstantEmitter(check: string): boolean {
+  const command = check.trim()
+  if (ALWAYS_SUCCEEDS.has(command)) return true
+  const argumentStart = command.search(/\s/)
+  const head = argumentStart === -1 ? command : command.slice(0, argumentStart)
+  if (!PRINTS_ITS_ARGUMENTS.has(head)) return false
+  const argumentText = argumentStart === -1 ? '' : command.slice(argumentStart + 1)
+  return !READS_SOMETHING_ELSE.test(argumentText)
+}
+
+/** A verdict with the reason a grader may report to the claim's author. */
+export interface ClaimGrade {
+  verdict: ClaimVerdict
+  /** Present when the verdict is a refusal the author can fix, absent otherwise. */
+  note?: string
+}
+
+const NO_CHECK_NOTE =
+  'a claim at this rung asserts that a command reproduces a value, and no command was recorded'
+
+const NO_EXPECTATION_NOTE =
+  'exit code alone cannot verify a claim at this rung — record the value the check must print'
+
+const CONSTANT_EMITTER_NOTE =
+  'the check is a constant emitter: it prints a fixed string and exits zero whatever the claim ' +
+  'describes, so it can never fail — record a command that reads the artifact the claim is about'
+
 /**
  * The calibrated verdict function, pure so every grader shares one semantics. Callers execute the
  * check however their environment requires and pass the observation; this function only judges.
+ *
+ * At and above `CHECKABLE_RUNG_THRESHOLD` a check must be able to fail. An acceptance criterion
+ * that cannot fail is not one, so a constant-emitter check and a check with nothing to print are
+ * `uncheckable`: the check did not decide the claim. They are not `contradicted`, because nothing
+ * contradicted the claim. An execution that ran and refuted the claim still outranks both, so a
+ * missing input stays `unrunnable` and a nonzero exit stays `contradicted`.
  */
-export function verdictFor(
+export function gradeFor(
   evidence: Pick<ClaimEvidence, 'rung' | 'check' | 'expect'>,
   execution: CheckExecution | null,
-): ClaimVerdict {
-  if (evidence.rung >= CHECKABLE_RUNG_THRESHOLD && !evidence.check) return 'uncheckable'
-  if (!execution) return 'unrunnable'
+): ClaimGrade {
+  const mustBeCheckable = evidence.rung >= CHECKABLE_RUNG_THRESHOLD
+  if (mustBeCheckable && !evidence.check) return { verdict: 'uncheckable', note: NO_CHECK_NOTE }
+  if (mustBeCheckable && evidence.check && isConstantEmitter(evidence.check)) {
+    return { verdict: 'uncheckable', note: CONSTANT_EMITTER_NOTE }
+  }
+  if (!execution) return { verdict: 'unrunnable' }
   const output = `${execution.stdout}\n${execution.stderr}`.trim()
   if (execution.exitCode !== 0) {
-    return UNRUNNABLE_SIGNATURES.test(output) ? 'unrunnable' : 'contradicted'
+    return { verdict: UNRUNNABLE_SIGNATURES.test(output) ? 'unrunnable' : 'contradicted' }
+  }
+  if (mustBeCheckable && !evidence.expect?.trim()) {
+    return { verdict: 'uncheckable', note: NO_EXPECTATION_NOTE }
   }
   if (evidence.expect && !output.includes(evidence.expect)) {
-    return output === '' ? 'silent-check' : 'contradicted'
+    return { verdict: output === '' ? 'silent-check' : 'contradicted' }
   }
-  return 'verified'
+  return { verdict: 'verified' }
+}
+
+/** The verdict alone, for graders that report a lattice member and not a reason. */
+export function verdictFor(
+  evidence: Pick<ClaimEvidence, 'rung' | 'check' | 'expect'>,
+  execution: CheckExecution | null,
+): ClaimVerdict {
+  return gradeFor(evidence, execution).verdict
 }