diff --git a/CHANGELOG.md b/CHANGELOG.md index 7623071..12e466a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ ## Unreleased +### Changed + +- **`verdictFor` refuses a check that cannot fail.** At rung 4 and above, two evidence shapes that + used to grade `verified` now grade `uncheckable`: a check recorded without an `expect` value, and + a constant-emitter check. An exit code alone does not reproduce a value, and a command that + prints its own expectation cannot refute the claim it is attached to. +- A constant emitter is defined narrowly and mechanically: the whole command is `true` or `:`, or + the whole command is one `echo` or `printf` whose arguments hold no command substitution, no + pipe, no command separator, no redirection from a file, and no variable reference. A check that + reads a value, such as `echo "n=$(grep -c x out.txt)"`, still verifies. +- Behaviour below rung 4 does not change. An execution that decided the claim still outranks these + refusals: a missing input stays `unrunnable`, and a nonzero exit stays `contradicted`. + +### Added + +- `gradeFor(evidence, execution)` returns `{ verdict, note }`. The note names the refused shape and + tells the author what to record. `verdictFor` keeps its signature and returns the verdict alone. + ## 7.2.6 ### Changed diff --git a/src/claim-evidence.test.ts b/src/claim-evidence.test.ts index 42991ec..ecf2072 100644 --- a/src/claim-evidence.test.ts +++ b/src/claim-evidence.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest' -import { assertGradeableEvidence, UncheckableClaimError, verdictFor } from './claim-evidence' +import { + assertGradeableEvidence, + gradeFor, + UncheckableClaimError, + verdictFor, +} from './claim-evidence' describe('assertGradeableEvidence', () => { it('refuses rung 4+ without a check — a self-grade must fail at record time', () => { @@ -51,3 +56,117 @@ describe('verdictFor — the calibration cases that were graded wrong before thi expect(verdictFor({ rung: 4, check: 'x' }, null)).toBe('unrunnable') }) }) + +const PASSED = { exitCode: 0, stdout: '', stderr: '' } + +describe('verdictFor — a check that cannot fail is refused at the checkable rungs', () => { + it('an exit code with no expectation decides nothing', () => { + expect(verdictFor({ rung: 4, check: 'pnpm test' }, { ...PASSED, stdout: 'ok' })).toBe( + 'uncheckable', + ) + expect( + verdictFor({ rung: 4, check: 'pnpm test', expect: ' ' }, { ...PASSED, stdout: 'ok' }), + ).toBe('uncheckable') + }) + + it('`true` and `:` as the whole check are constant emitters', () => { + expect(verdictFor({ rung: 4, check: 'true', expect: '1' }, PASSED)).toBe('uncheckable') + expect(verdictFor({ rung: 5, check: ' : ', expect: '1' }, PASSED)).toBe('uncheckable') + }) + + it('an echo that prints its own expectation is a constant emitter', () => { + expect( + verdictFor( + { rung: 4, check: "echo 'roots_checked=2983'", expect: 'roots_checked=2983' }, + { ...PASSED, stdout: 'roots_checked=2983' }, + ), + ).toBe('uncheckable') + }) + + it('a printf that prints its own expectation is a constant emitter', () => { + expect( + verdictFor( + { rung: 4, check: "printf '%s\\n' 'ratio=0.91'", expect: 'ratio=0.91' }, + { ...PASSED, stdout: 'ratio=0.91' }, + ), + ).toBe('uncheckable') + }) + + it('an echo that reads a value through command substitution still verifies', () => { + expect( + verdictFor( + { rung: 4, check: 'echo "roots=$(grep -c root out.txt)"', expect: 'roots=2983' }, + { ...PASSED, stdout: 'roots=2983' }, + ), + ).toBe('verified') + expect( + verdictFor( + { rung: 4, check: 'echo "roots=`grep -c root out.txt`"', expect: 'roots=2983' }, + { ...PASSED, stdout: 'roots=2983' }, + ), + ).toBe('verified') + expect( + verdictFor( + { rung: 4, check: 'echo $ROOTS', expect: 'roots=2983' }, + { ...PASSED, stdout: 'roots=2983' }, + ), + ).toBe('verified') + }) + + it('a real command that prints the decisive value keeps verifying', () => { + expect( + verdictFor( + { rung: 4, check: 'grep -c root out.txt', expect: '2983' }, + { ...PASSED, stdout: '2983' }, + ), + ).toBe('verified') + }) + + it('below the threshold nothing tightens', () => { + expect(verdictFor({ rung: 3, check: 'true' }, PASSED)).toBe('verified') + expect( + verdictFor( + { rung: 3, check: "echo 'tests pass'", expect: 'tests pass' }, + { ...PASSED, stdout: 'tests pass' }, + ), + ).toBe('verified') + expect(verdictFor({ rung: 1 }, PASSED)).toBe('verified') + }) + + it('an unrunnable or contradicting execution still outranks a missing expectation', () => { + expect( + verdictFor( + { rung: 4, check: 'python k3.py' }, + { exitCode: 1, stdout: '', stderr: 'FileNotFoundError: k3.json' }, + ), + ).toBe('unrunnable') + expect( + verdictFor( + { rung: 4, check: 'python k3.py' }, + { exitCode: 1, stdout: 'assert failed', stderr: '' }, + ), + ).toBe('contradicted') + }) +}) + +describe('gradeFor — the refusal says which shape it refused', () => { + it('names the missing expectation', () => { + const grade = gradeFor({ rung: 4, check: 'pnpm test' }, { ...PASSED, stdout: 'ok' }) + expect(grade.verdict).toBe('uncheckable') + expect(grade.note).toContain('exit code alone') + }) + + it('names the constant-emitter shape', () => { + const grade = gradeFor({ rung: 4, check: "echo 'ratio=0.91'", expect: 'ratio=0.91' }, PASSED) + expect(grade.verdict).toBe('uncheckable') + expect(grade.note).toContain('constant emitter') + }) + + it('carries no note when the check decided the claim', () => { + const grade = gradeFor( + { rung: 4, check: 'grep -c root out.txt', expect: '2983' }, + { ...PASSED, stdout: '2983' }, + ) + expect(grade).toEqual({ verdict: 'verified' }) + }) +}) diff --git a/src/claim-evidence.ts b/src/claim-evidence.ts index ff78de7..023b7e7 100644 --- a/src/claim-evidence.ts +++ b/src/claim-evidence.ts @@ -76,7 +76,8 @@ export function assertGradeableEvidence(evidence: ClaimEvidence): ClaimEvidence * lacks the expectation * unrunnable the check itself could not execute (missing input, missing module) — an * environment verdict, never a claim verdict - * uncheckable rung demanded a check and none was recorded — a self-grade, counted against + * uncheckable the recorded evidence cannot decide the claim at this rung: no check, no + * expectation for the check to print, or a check that cannot fail */ export type ClaimVerdict = | 'verified' @@ -95,22 +96,97 @@ export interface CheckExecution { stderr: string } +/** The commands that succeed unconditionally, so their exit code carries no information. */ +const ALWAYS_SUCCEEDS = new Set(['true', ':']) + +/** The commands that print their arguments back, so their output carries only their arguments. */ +const PRINTS_ITS_ARGUMENTS = new Set(['echo', 'printf']) + +/** + * A character that makes an argument depend on something outside the command line: a pipe, a + * command separator, a redirection from a file, a backtick or `$(` substitution, or a variable + * reference. `$'` and `$"` are quoting sigils, not variable references. + */ +const READS_SOMETHING_ELSE = /[|;&\n<`]|\$(?!['"])/ + +/** + * Whether a check emits a constant, and therefore cannot fail whatever the claim's subject does. + * + * The test is deliberately narrow and mechanical. It recognizes exactly two shapes: the whole + * command is `true` or `:`, or the whole command is one `echo` or `printf` whose arguments read + * nothing outside the command line. Anything else is treated as a real check. + * + * This CANNOT catch every check that cannot fail. A script that prints a hard-coded number, a + * command whose output an author copied into `expect`, and an `echo` behind a shell alias all look + * identical to a real check from here. Catching those is not this function's job: it needs an + * independent party to re-derive the value, which is what rung 5 means. This function refuses only + * the shapes that carry zero information on their face, where refusing costs the author nothing + * but a rewrite of the command. + */ +function isConstantEmitter(check: string): boolean { + const command = check.trim() + if (ALWAYS_SUCCEEDS.has(command)) return true + const argumentStart = command.search(/\s/) + const head = argumentStart === -1 ? command : command.slice(0, argumentStart) + if (!PRINTS_ITS_ARGUMENTS.has(head)) return false + const argumentText = argumentStart === -1 ? '' : command.slice(argumentStart + 1) + return !READS_SOMETHING_ELSE.test(argumentText) +} + +/** A verdict with the reason a grader may report to the claim's author. */ +export interface ClaimGrade { + verdict: ClaimVerdict + /** Present when the verdict is a refusal the author can fix, absent otherwise. */ + note?: string +} + +const NO_CHECK_NOTE = + 'a claim at this rung asserts that a command reproduces a value, and no command was recorded' + +const NO_EXPECTATION_NOTE = + 'exit code alone cannot verify a claim at this rung — record the value the check must print' + +const CONSTANT_EMITTER_NOTE = + 'the check is a constant emitter: it prints a fixed string and exits zero whatever the claim ' + + 'describes, so it can never fail — record a command that reads the artifact the claim is about' + /** * The calibrated verdict function, pure so every grader shares one semantics. Callers execute the * check however their environment requires and pass the observation; this function only judges. + * + * At and above `CHECKABLE_RUNG_THRESHOLD` a check must be able to fail. An acceptance criterion + * that cannot fail is not one, so a constant-emitter check and a check with nothing to print are + * `uncheckable`: the check did not decide the claim. They are not `contradicted`, because nothing + * contradicted the claim. An execution that ran and refuted the claim still outranks both, so a + * missing input stays `unrunnable` and a nonzero exit stays `contradicted`. */ -export function verdictFor( +export function gradeFor( evidence: Pick, execution: CheckExecution | null, -): ClaimVerdict { - if (evidence.rung >= CHECKABLE_RUNG_THRESHOLD && !evidence.check) return 'uncheckable' - if (!execution) return 'unrunnable' +): ClaimGrade { + const mustBeCheckable = evidence.rung >= CHECKABLE_RUNG_THRESHOLD + if (mustBeCheckable && !evidence.check) return { verdict: 'uncheckable', note: NO_CHECK_NOTE } + if (mustBeCheckable && evidence.check && isConstantEmitter(evidence.check)) { + return { verdict: 'uncheckable', note: CONSTANT_EMITTER_NOTE } + } + if (!execution) return { verdict: 'unrunnable' } const output = `${execution.stdout}\n${execution.stderr}`.trim() if (execution.exitCode !== 0) { - return UNRUNNABLE_SIGNATURES.test(output) ? 'unrunnable' : 'contradicted' + return { verdict: UNRUNNABLE_SIGNATURES.test(output) ? 'unrunnable' : 'contradicted' } + } + if (mustBeCheckable && !evidence.expect?.trim()) { + return { verdict: 'uncheckable', note: NO_EXPECTATION_NOTE } } if (evidence.expect && !output.includes(evidence.expect)) { - return output === '' ? 'silent-check' : 'contradicted' + return { verdict: output === '' ? 'silent-check' : 'contradicted' } } - return 'verified' + return { verdict: 'verified' } +} + +/** The verdict alone, for graders that report a lattice member and not a reason. */ +export function verdictFor( + evidence: Pick, + execution: CheckExecution | null, +): ClaimVerdict { + return gradeFor(evidence, execution).verdict }