diff --git a/CHANGELOG.md b/CHANGELOG.md index db204b5d..d16f10a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,36 @@ new version heading in the same commit. ## [Unreleased] +## [0.323.0] β€” 2026-08-08 +### Added +- **A run outcome derived from what the OS observed, not from the agent's grade of its own homework** + (`src/edge/outcome.ts`, `docs/insights-revisit.md` Step 1). Step 0 deleted every channel that broadcast + the old self-reported rate; this is the replacement it had to earn. Ordered rules over observed facts β€” + the process crashed, the run made no tool calls, an unattended run died in seconds, a task closed while + this run held it, another run had to pick the same task up β€” each carrying the `basis` that decided it, + so a number can always be traced to its evidence. `report` stays one input among several; a human's + πŸ‘/πŸ‘Ž outranks everything. Two framing calls did more work than any rule: **the unit is a conversation** + (a `poke:` resume continues a transcript, so scoring rows counts one job several times) and **not + everything is scorable** (a person closing their own pane is not a failure β€” those leave the denominator + instead of quietly counting as not-success). + On the live 30-day instapods corpus (443 conversations): **unknown 6%**, down from ~40%, and **28 + failures against the 1** the fleet self-reported over the same window. + Two rules were bought by the falsifier rather than designed: **`died-early`** β€” unattended runs split by + wall-clock at 2m+ β†’ 96% report, 30–120s β†’ 84%, **<30s β†’ 0 of 44**, which are quota/auth deaths + (`You've hit your weekly limit`, `401 … token has expired`), this fleet's most common real failure and + structurally impossible to self-report since the agent is what stopped existing (19 found in 30 days); + and **`human-session`**, after v1 scored four completed interactive sessions as `abandoned`. + Falsifier: 35 conversations sampled stratified by basis and labelled blind from transcripts + (`scripts/outcome-label-sample.cjs`, `outcome-labels.json`, `outcome-label-score.cjs`). **v1: 50% exact + vs a 43% always-success baseline. After the two rules above: 63% vs 32%** β€” with the caveat, stated in + the doc, that the rules were revised after seeing v1's errors, so the unfitted number is 50% and a fresh + blind sample is owed before Step 2 leans on it. Pinned by `scripts/outcome-derivation-test.cjs` + (23 assertions, in `test:governance`), including the property that the metric must move when work fails + and *not* when reporting discipline changes. + Nothing consumes this yet and nothing is stamped or written β€” it is a pure read, evaluated as-of a time, + so a task that reopens tomorrow changes yesterday's verdict. The Stop-hook half of the plan was **not** + built: once runs were classified, most of the hole was unscorable or already decidable, leaving 6% + without touching the teardown path. Deferred with the reasoning recorded, not cancelled. ## [0.322.0] β€” 2026-08-08 ### Changed - **Session status is an ICON now, not a coloured dot + a word.** v0.321.0 unified the vocabulary but diff --git a/docs/insights-revisit.md b/docs/insights-revisit.md index 50a9e7d4..ce7fc984 100644 --- a/docs/insights-revisit.md +++ b/docs/insights-revisit.md @@ -176,7 +176,51 @@ no longer derives a percentage from them, and states that outcome is self-report case is the fixture Step 1 must satisfy: two states differing **only** in how many runs reported, with identical real failures, must produce identical guidance. -### Step 1 β€” an outcome that isn't self-graded +### Step 1 β€” an outcome that isn't self-graded βœ… shipped v0.323.0 + +`src/edge/outcome.ts`. Rules over facts the OS observed itself, ordered, each carrying the `basis` that +decided it so any number traces back to its evidence. Live 30-day instapods corpus, 443 conversations: + +| | conversations | note | +|---|---|---| +| scorable | **309** | the denominator | +| success | 186 | | +| partial | 37 | | +| failure | **28** | against **1** self-reported failure in the same corpus | +| noop | 30 | ran, called nothing β€” previously invisible | +| incomplete | 8 | someone else had to pick the task up | +| **unknown** | **20 = 6%** | was ~40% | +| unscorable | 134 | a person's own interactive session β€” outside the denominator, not counted as not-success | + +Two framing decisions did more work than any rule: **the unit is a conversation** (a `poke:` resume +continues a transcript β€” scoring rows counts one job several times), and **not everything is scorable** +(a human closing their own pane is not a failure). + +- **Exit β€” met.** `unknown` 6% (bar: <10%). Failure rate has variance: 9% derived vs 0.3% self-reported, + and non-success is 40% where the old metric's complement was mostly non-reporting. +- **Falsifier β€” run, and it bit.** 35 conversations sampled stratified by basis, labelled blind from + transcripts only (`scripts/outcome-label-sample.cjs` β†’ `outcome-labels.json` β†’ `outcome-label-score.cjs`). + **v1 scored 50% exact against a 43% always-success baseline** β€” beating the baseline, but not by enough + to build on. The disagreements were clustered and diagnostic, and bought two rules: + - **`died-early`.** Unattended runs split by wall-clock: 2m+ β†’ 96% report, 30–120s β†’ 84%, **<30s β†’ 0 of + 44**. Those are quota/auth deaths (`You've hit your weekly limit`, `401 … token has expired`) β€” this + fleet's most common real failure, structurally invisible to the agent because the agent is what + stopped existing. 19 found in 30 days. + - **`human-session`.** v1 called a person closing their own pane `abandoned`; the labels called four of + those successes. The OS has no verdict on an interactive session β€” same posture as chat. + After both, **63% exact against a 32% baseline** (19 judged, 9 declared unscorable, 7 unlabelable for + lack of a transcript on this box). +- **⚠ The 63% is not a clean number.** The labels were blind, but the rules were revised *after* seeing + which rows v1 got wrong, so it is partly fitted to 28 rows. **Before Step 2 leans on this, draw a fresh + blind sample against the current rules.** The honest, unfitted number is v1's 50%. +- **Known coverage gap:** the OS declines to judge 134 of 443 conversations (30%) β€” every interactive + human session. Fine for Steps 2–5, which are about unattended work; it would not be fine for a + fleet-wide "how are we doing" claim. +- **Pinned:** `scripts/outcome-derivation-test.cjs` (23 assertions, in `test:governance`), including the + property that the metric must move when work fails and *not* when reporting discipline changes. +- **Not done:** the Stop-hook half. See "What Step 1 did not do" below. + +### Step 1 (original plan) The blocking dependency for everything else. Derive a per-run outcome from **observable facts already in the DB**, not from the agent's own `report`: @@ -198,6 +242,25 @@ becomes rare rather than modal. publish the confusion counts in the PR. If the derived signal doesn't beat "always success", stop and rethink before Step 2. +#### What Step 1 did not do + +**The Stop-hook half was not built, and the reason is the evidence rather than the effort.** The plan +assumed the 40% hole had to be closed at the source β€” make unattended runs report. Once the runs were +classified it turned out most of the hole was not missing information at all: + +- 134 conversations were a person's own interactive session β€” never scorable, hook or no hook; +- 30 called no tool at all, and 19 died in seconds β€” both fully decidable from what the OS already + recorded, and *neither could ever have self-reported* (a run killed by a quota limit has nothing left + to report with). + +That left `unknown` at 6%, under the exit bar, without touching the hook. Building it anyway would have +added a runtime change to the teardown path for a residual the derivation already handles β€” so it is +**deferred, not cancelled**: if Step 2's cards turn out to be blocked by those 20 conversations, the +hook is the fix, and this paragraph is the record of why it was skipped. + +One thing the residual does say: those 20 are runs that did substantial work (11–95 tool calls) and left +no verdict. They are the most interesting runs in the corpus, and no observable fact decides them. + ### Step 2 β€” one signal, one card, one action Pick the single highest-evidence problem in the live data β€” **repeat agent crashes** (Β§2b) β€” and diff --git a/package-lock.json b/package-lock.json index 52fd0932..6987ed69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.322.0", + "version": "0.323.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.322.0", + "version": "0.323.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index f97a6eba..6d9c6a68 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.322.0", + "version": "0.323.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", @@ -27,7 +27,7 @@ "check-deps": "bash scripts/install-deps.sh --check", "dev": "ts-node src/cli.ts serve", "demo:dev": "ts-node src/demo.ts", - "test:governance": "node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/capability-registry-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/verbosity-test.cjs && node scripts/chain-model-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-resume-test.cjs && node scripts/warm-chat-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs", + "test:governance": "node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/capability-registry-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/verbosity-test.cjs && node scripts/chain-model-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-resume-test.cjs && node scripts/warm-chat-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs", "test:alert-staleness": "node scripts/alert-staleness-test.cjs", "test:deps": "node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs", "test:dm-continuity": "node scripts/dm-continuity-test.cjs", diff --git a/scripts/outcome-derivation-test.cjs b/scripts/outcome-derivation-test.cjs new file mode 100644 index 00000000..35cd3921 --- /dev/null +++ b/scripts/outcome-derivation-test.cjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +/* Derived-outcome rules (docs/insights-revisit.md, Step 1). + * + * Synthetic rows, so each rule and each precedence edge is exact. The live-corpus check is a separate, + * manual pair β€” `outcome-label-sample.cjs` + `outcome-label-score.cjs` against a DB snapshot β€” because it + * needs transcripts and a human read; this file is what CI can run. + * + * The property at the bottom is the one that matters most: the metric must move when real failures move + * and NOT when reporting discipline does. That is the exact defect Step 0 deleted, restated as a test. */ +const fs = require('fs'); +const os_ = require('os'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const HOME = fs.mkdtempSync(path.join(os_.tmpdir(), 'aos-outcome-test-')); +process.env.AGENT_OS_HOME = HOME; +process.env.AGENT_OS_TENANT = 'testco'; +delete process.env.AGENT_OS_SECRET_KEY; + +let pass = 0, fail = 0; +const assert = (c, name, d) => c ? (pass++, console.log(` \x1b[32mβœ“\x1b[0m ${name}`)) : (fail++, console.log(` \x1b[31mβœ— ${name}\x1b[0m${d ? ' β€” ' + d : ''}`)); + +const { loadAgentOS } = require(path.join(ROOT, 'dist/kernel.js')); +const { deriveRunOutcomes, foldConversations, summarize } = require(path.join(ROOT, 'dist/edge/outcome.js')); + +const aos = loadAgentOS(); +const NOW = Date.now(); +let n = 0; + +/** One terminated run. Everything the rules read is explicit β€” no hidden defaults to argue with later. */ +function mkRun(o) { + const id = 'ts_' + (++n); + const at = o.at ?? NOW - 3600_000; + aos.db.prepare( + 'INSERT INTO term_sessions (id,agent,title,task,tmux,status,headless,spawned_by,created_at,updated_at,' + + 'claude_session_id,rating,outcome,tool_calls,active_ms) VALUES (?,?,?,?,?,?,1,?,?,?,?,?,?,?,?)', + ).run(id, o.agent ?? 'worker', 't', 'x', 'aos-' + id, o.status ?? 'done', o.spawnedBy ?? 'automation:au_1', + at, at + (o.activeMs ?? 300_000), o.convo ?? null, o.rating ?? null, o.outcome ?? null, + o.toolCalls === undefined ? 20 : o.toolCalls, o.activeMs ?? 300_000); + return id; +} +const verdictOf = (id) => deriveRunOutcomes(aos, { since: 0, until: NOW + 1 }).find((r) => r.runId === id); + +console.log('\n\x1b[1mDerived outcome β€” rules over observed facts, not the agent\'s own grade\x1b[0m'); + +// ── each rule fires ──────────────────────────────────────────────────────────────────────────────── +{ + const up = mkRun({ rating: 'up', outcome: 'failure' }); // human outranks the report + const down = mkRun({ rating: 'down', outcome: 'success' }); + assert(verdictOf(up).verdict === 'success' && verdictOf(up).basis === 'human-rating', 'πŸ‘ outranks a reported failure'); + assert(verdictOf(down).verdict === 'failure' && verdictOf(down).basis === 'human-rating', 'πŸ‘Ž outranks a reported success'); + + const crashed = mkRun({ status: 'crashed', outcome: 'success' }); + assert(verdictOf(crashed).verdict === 'failure' && verdictOf(crashed).basis === 'crashed', 'a crash outranks a reported success'); + + const early = mkRun({ activeMs: 9_000, toolCalls: 2 }); + assert(verdictOf(early).basis === 'died-early' && verdictOf(early).verdict === 'failure', 'an automation run dead in 9s is a failure'); + + const noop = mkRun({ toolCalls: 0, activeMs: 600_000 }); + assert(verdictOf(noop).verdict === 'noop' && verdictOf(noop).basis === 'no-tool-calls', 'zero tool calls is a noop, not an unknown'); + + const reported = mkRun({ outcome: 'partial' }); + assert(verdictOf(reported).verdict === 'partial' && verdictOf(reported).basis === 'reported', 'the report is still used when it exists'); + + const nothing = mkRun({}); + assert(verdictOf(nothing).verdict === 'unknown' && verdictOf(nothing).basis === 'no-evidence', 'no evidence stays honestly unknown'); +} + +// ── the guards that cost a wrong answer on live data ─────────────────────────────────────────────── +{ + const shortTask = mkRun({ spawnedBy: 'task:tsk_short', activeMs: 8_000, toolCalls: 3 }); + assert(verdictOf(shortTask).basis !== 'died-early', 'a TASK run finishing in 8s is not a died-early failure (smoke tests are fast)'); + + const chatty = mkRun({ activeMs: 5_000, toolCalls: 0 }); + assert(verdictOf(chatty).verdict === 'noop', 'a short automation run that called nothing is a noop, not a failure'); + + const human = mkRun({ spawnedBy: 'm_alice', status: 'stopped', toolCalls: 40 }); + assert(verdictOf(human).verdict === 'abandoned' && verdictOf(human).basis === 'human-session', "a person's own interactive session is unscorable, not a failure"); + + const chat = mkRun({ spawnedBy: 'chat:support' }); + assert(!verdictOf(chat), 'chat-triggered runs are excluded entirely'); + + const stopped = mkRun({ spawnedBy: 'task:tsk_stop', status: 'stopped', toolCalls: 90 }); + assert(verdictOf(stopped).verdict === 'incomplete' && verdictOf(stopped).basis === 'stopped-midway', 'a killed unattended run is incomplete, never a success'); +} + +// ── conversation folding ─────────────────────────────────────────────────────────────────────────── +{ + const cid = 'convo-1'; + mkRun({ convo: cid, at: NOW - 7200_000, spawnedBy: 'task:tsk_a' }); // unknown + mkRun({ convo: cid, at: NOW - 3600_000, spawnedBy: 'poke:tsk_a', outcome: 'success' }); // then finished + const folded = foldConversations(deriveRunOutcomes(aos, { since: 0, until: NOW + 1 })).find((c) => c.convoId === cid); + assert(folded && folded.runs === 2, 'resumes fold into one conversation, not two outcomes'); + assert(folded && folded.verdict === 'success', 'a hand-off that needed a poke and then landed is a success'); + + const cid2 = 'convo-2'; + mkRun({ convo: cid2, at: NOW - 7200_000, status: 'crashed' }); + mkRun({ convo: cid2, at: NOW - 3600_000, outcome: 'success' }); + const f2 = foldConversations(deriveRunOutcomes(aos, { since: 0, until: NOW + 1 })).find((c) => c.convoId === cid2); + assert(f2 && f2.verdict === 'failure', 'a later clean run does not erase an earlier crash'); +} + +// ── the summary contract ─────────────────────────────────────────────────────────────────────────── +{ + const all = summarize(foldConversations(deriveRunOutcomes(aos, { since: 0, until: NOW + 1 }))); + assert(all.scorable === all.success + all.partial + all.incomplete + all.noop + all.failure + all.unknown, + 'scorable is exactly the decided verdicts plus the honest unknowns', JSON.stringify(all)); + assert(!Number.isNaN(all.abandoned) && all.abandoned > 0 && all.scorable > 0, 'abandoned is tracked but sits outside scorable'); + const tiny = summarize([{ convoId: 'x', agent: 'a', verdict: 'success', basis: 'reported', runs: 1, at: NOW }]); + assert(tiny.successRate === null, 'no success rate below the minimum sample'); + assert(tiny.unknownShare === 0, 'unknown share is still reported on a tiny sample'); +} + +// ── THE property: the metric must move with failures, not with reporting discipline ──────────────── +{ + const base = () => { + const rows = []; + for (let i = 0; i < 20; i++) rows.push({ convoId: 'c' + i, agent: 'a', verdict: 'success', basis: 'reported', runs: 1, at: NOW }); + return rows; + }; + // Ten runs stop calling `report` β€” but nothing actually failed. Under the old metric this alone moved + // the rate 50 points. Here they become `unknown`: the rate barely moves and the unknown SHARE shouts. + const quiet = base(); + for (let i = 0; i < 10; i++) quiet[i] = { ...quiet[i], verdict: 'unknown', basis: 'no-evidence' }; + // Ten runs genuinely fail. + const broken = base(); + for (let i = 0; i < 10; i++) broken[i] = { ...broken[i], verdict: 'failure', basis: 'crashed' }; + + const b = summarize(base()), q = summarize(quiet), r = summarize(broken); + assert(q.unknownShare === 50 && q.unknown === 10, 'ten silent runs show up as a 50% unknown share, not as failures', JSON.stringify(q)); + assert(r.failure === 10 && r.unknownShare === 0, 'ten real failures show up as failures with no unknowns', JSON.stringify(r)); + assert(b.successRate === 100 && r.successRate === 50, 'the rate moves when work actually fails'); + assert(q.successRate === 50 && q.unknownShare === 50, + 'when the rate moves for a REPORTING reason, the unknown share moves with it β€” the reader can always tell the two apart'); +} + +fs.rmSync(HOME, { recursive: true, force: true }); +console.log(`\n${fail ? '\x1b[31m' : '\x1b[32m'}${pass} passed, ${fail} failed\x1b[0m\n`); +process.exit(fail ? 1 : 0); diff --git a/scripts/outcome-label-sample.cjs b/scripts/outcome-label-sample.cjs new file mode 100644 index 00000000..db6cfd04 --- /dev/null +++ b/scripts/outcome-label-sample.cjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +/* Blind-labelling sampler for the derived outcome (docs/insights-revisit.md, Step 1). + * + * The derivation must be checked against something it cannot see. This dumps the EVIDENCE for a sample of + * conversations β€” the task it was given and what the agent actually said in the transcript β€” and writes the + * derived verdicts to a SEPARATE file the labeller is not meant to open first. Label from the evidence, + * then run `outcome-label-score.cjs` to get the confusion counts. + * + * Selection is stratified by basis (up to `PER_BASIS` each) so rare rules β€” a crash, a task close, a + * retry β€” actually appear in 30 rows instead of being drowned by the reported majority. Stratifying the + * SELECTION is fine; the labelling stays blind because the evidence file carries no verdict, no basis, and + * none of the fields the rules key on (status, tool_calls, rating, the reported outcome). + * + * AGENT_OS_HOME= node scripts/outcome-label-sample.cjs + * + * ⚠ Point AGENT_OS_HOME at a SNAPSHOT (`sqlite3 live.db ".backup snap.db"`), never the live home β€” this + * opens the DB through the kernel, which migrates on open. + * ⚠ The evidence file quotes real transcripts. It is written outside the repo and must not be committed; + * only the labels (a verdict per opaque conversation id) belong in git. + */ +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const OUT = process.argv[2] || path.join(require('os').tmpdir(), 'aos-outcome-sample'); +const PER_BASIS = 5; +const TAIL_CHARS = 1400; // how much of the transcript tail to quote per conversation + +if (!process.env.AGENT_OS_HOME) { console.error('refusing to run without AGENT_OS_HOME (use a snapshot, not the live home)'); process.exit(1); } + +const { loadAgentOS } = require(path.join(ROOT, 'dist/kernel.js')); +const { deriveRunOutcomes, foldConversations } = require(path.join(ROOT, 'dist/edge/outcome.js')); +const { readConversation } = require(path.join(ROOT, 'dist/edge/conversation.js')); + +const os = loadAgentOS(); +const convos = foldConversations(deriveRunOutcomes(os)); + +// Stratify: up to PER_BASIS per basis, taken evenly across each stratum so it isn't just the newest rows. +const byBasis = new Map(); +for (const c of convos) { const a = byBasis.get(c.basis) ?? []; a.push(c); byBasis.set(c.basis, a); } +const picked = []; +for (const [, list] of [...byBasis].sort((a, b) => a[0].localeCompare(b[0]))) { + const stride = Math.max(1, Math.floor(list.length / PER_BASIS)); + for (let i = 0; i < list.length && picked.length % PER_BASIS !== PER_BASIS - 1 + 1; i += stride) { + picked.push(list[i]); + if (picked.filter((p) => p.basis === list[i].basis).length >= PER_BASIS) break; + } +} + +// Deterministic shuffle (no Math.random β€” the sample must be reproducible), so adjacent rows in the +// evidence file don't share a basis and give the game away. +picked.sort((a, b) => (a.convoId + a.agent).localeCompare(b.convoId + b.agent)); + +const evidence = []; +const truth = []; +for (const c of picked) { + const runs = os.db.prepare('SELECT id, title, task, spawned_by, created_at FROM term_sessions WHERE COALESCE(claude_session_id, id) = ? ORDER BY created_at').all(c.convoId); + const first = runs[0] ?? {}; + let tail = ''; + try { + const conv = readConversation(c.convoId); + const text = (conv.turns ?? conv.messages ?? []) + .map((t) => `${t.role ?? t.kind ?? '?'}: ${String(t.text ?? t.content ?? '').replace(/\s+/g, ' ')}`) + .join('\n'); + tail = text.slice(-TAIL_CHARS); + } catch { tail = '(no transcript on this box)'; } + + evidence.push({ + id: c.convoId, + agent: c.agent, + runs: c.runs, + spawn: String(first.spawned_by ?? '').replace(/^(task|poke):.*/, '$1:…'), // kind only, not the task id + title: first.title ?? '', + task: String(first.task ?? '').slice(0, 700), + transcriptTail: tail, + }); + truth.push({ id: c.convoId, verdict: c.verdict, basis: c.basis }); +} + +fs.mkdirSync(OUT, { recursive: true }); +fs.writeFileSync(path.join(OUT, 'evidence.json'), JSON.stringify(evidence, null, 2)); +fs.writeFileSync(path.join(OUT, 'derived.json'), JSON.stringify(truth, null, 2)); +console.log(`${evidence.length} conversations β†’ ${OUT}/evidence.json (derived verdicts held in derived.json β€” label first)`); +console.log(`bases sampled: ${[...new Set(truth.map((t) => t.basis))].join(', ')}`); diff --git a/scripts/outcome-label-score.cjs b/scripts/outcome-label-score.cjs new file mode 100644 index 00000000..a9968122 --- /dev/null +++ b/scripts/outcome-label-score.cjs @@ -0,0 +1,83 @@ +#!/usr/bin/env node +/* Score the derived outcome against the blind hand-labels (docs/insights-revisit.md, Step 1). + * + * AGENT_OS_HOME= node scripts/outcome-label-score.cjs + * + * The bar is NOT "the rules agree with a human". It is that the derivation beats the thing it replaces. + * Two baselines are printed alongside it, both of which the old metric effectively was: + * Β· always-success β€” what a 0.3%-failure self-report is, rounded; + * Β· the reported outcome alone β€” the actual old signal, with its unknowns. + * + * A disagreement is reported per pair so the failure mode is visible rather than summarised away, and the + * `?` rows (no transcript on this box β†’ no independent evidence) are excluded from the denominator instead + * of being scored as agreement. Exit code is non-zero only if the derivation loses to a baseline. */ +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +if (!process.env.AGENT_OS_HOME) { console.error('refusing to run without AGENT_OS_HOME (use a snapshot, not the live home)'); process.exit(1); } + +const { loadAgentOS } = require(path.join(ROOT, 'dist/kernel.js')); +const { deriveRunOutcomes, foldConversations } = require(path.join(ROOT, 'dist/edge/outcome.js')); + +const labels = JSON.parse(fs.readFileSync(path.join(__dirname, 'outcome-labels.json'), 'utf8')).labels; +const os = loadAgentOS(); +const convos = new Map(foldConversations(deriveRunOutcomes(os)).map((c) => [c.convoId, c])); + +const rows = []; +for (const [id, human] of Object.entries(labels)) { + const c = convos.get(id); + if (!c) { rows.push({ id, human, derived: '(missing)', basis: '-', reported: '-' }); continue; } + rows.push({ id, human, derived: c.verdict, basis: c.basis }); +} + +// Three populations, kept apart on purpose: +// Β· judged β€” the OS claims a verdict and the labeller had evidence. The only accuracy number. +// Β· unscorable β€” the OS declares it can't judge (a person's own interactive session). Not an error, but +// NOT a free pass either: the labeller often COULD judge these, so they are reported as a +// coverage gap with the labels that were lost. +// Β· skipped β€” no transcript on this box, so no independent evidence existed to label against. +const labelled = rows.filter((r) => r.human !== '?' && r.derived !== '(missing)'); +const unscorable = labelled.filter((r) => r.derived === 'abandoned'); +const judged = labelled.filter((r) => r.derived !== 'abandoned'); +const skipped = rows.length - labelled.length; + +// Baseline 1: call everything a success (what a self-report with one failure in 329 amounts to). +const alwaysSuccess = judged.filter((r) => r.human === 'success').length; +// Baseline 2: the old signal β€” the agent's own report, `unknown` where it didn't call one. +const reportedOnly = judged.filter((r) => r.basis === 'reported' && r.human === r.derived).length; +const agree = judged.filter((r) => r.human === r.derived).length; + +// A softer, arguably fairer read: did the derivation get the SIGN right β€” did work land or not? +const sign = (v) => (v === 'success' ? 'ok' : v === 'abandoned' ? 'n/a' : 'not-ok'); +const signAgree = judged.filter((r) => sign(r.human) === sign(r.derived)).length; + +const pct = (n) => `${Math.round((n / judged.length) * 100)}%`; + +console.log(`\n\x1b[1mDerived outcome vs blind hand-labels\x1b[0m β€” ${judged.length} judged, ${unscorable.length} declared unscorable, ${skipped} skipped (no transcript β‡’ no independent evidence)\n`); +if (unscorable.length) { + const spread = unscorable.reduce((m, r) => ({ ...m, [r.human]: (m[r.human] ?? 0) + 1 }), {}); + console.log(` coverage gap: the OS declines to judge ${unscorable.length} conversations a human could label (${Object.entries(spread).map(([k, v]) => `${v} ${k}`).join(', ')})\n`); +} +console.log(` exact agreement ${agree}/${judged.length} ${pct(agree)}`); +console.log(` sign agreement ${signAgree}/${judged.length} ${pct(signAgree)} (landed vs didn't)`); +console.log(` baseline: always-success ${alwaysSuccess}/${judged.length} ${pct(alwaysSuccess)}`); +console.log(` of which the report alone got right: ${reportedOnly}\n`); + +const confusion = new Map(); +for (const r of judged) { + const k = `${r.human} β†’ ${r.derived}`; + confusion.set(k, (confusion.get(k) ?? 0) + 1); +} +console.log(' human β†’ derived:'); +for (const [k, n] of [...confusion].sort((a, b) => b[1] - a[1])) console.log(` ${n}Γ— ${k}${k.split(' β†’ ')[0] === k.split(' β†’ ')[1] ? '' : ' βœ—'}`); + +const wrong = judged.filter((r) => r.human !== r.derived); +if (wrong.length) { + console.log('\n disagreements (basis that produced the derived verdict):'); + for (const r of wrong) console.log(` ${r.id.slice(0, 8)} human=${r.human} derived=${r.derived} via ${r.basis}`); +} + +const beatsBaseline = agree > alwaysSuccess; +console.log(`\n ${beatsBaseline ? '\x1b[32mβœ“' : '\x1b[31mβœ—'} derivation ${beatsBaseline ? 'beats' : 'does NOT beat'} the always-success baseline\x1b[0m\n`); +process.exit(beatsBaseline ? 0 : 1); diff --git a/scripts/outcome-labels.json b/scripts/outcome-labels.json new file mode 100644 index 00000000..9dcbea27 --- /dev/null +++ b/scripts/outcome-labels.json @@ -0,0 +1,43 @@ +{ + "_": "Blind hand-labels for the derived-outcome falsifier (docs/insights-revisit.md, Step 1).", + "_how": "Produced by reading ONLY scripts/outcome-label-sample.cjs's evidence.json β€” the task text and the transcript tail. The labeller did not see the derived verdict, the basis, or any field the rules key on (status, tool_calls, rating, reported outcome) while labelling.", + "_corpus": "instapods, 30-day window, snapshot taken 2026-08-08. Conversation ids are opaque; no live content is committed here.", + "_scale": "success | partial | incomplete | noop | failure | abandoned | ? β€” where '?' means the transcript was not on this box, so there was no independent evidence to judge from. Those rows are reported separately and never counted as agreement.", + "labels": { + "019fa7e6-87cb-74d1-a452-3b71e68d9274": "?", + "019fb898-6a14-7323-a604-806880ed162c": "?", + "02a71d16-545e-49aa-8755-bd1af10f6273": "partial", + "04a2b38e-644a-46ff-b970-87d9f0f5339f": "incomplete", + "08f48b35-5102-4372-913a-633946bafceb": "success", + "09b9a27c-df17-4bc6-af16-7a3ec09d6333": "success", + "179297b9-3234-417a-9d19-612cf27606f5": "failure", + "1e08af09-0272-49c5-8bb0-2083f7ec9e7e": "?", + "2541afa0-4870-4639-8f58-a557490e7763": "incomplete", + "25f90fb0-a65e-484e-9bdd-a6095dc2c460": "success", + "2a52af81-2c80-4d29-b59e-fb64985d583f": "success", + "2ea86281-efab-42e2-81e2-f62afa7660fd": "abandoned", + "3d74970f-26e8-4797-ab81-0d5f77e327c1": "?", + "3d9708c4-5672-4c80-a3bf-d09fbf6d4e1d": "noop", + "4372646c-9b07-431a-b581-791cdb9a6b0b": "failure", + "46f0e2d2-8152-4d85-ba84-5a286a25f89e": "noop", + "5c5de211-4bce-4385-b91d-e1ef59e2e9a7": "success", + "68177cae-a6b3-4466-8417-ef06f24fcfff": "success", + "6a7aa2b8-202f-4f11-b7f7-7debcc67602b": "success", + "6eb7112e-da7e-4aa1-ae4e-3d848caa5b6f": "failure", + "749ce7fd-e6ce-48a1-94a8-c8482bea8e5a": "failure", + "76ddb322-791c-465e-8779-6122fc4e186d": "success", + "89a76e88-5146-423c-8c77-552aa8f957aa": "abandoned", + "9d255646-4a62-4e7e-b0ea-e3c869465249": "partial", + "9dafe94f-e9b1-4303-9d13-32488bb49338": "noop", + "a19b6dd8-26c3-4b29-bf7f-67e0ff10d159": "incomplete", + "a63a32c6-233d-4627-af8b-107fe95285d0": "failure", + "b54f45ec-56b8-43f7-ab1e-e5b377e53394": "?", + "c942ec31-e6bb-473e-a311-f6d36663925f": "success", + "d11aea9c-0940-4544-a5a0-a2857fd1eef9": "?", + "dc7f0952-14ca-48ca-810d-708a6d92a051": "?", + "e2dd73c5-ad7c-43b3-8638-919f3d06334c": "failure", + "f05f1f7e-d32e-4347-a5f6-6e481130c8ab": "success", + "fc1eef59-78d6-4701-a407-f2acf0c4f03c": "success", + "fccaeec3-60fb-48b2-88d3-6c98b288e770": "success" + } +} diff --git a/src/edge/outcome.ts b/src/edge/outcome.ts new file mode 100644 index 00000000..c1d5f5c3 --- /dev/null +++ b/src/edge/outcome.ts @@ -0,0 +1,292 @@ +/** + * **Derived run outcome** β€” Step 1 of `docs/insights-revisit.md`. + * + * Everything Insights ever claimed rested on one number the agent wrote about itself. On live instapods + * that number had no variance and a large hole: **one** reported failure in 329 reports lifetime, and + * ~40% of terminated runs never called `report` at all. So "success rate" measured whether `report()` was + * called. Step 0 deleted every channel that broadcast it; this module is the replacement it has to earn. + * + * The rules below read only **facts the OS observed itself** β€” the process crashed, the run made no tool + * calls, a task closed while this run held it, another run had to pick the same task up afterwards. The + * agent's own `report` is kept as ONE input among several, never the only one, and a human's πŸ‘/πŸ‘Ž + * outranks everything (it is the only ground truth in the system, and covers 1 run in 512 β€” which is + * exactly why it can't be the metric). + * + * Two framing choices matter as much as the rules: + * + * - **The unit is a conversation, not a run.** A `poke:`/resume run continues an existing transcript, so + * scoring each row separately counts one piece of work several times (the same folding `sessionChain` + * does β€” runs group by `claude_session_id`). + * - **Not everything is scorable.** A human who spawns an interactive session and closes it hasn't + * produced a failure; they walked away. Those fold to `abandoned` and leave the denominator entirely, + * rather than quietly counting as not-success the way the old rate did (45 of 443 conversations on the + * live 30-day corpus). + * + * **Pure, and evaluated as-of `now`.** Nothing is written or stamped: a task that reopens tomorrow should + * change yesterday's verdict, and freezing a snapshot at teardown would lock in an answer taken before the + * evidence existed. Callers that need stability must pass an explicit window. + * + * Live 30-day corpus at the time of writing (443 conversations): 259 success Β· 48 noop Β· 41 partial Β· + * 10 failure Β· 5 incomplete Β· 45 abandoned (unscored) Β· **35 unknown = 8.8% of the 398 scorable**, against + * 40% before. Non-success is 26% where the self-report claimed 0.3% β€” the point of the exercise is that + * the number can now MOVE. + */ +import type { AgentOS } from '../kernel'; + +const DAY = 24 * 3_600_000; + +/** Ordered loosely worstβ†’best; `abandoned` is outside the ordering (it leaves the denominator). */ +export type RunVerdict = 'success' | 'partial' | 'incomplete' | 'noop' | 'failure' | 'abandoned' | 'unknown'; + +/** Which observed fact decided the verdict. Carried everywhere so a number can always be traced back to + * the evidence that produced it β€” the thing the old success rate could not do. */ +export type OutcomeBasis = + | 'human-rating' // a person pressed πŸ‘/πŸ‘Ž β€” ground truth, outranks all + | 'crashed' // the pane/process died: infra failure, not a judgement of the work + | 'died-early' // an unattended run that "finished" in seconds β€” it never got going + | 'no-tool-calls' // the run took a turn and called nothing: it did no work at all + | 'reported' // the agent's own `report` outcome + | 'stopped-midway' // a person killed an unattended run while it was working + | 'task-completed' // a task closed while this run held it + | 'task-retried' // another run had to pick the same task up after this one ended + | 'human-session' // a person's own interactive session β€” the OS has no verdict on it + | 'no-evidence'; // nothing observable said anything: honestly unknown + +export interface RunOutcome { + runId: string; + convoId: string; // claude_session_id when present β€” the transcript this run belongs to + agent: string; + verdict: RunVerdict; + basis: OutcomeBasis; + at: number; // run start + taskId?: string; +} + +export interface ConversationOutcome { + convoId: string; + agent: string; + verdict: RunVerdict; + basis: OutcomeBasis; + runs: number; + at: number; // first run's start +} + +export interface OutcomeSummary { + /** Conversations with a verdict we can defend β€” excludes `abandoned`. The denominator. */ + scorable: number; + success: number; + partial: number; + incomplete: number; + noop: number; + failure: number; + /** Excluded from `scorable`: a human walked away from their own interactive session. */ + abandoned: number; + /** Scorable but undecidable β€” the honest residual. */ + unknown: number; + /** success / scorable, or null below `minSample`. Never derived from a denominator that includes + * `abandoned`, and never reported without `unknown` beside it. */ + successRate: number | null; + /** Share of scorable conversations we could not decide. A rate is meaningless if this is large β€” the + * precise failure of the metric this module replaces. */ + unknownShare: number | null; +} + +/** Below this many scorable conversations, `successRate` stays null β€” a rate needs a sample (the standing + * lesson from the 2026-07-31 approval-friction correction, applied up front this time). */ +export const MIN_SAMPLE = 10; + +interface Row { + id: string; agent: string; status: string; spawned_by: string | null; claude_session_id: string | null; + rating: string | null; outcome: string | null; tool_calls: number | null; active_ms: number | null; + created_at: number; ended: number; +} + +/** + * An unattended run that terminated in under this much working time never got going. Not a guess β€” on the + * live 30-day corpus, unattended runs split cleanly by wall-clock: + * + * 2m+ 67 runs, 64 reported an outcome (96%) + * 30–120s 50 runs, 42 reported (84%) + * <30s 44 runs, 0 reported ( 0%) + * + * Nothing under 30 seconds has EVER reported. Reading the transcripts explains why: they die on the first + * API call β€” `You've hit your weekly limit`, `401 OAuth access token has expired` β€” which is this fleet's + * single most common real failure (see the weekly-limit zombie sessions we've chased repeatedly) and is + * structurally invisible to the agent, since the agent is what stopped existing. The blind labels caught + * four of these; the first version of these rules scored them `unknown` or `noop`. + * + * Two guards, both bought with a wrong answer: + * - **Automation runs only.** The statistic above is over `automation:`-spawned runs. Applied to every + * unattended run it mislabelled a poke-test ping, an engineer smoke test and a stand-down β€” task runs + * that legitimately finish in seconds β€” as failures. + * - **It must have called something.** A run that died mid-flight had started working; one that called + * nothing just answered and stopped (an agent replying "looks like a test message"). Zero tool calls + * falls through to `noop`, which is what it is. + */ +const DIED_EARLY_MS = 30_000; + +/** A run spawned by chat is conversational Q&A β€” it answers a person and rarely `report`s, so it is not + * work with an outcome. Matches the exclusion `insights.ts` and `measurement.ts` already apply. */ +function isChat(spawnedBy: string | null): boolean { + return (spawnedBy ?? '').startsWith('chat:'); +} + +/** A person (member id or email) spawned this, as opposed to an automation / task / poke / goal. */ +function isHumanSpawned(spawnedBy: string | null): boolean { + const s = spawnedBy ?? ''; + return s.startsWith('m_') || s.includes('@'); +} + +/** The task a run was dispatched for, if any. `task:` is a first attempt, `poke:` a resume. */ +function taskIdOf(spawnedBy: string | null): string | undefined { + const s = spawnedBy ?? ''; + if (s.startsWith('task:')) return s.slice(5); + if (s.startsWith('poke:')) return s.slice(5); + return undefined; +} + +/** + * Classify every terminated run in the window. First matching rule wins; the order is the point, so it is + * spelled out rather than scored: + * + * 1. a human's verdict, when there is one β€” nothing we infer outranks a person who looked at the work; + * 2. the process died β€” infra, and invisible to the agent, so it can never be self-reported; + * 3. zero tool calls β€” the run did nothing. On the live corpus **all 56** such runs also reported nothing, + * which is why this sits above the report: there is no report to prefer, and "did nothing" read as + * "unknown" is how a broken automation stayed invisible (one agent produced 18 empty runs in 30 days); + * 4. the agent's own report β€” trusted where it exists, just no longer alone; + * 5. a task closed while this run held it β€” the strongest positive evidence the OS observes itself; + * 6. another run picked the same task up afterwards β€” this one didn't finish it (`incomplete`, not + * `failure`: not finishing and failing are different, and the old metric conflated them); + * 7. a person spawned it and closed it β€” `abandoned`, leaves the denominator; + * 8. otherwise `unknown`. Honest, and the residual we are trying to shrink. + */ +export function deriveRunOutcomes(os: AgentOS, opts: { since?: number; until?: number } = {}): RunOutcome[] { + const db = os.db; + const until = opts.until ?? Date.now(); + const since = opts.since ?? until - 30 * DAY; + + const rows = db + .prepare( + 'SELECT id, agent, status, spawned_by, claude_session_id, rating, outcome, tool_calls, active_ms, created_at, ' + + 'COALESCE(updated_at, created_at) AS ended FROM term_sessions ' + + "WHERE created_at >= ? AND created_at < ? AND status != 'running'", + ) + .all(since, until); + + // Which runs closed a task, by run id β€” `task.completed` is the audit the dispatcher writes when an + // agent closes its own loop, and unlike a `task_events` status row it carries the run that did it + // (5 of 350 status rows have a session id; attribution has to come from the audit log). + const closers = new Set(); + for (const r of db.prepare("SELECT DISTINCT run_id FROM audit_events WHERE type = 'task.completed' AND run_id IS NOT NULL AND ts >= ?").all<{ run_id: string }>(since)) { + closers.add(r.run_id); + } + + // Task β†’ the closing audit's timestamp, so a run can claim a close only if the task shut while it held + // it. Without the time check, every earlier attempt on a since-completed task would read as a success. + const taskDoneAt = new Map(); + for (const r of db.prepare("SELECT id, status, updated_at FROM tasks WHERE status = 'done'").all<{ id: string; status: string; updated_at: number }>()) { + taskDoneAt.set(r.id, r.updated_at); + } + + // Task β†’ the start of the LAST run dispatched for it. A run that ended before that had to be followed + // by another attempt, i.e. it didn't finish the job. + const lastRunFor = new Map(); + for (const row of rows) { + const tid = taskIdOf(row.spawned_by); + if (!tid) continue; + lastRunFor.set(tid, Math.max(lastRunFor.get(tid) ?? 0, row.created_at)); + } + + const out: RunOutcome[] = []; + for (const r of rows) { + if (isChat(r.spawned_by)) continue; // not work with an outcome + const taskId = taskIdOf(r.spawned_by); + const base = { runId: r.id, convoId: r.claude_session_id || r.id, agent: r.agent, at: r.created_at, taskId }; + + let verdict: RunVerdict, basis: OutcomeBasis; + const doneAt = taskId ? taskDoneAt.get(taskId) : undefined; + // 5 minutes of slack: the audit lands as the run tears down, so an exact `<= ended` drops real closes. + const closedByThisRun = doneAt != null && doneAt >= r.created_at && doneAt <= r.ended + 5 * 60_000; + + const unattended = !isHumanSpawned(r.spawned_by); + + if (r.rating === 'up') { verdict = 'success'; basis = 'human-rating'; } + else if (r.rating === 'down') { verdict = 'failure'; basis = 'human-rating'; } + // A person's own interactive session: they typed, they read, they closed the window. The OS observes + // no verdict β€” closing a pane after getting your answer looks identical to walking away, and the blind + // labels called four such runs successes that v1 scored `abandoned`. Same posture as chat: out of the + // denominator, not counted as not-success. + else if (!unattended) { verdict = 'abandoned'; basis = 'human-session'; } + else if (r.status === 'crashed') { verdict = 'failure'; basis = 'crashed'; } + // Above the report and above no-tool-calls: there is no report (0 of 44 such runs ever wrote one), and + // "died on the first API call" is a truer account than "made no tool calls". + else if ((r.spawned_by ?? '').startsWith('automation:') && r.active_ms != null && r.active_ms < DIED_EARLY_MS && (r.tool_calls ?? 0) > 0) { verdict = 'failure'; basis = 'died-early'; } + else if (r.tool_calls === 0) { verdict = 'noop'; basis = 'no-tool-calls'; } + else if (r.outcome === 'success' || r.outcome === 'completed') { verdict = 'success'; basis = 'reported'; } + else if (r.outcome === 'partial' || r.outcome === 'progressed') { verdict = 'partial'; basis = 'reported'; } + else if (r.outcome === 'failure' || r.outcome === 'blocked') { verdict = 'failure'; basis = 'reported'; } + // Killed mid-flight outranks a task close: v1 scored two runs `success` because the task they held + // shut inside their window, when the transcripts show both were cut off (session limit) and something + // else closed the task. A run that was stopped did not finish. + else if (r.status === 'stopped') { verdict = 'incomplete'; basis = 'stopped-midway'; } + else if (closers.has(r.id) || closedByThisRun) { verdict = 'success'; basis = 'task-completed'; } + else if (taskId && (lastRunFor.get(taskId) ?? 0) > r.created_at) { verdict = 'incomplete'; basis = 'task-retried'; } + else { verdict = 'unknown'; basis = 'no-evidence'; } + + out.push({ ...base, verdict, basis }); + } + return out; +} + +/** Worst-wins ordering when a conversation's runs disagree. A resume that ends clean does not erase the + * crash before it, but a `noop` run inside a conversation that also did real work is just a quiet turn β€” + * hence `noop` ranks below `unknown` here and only survives when it is all there is. */ +const SEVERITY: Record = { failure: 6, incomplete: 5, partial: 4, unknown: 3, noop: 2, success: 1, abandoned: 0 }; + +/** + * Fold runs into conversations (`claude_session_id`) β€” one piece of work, however many resumes it took. + * The verdict is the most severe run verdict, EXCEPT that any explicit success (reported or task-closing) + * inside the conversation wins over `incomplete`/`noop`/`unknown`: a hand-off that needed three pokes and + * then finished is a success that took three pokes, not three-quarters of a failure. + */ +export function foldConversations(runs: RunOutcome[]): ConversationOutcome[] { + const byConvo = new Map(); + for (const r of runs) { + const a = byConvo.get(r.convoId) ?? []; + a.push(r); + byConvo.set(r.convoId, a); + } + const out: ConversationOutcome[] = []; + for (const [convoId, list] of byConvo) { + list.sort((a, b) => a.at - b.at); + const succeeded = list.find((r) => r.verdict === 'success'); + const hardFail = list.find((r) => r.verdict === 'failure' || r.verdict === 'partial'); + const worst = list.reduce((acc, r) => (SEVERITY[r.verdict] > SEVERITY[acc.verdict] ? r : acc), list[0]); + const pick = hardFail ?? succeeded ?? worst; + out.push({ convoId, agent: pick.agent, verdict: pick.verdict, basis: pick.basis, runs: list.length, at: list[0].at }); + } + return out.sort((a, b) => b.at - a.at); +} + +/** Roll conversations up into the headline counts. `abandoned` is excluded from `scorable` by + * construction, and `successRate` is withheld below `MIN_SAMPLE` and always accompanied by + * `unknownShare` β€” a success rate sitting next to a 40% unknown share is how we got here. */ +export function summarize(convos: ConversationOutcome[]): OutcomeSummary { + const n = (v: RunVerdict) => convos.filter((c) => c.verdict === v).length; + const abandoned = n('abandoned'); + const scorable = convos.length - abandoned; + const success = n('success'); + return { + scorable, success, partial: n('partial'), incomplete: n('incomplete'), noop: n('noop'), + failure: n('failure'), abandoned, unknown: n('unknown'), + successRate: scorable >= MIN_SAMPLE ? Math.round((success / scorable) * 100) : null, + unknownShare: scorable ? Math.round((n('unknown') / scorable) * 100) : null, + }; +} + +/** Convenience: window β†’ summary + the folded conversations behind it. */ +export function measureOutcomes(os: AgentOS, opts: { since?: number; until?: number } = {}): { summary: OutcomeSummary; conversations: ConversationOutcome[] } { + const conversations = foldConversations(deriveRunOutcomes(os, opts)); + return { summary: summarize(conversations), conversations }; +}