diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c2d3cc1..7dcfab99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.145.3] — 2026-08-12 + +### Fixed + +- Accept and preserve a provider-qualified snapshot when an optimizer callback reports the requested model. +- Reject true model substitution and conflicting response and receipt identities. + +--- + ## [0.145.2] — 2026-08-12 ### Changed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 8a04f0d3..e3be8424 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.145.2" +version = "0.145.3" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 308bdf82..a152391c 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.145.2" + __version__ = "0.145.3" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 024a85ff..976c20cd 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.145.2" +version = "0.145.3" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index a16c4ca7..5d451533 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.145.2", + "version": "0.145.3", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index ea6e5adc..5e741358 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '653ca1243a688124c0e879e0c59fce8e869c97a31ab81d2b0f56fcbfa5675de4' + '1f2e3c30a2210ea5f646c822afdd0a5dd4060bfde5f3660c0ccbf1342d37dbec' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the @@ -136,7 +136,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '81709d08fb59b31296fed615bddcc1fa5cef1dc2c09579f5002d39e29d8c0315' + '4a960df94935a8177be12a64d20083510ca4afbfa050533a9356447f39c6f846' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/campaign/external-optimizer-model-proxy.ts b/src/campaign/external-optimizer-model-proxy.ts index 20ba9728..03af49ce 100644 --- a/src/campaign/external-optimizer-model-proxy.ts +++ b/src/campaign/external-optimizer-model-proxy.ts @@ -8,6 +8,7 @@ import type { CustomTokenPricing, } from '../cost-ledger' import { costForTokenPricing } from '../cost-ledger' +import { assertServedModel, ModelSubstitutionError } from '../integrity/served-model' import { canonicalJson } from '../verdict-cache' import { assertExternalOptimizerModelBudget, @@ -345,7 +346,8 @@ async function handleModelProxyRequest(args: { : paid.error instanceof ProviderResponseTooLargeError || paid.error instanceof MissingModelExecutionError || paid.error instanceof ModelExecutionPersistenceError || - paid.error instanceof OwnerModelContractError + paid.error instanceof OwnerModelContractError || + paid.error instanceof ModelSubstitutionError ? 502 : 429, { error: paid.error.message }, @@ -539,11 +541,9 @@ function snapshotModelReceipt(value: CostReceiptInput, expectedModel: string): C `optimizer model callback returned an invalid receipt: ${toErrorMessage(error)}`, ) } - if (snapshot.model !== expectedModel) { - throw new OwnerModelContractError( - `optimizer model callback receipt used '${snapshot.model}' instead of '${expectedModel}'`, - ) - } + assertServedModel(expectedModel, snapshot.model, { + context: 'optimizer model callback receipt', + }) return snapshot } @@ -564,6 +564,11 @@ function assertResponseUsageMatchesReceipt( response: ChatResponse, receipt: CostReceiptInput, ): void { + if (response.model !== receipt.model) { + throw new OwnerModelContractError( + 'optimizer model response and execution receipt disagree about the served model', + ) + } if (response.usage.captured === false || receipt.usageUnknown === true) { if (response.usage.captured !== false || receipt.usageUnknown !== true) { throw new OwnerModelContractError( @@ -936,11 +941,9 @@ function snapshotChatResponse(value: unknown, expectedModel: string): ChatRespon if (typeof value.content !== 'string') { throw new OwnerModelContractError('optimizer model callback response content must be a string') } - if (value.model !== expectedModel) { - throw new OwnerModelContractError( - `optimizer model callback response used '${String(value.model)}' instead of '${expectedModel}'`, - ) - } + assertServedModel(expectedModel, typeof value.model === 'string' ? value.model : null, { + context: 'optimizer model callback response', + }) if (!isNonnegativeFiniteNumber(value.durationMs)) { throw new OwnerModelContractError( 'optimizer model callback response durationMs must be finite and non-negative', diff --git a/tests/campaign/external-optimizer-process.test.ts b/tests/campaign/external-optimizer-process.test.ts index 59c63414..fc4611ca 100644 --- a/tests/campaign/external-optimizer-process.test.ts +++ b/tests/campaign/external-optimizer-process.test.ts @@ -857,6 +857,149 @@ describe('external optimizer model proxy', () => { } }) + it('accepts and preserves a provider-qualified snapshot of the requested model', async () => { + const servedModel = + 'deepseek/deepseek-v4-flash@fp_a18b46594c_prod0820_fp8_kvcache_20260402' + const ledger = new CostLedger() + const proxy = await startRuntimeOwnedModelProxy({ + callRef: 'runtime-profile:qualified-snapshot', + call: async () => ({ + succeeded: true, + response: { + content: 'revised', + usage: { promptTokens: 7, completionTokens: 3, totalTokens: 10 }, + costUsd: null, + model: servedModel, + durationMs: 1, + finishReason: 'stop', + raw: { owner: 'runtime-profile' }, + }, + receipt: { + model: servedModel, + inputTokens: 7, + outputTokens: 3, + costUnknown: true, + }, + execution: { kind: 'runtime-profile-call', model: servedModel }, + }), + recordExecution: () => {}, + model: 'deepseek-v4-flash', + budget: modelBudget({ maxRequests: 1, maxOutputTokensPerRequest: 3 }), + costLedger: ledger, + phase: 'optimizer', + actor: 'official-library', + }) + + try { + const response = await postModel(proxy, { + model: 'deepseek-v4-flash', + messages: [], + max_tokens: 3, + }) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ model: servedModel }) + expect(ledger.list()).toEqual([ + expect.objectContaining({ model: servedModel, inputTokens: 7, outputTokens: 3 }), + ]) + proxy.assertExecutionComplete() + } finally { + await proxy.close() + } + }) + + it('rejects a provider-qualified snapshot of a different model', async () => { + const servedModel = 'deepseek/deepseek-v3@fp_other' + const proxy = await startRuntimeOwnedModelProxy({ + callRef: 'runtime-profile:substituted-snapshot', + call: async () => ({ + succeeded: true, + response: { + content: 'wrong model', + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, + costUsd: null, + model: servedModel, + durationMs: 1, + finishReason: 'stop', + raw: { owner: 'runtime-profile' }, + }, + receipt: { + model: servedModel, + inputTokens: 1, + outputTokens: 1, + costUnknown: true, + }, + execution: { kind: 'runtime-profile-call', model: servedModel }, + }), + recordExecution: () => {}, + model: 'deepseek-v4-flash', + budget: modelBudget({ maxRequests: 1, maxOutputTokensPerRequest: 1 }), + costLedger: new CostLedger(), + phase: 'optimizer', + actor: 'official-library', + }) + + try { + const response = await postModel(proxy, { + model: 'deepseek-v4-flash', + messages: [], + max_tokens: 1, + }) + expect(response.status).toBe(502) + expect(await response.text()).toContain('model substitution') + expect(proxy.successfulCompletions()).toBe(0) + } finally { + await proxy.close() + } + }) + + it('rejects different served snapshots in the response and receipt', async () => { + const responseModel = 'deepseek/deepseek-v4-flash@fp_response' + const receiptModel = 'deepseek/deepseek-v4-flash@fp_receipt' + const proxy = await startRuntimeOwnedModelProxy({ + callRef: 'runtime-profile:conflicting-snapshots', + call: async () => ({ + succeeded: true, + response: { + content: 'conflicting evidence', + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, + costUsd: null, + model: responseModel, + durationMs: 1, + finishReason: 'stop', + raw: { owner: 'runtime-profile' }, + }, + receipt: { + model: receiptModel, + inputTokens: 1, + outputTokens: 1, + costUnknown: true, + }, + execution: { kind: 'runtime-profile-call', model: responseModel }, + }), + recordExecution: () => {}, + model: 'deepseek-v4-flash', + budget: modelBudget({ maxRequests: 1, maxOutputTokensPerRequest: 1 }), + costLedger: new CostLedger(), + phase: 'optimizer', + actor: 'official-library', + }) + + try { + const response = await postModel(proxy, { + model: 'deepseek-v4-flash', + messages: [], + max_tokens: 1, + }) + expect(response.status).toBe(502) + expect(await response.text()).toContain( + 'optimizer model response and execution receipt disagree about the served model', + ) + expect(proxy.successfulCompletions()).toBe(0) + } finally { + await proxy.close() + } + }) + it('fails loud when the execution owner rejects without evidence', async () => { const records: unknown[] = [] const proxy = await startRuntimeOwnedModelProxy({