fix(optimizer): preserve served model snapshots

CHANGELOG.md

@@ -8,6 +8,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.145.3] — 2026-08-12
+
+### Fixed
+
+- Accept and preserve a provider-qualified snapshot when an optimizer callback reports the requested model.
+- Reject true model substitution and conflicting response and receipt identities.
+
+---
+
 ## [0.145.2] — 2026-08-12
 
 ### Changed

clients/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agent-eval-rpc"
-version = "0.145.2"
+version = "0.145.3"
 description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval."
 readme = "README.md"
 requires-python = ">=3.10"

clients/python/src/agent_eval_rpc/__init__.py

@@ -53,7 +53,7 @@
 try:
     __version__ = version("agent-eval-rpc")
 except PackageNotFoundError:
-    __version__ = "0.145.2"
+    __version__ = "0.145.3"
 
 __all__ = [
     "Client",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.145.2",
+  "version": "0.145.3",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

src/analyst/benchmark-implementation.ts

@@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
 ])
 
 export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 =
-  '653ca1243a688124c0e879e0c59fce8e869c97a31ab81d2b0f56fcbfa5675de4'
+  '1f2e3c30a2210ea5f646c822afdd0a5dd4060bfde5f3660c0ccbf1342d37dbec'
 
 /** The published benchmark evidence was produced at this package version, by
  * the retired one-shot direct runner, before trace analysts moved to the
@@ -136,7 +136,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 ])
 
 export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 =
-  '81709d08fb59b31296fed615bddcc1fa5cef1dc2c09579f5002d39e29d8c0315'
+  '4a960df94935a8177be12a64d20083510ca4afbfa050533a9356447f39c6f846'
 
 export function analystBenchmarkImplementationDigest() {
   return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256

src/campaign/external-optimizer-model-proxy.ts

@@ -8,6 +8,7 @@ import type {
   CustomTokenPricing,
 } from '../cost-ledger'
 import { costForTokenPricing } from '../cost-ledger'
+import { assertServedModel, ModelSubstitutionError } from '../integrity/served-model'
 import { canonicalJson } from '../verdict-cache'
 import {
   assertExternalOptimizerModelBudget,
@@ -345,7 +346,8 @@ async function handleModelProxyRequest(args: {
             : paid.error instanceof ProviderResponseTooLargeError ||
                 paid.error instanceof MissingModelExecutionError ||
                 paid.error instanceof ModelExecutionPersistenceError ||
-                paid.error instanceof OwnerModelContractError
+                paid.error instanceof OwnerModelContractError ||
+                paid.error instanceof ModelSubstitutionError
               ? 502
               : 429,
           { error: paid.error.message },
@@ -539,11 +541,9 @@ function snapshotModelReceipt(value: CostReceiptInput, expectedModel: string): C
       `optimizer model callback returned an invalid receipt: ${toErrorMessage(error)}`,
     )
   }
-  if (snapshot.model !== expectedModel) {
-    throw new OwnerModelContractError(
-      `optimizer model callback receipt used '${snapshot.model}' instead of '${expectedModel}'`,
-    )
-  }
+  assertServedModel(expectedModel, snapshot.model, {
+    context: 'optimizer model callback receipt',
+  })
   return snapshot
 }
 
@@ -564,6 +564,11 @@ function assertResponseUsageMatchesReceipt(
   response: ChatResponse,
   receipt: CostReceiptInput,
 ): void {
+  if (response.model !== receipt.model) {
+    throw new OwnerModelContractError(
+      'optimizer model response and execution receipt disagree about the served model',
+    )
+  }
   if (response.usage.captured === false || receipt.usageUnknown === true) {
     if (response.usage.captured !== false || receipt.usageUnknown !== true) {
       throw new OwnerModelContractError(
@@ -936,11 +941,9 @@ function snapshotChatResponse(value: unknown, expectedModel: string): ChatRespon
   if (typeof value.content !== 'string') {
     throw new OwnerModelContractError('optimizer model callback response content must be a string')
   }
-  if (value.model !== expectedModel) {
-    throw new OwnerModelContractError(
-      `optimizer model callback response used '${String(value.model)}' instead of '${expectedModel}'`,
-    )
-  }
+  assertServedModel(expectedModel, typeof value.model === 'string' ? value.model : null, {
+    context: 'optimizer model callback response',
+  })
   if (!isNonnegativeFiniteNumber(value.durationMs)) {
     throw new OwnerModelContractError(
       'optimizer model callback response durationMs must be finite and non-negative',

tests/campaign/external-optimizer-process.test.ts

@@ -857,6 +857,149 @@ describe('external optimizer model proxy', () => {
     }
   })
 
+  it('accepts and preserves a provider-qualified snapshot of the requested model', async () => {
+    const servedModel =
+      'deepseek/deepseek-v4-flash@fp_a18b46594c_prod0820_fp8_kvcache_20260402'
+    const ledger = new CostLedger()
+    const proxy = await startRuntimeOwnedModelProxy({
+      callRef: 'runtime-profile:qualified-snapshot',
+      call: async () => ({
+        succeeded: true,
+        response: {
+          content: 'revised',
+          usage: { promptTokens: 7, completionTokens: 3, totalTokens: 10 },
+          costUsd: null,
+          model: servedModel,
+          durationMs: 1,
+          finishReason: 'stop',
+          raw: { owner: 'runtime-profile' },
+        },
+        receipt: {
+          model: servedModel,
+          inputTokens: 7,
+          outputTokens: 3,
+          costUnknown: true,
+        },
+        execution: { kind: 'runtime-profile-call', model: servedModel },
+      }),
+      recordExecution: () => {},
+      model: 'deepseek-v4-flash',
+      budget: modelBudget({ maxRequests: 1, maxOutputTokensPerRequest: 3 }),
+      costLedger: ledger,
+      phase: 'optimizer',
+      actor: 'official-library',
+    })
+
+    try {
+      const response = await postModel(proxy, {
+        model: 'deepseek-v4-flash',
+        messages: [],
+        max_tokens: 3,
+      })
+      expect(response.status).toBe(200)
+      expect(await response.json()).toMatchObject({ model: servedModel })
+      expect(ledger.list()).toEqual([
+        expect.objectContaining({ model: servedModel, inputTokens: 7, outputTokens: 3 }),
+      ])
+      proxy.assertExecutionComplete()
+    } finally {
+      await proxy.close()
+    }
+  })
+
+  it('rejects a provider-qualified snapshot of a different model', async () => {
+    const servedModel = 'deepseek/deepseek-v3@fp_other'
+    const proxy = await startRuntimeOwnedModelProxy({
+      callRef: 'runtime-profile:substituted-snapshot',
+      call: async () => ({
+        succeeded: true,
+        response: {
+          content: 'wrong model',
+          usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+          costUsd: null,
+          model: servedModel,
+          durationMs: 1,
+          finishReason: 'stop',
+          raw: { owner: 'runtime-profile' },
+        },
+        receipt: {
+          model: servedModel,
+          inputTokens: 1,
+          outputTokens: 1,
+          costUnknown: true,
+        },
+        execution: { kind: 'runtime-profile-call', model: servedModel },
+      }),
+      recordExecution: () => {},
+      model: 'deepseek-v4-flash',
+      budget: modelBudget({ maxRequests: 1, maxOutputTokensPerRequest: 1 }),
+      costLedger: new CostLedger(),
+      phase: 'optimizer',
+      actor: 'official-library',
+    })
+
+    try {
+      const response = await postModel(proxy, {
+        model: 'deepseek-v4-flash',
+        messages: [],
+        max_tokens: 1,
+      })
+      expect(response.status).toBe(502)
+      expect(await response.text()).toContain('model substitution')
+      expect(proxy.successfulCompletions()).toBe(0)
+    } finally {
+      await proxy.close()
+    }
+  })
+
+  it('rejects different served snapshots in the response and receipt', async () => {
+    const responseModel = 'deepseek/deepseek-v4-flash@fp_response'
+    const receiptModel = 'deepseek/deepseek-v4-flash@fp_receipt'
+    const proxy = await startRuntimeOwnedModelProxy({
+      callRef: 'runtime-profile:conflicting-snapshots',
+      call: async () => ({
+        succeeded: true,
+        response: {
+          content: 'conflicting evidence',
+          usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+          costUsd: null,
+          model: responseModel,
+          durationMs: 1,
+          finishReason: 'stop',
+          raw: { owner: 'runtime-profile' },
+        },
+        receipt: {
+          model: receiptModel,
+          inputTokens: 1,
+          outputTokens: 1,
+          costUnknown: true,
+        },
+        execution: { kind: 'runtime-profile-call', model: responseModel },
+      }),
+      recordExecution: () => {},
+      model: 'deepseek-v4-flash',
+      budget: modelBudget({ maxRequests: 1, maxOutputTokensPerRequest: 1 }),
+      costLedger: new CostLedger(),
+      phase: 'optimizer',
+      actor: 'official-library',
+    })
+
+    try {
+      const response = await postModel(proxy, {
+        model: 'deepseek-v4-flash',
+        messages: [],
+        max_tokens: 1,
+      })
+      expect(response.status).toBe(502)
+      expect(await response.text()).toContain(
+        'optimizer model response and execution receipt disagree about the served model',
+      )
+      expect(proxy.successfulCompletions()).toBe(0)
+    } finally {
+      await proxy.close()
+    }
+  })
+
   it('fails loud when the execution owner rejects without evidence', async () => {
     const records: unknown[] = []
     const proxy = await startRuntimeOwnedModelProxy({