From c291976692f20dede5bc2d5a217b6c43ff3da97d Mon Sep 17 00:00:00 2001 From: justcodebruh Date: Wed, 29 Jul 2026 17:51:43 -0400 Subject: [PATCH 1/4] fix: Capture systemPrompt in Strands agent model span input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TypeScript Strands plugin read event.agent.messages to build the model span input but never read event.agent.systemPrompt, so the system prompt was silently absent from every model span — even though it was set on the agent and visible in other observability tools. Add buildModelSpanInput() which prepends a { role: "system", content } message to the input array when systemPrompt is present, matching the behavior already documented for the Python integration. Also add systemPrompt to the StrandsAgent vendor type. Co-Authored-By: Claude Sonnet 4.6 --- .../plugins/strands-agent-sdk-plugin.ts | 30 +++++++++++++++---- js/src/vendor-sdk-types/strands-agent-sdk.ts | 1 + 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts index 92c467db4..9d873bbf9 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts +++ b/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts @@ -472,6 +472,29 @@ function handleMultiAgentStreamEvent( } } +function buildModelSpanInput( + event: StrandsBeforeModelCallEvent, + attachmentCache: StrandsAttachmentCache, +): unknown { + const systemPrompt = + typeof event.agent?.systemPrompt === "string" && + event.agent.systemPrompt.length > 0 + ? event.agent.systemPrompt + : undefined; + const processedMessages = Array.isArray(event.agent?.messages) + ? processStrandsInputAttachments(event.agent.messages, attachmentCache) + : undefined; + + if (!systemPrompt) { + return processedMessages; + } + + const systemMessage = { role: "system", content: systemPrompt }; + return Array.isArray(processedMessages) + ? [systemMessage, ...processedMessages] + : [systemMessage]; +} + function startModelSpan( state: AgentStreamState, event: StrandsBeforeModelCallEvent, @@ -494,12 +517,7 @@ function startModelSpan( withSpanInstrumentationName( { event: { - input: Array.isArray(event.agent?.messages) - ? processStrandsInputAttachments( - event.agent.messages, - state.attachmentCache, - ) - : undefined, + input: buildModelSpanInput(event, state.attachmentCache), metadata, }, name: formatModelSpanName(model), diff --git a/js/src/vendor-sdk-types/strands-agent-sdk.ts b/js/src/vendor-sdk-types/strands-agent-sdk.ts index 16eaa6e87..30a7e6697 100644 --- a/js/src/vendor-sdk-types/strands-agent-sdk.ts +++ b/js/src/vendor-sdk-types/strands-agent-sdk.ts @@ -28,6 +28,7 @@ export interface StrandsAgent { readonly description?: string; readonly model?: StrandsModel; readonly messages?: StrandsMessage[]; + readonly systemPrompt?: string; invoke( args: StrandsInvokeArgs, options?: StrandsInvokeOptions, From 6abf04ff30b853cad76d3a4759aa6f5f992a0937 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:14:57 +0000 Subject: [PATCH 2/4] e2e test --- ...ands-agent-sdk-v1-auto-hook.span-tree.json | 200 ++++++++++++++++++ ...rands-agent-sdk-v1-auto-hook.span-tree.txt | 200 ++++++++++++++++++ ...ent-sdk-v1-latest-auto-hook.span-tree.json | 200 ++++++++++++++++++ ...gent-sdk-v1-latest-auto-hook.span-tree.txt | 200 ++++++++++++++++++ ...agent-sdk-v1-latest-wrapped.span-tree.json | 200 ++++++++++++++++++ ...-agent-sdk-v1-latest-wrapped.span-tree.txt | 200 ++++++++++++++++++ ...trands-agent-sdk-v1-wrapped.span-tree.json | 200 ++++++++++++++++++ ...strands-agent-sdk-v1-wrapped.span-tree.txt | 200 ++++++++++++++++++ .../assertions.ts | 52 ++++- 9 files changed, 1649 insertions(+), 3 deletions(-) diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.json index e9d2b6a9b..a46268727 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.json @@ -15,6 +15,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -64,6 +78,61 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + }, + { + "content": [ + { + "toolUse": { + "input": { + "city": "Vienna" + }, + "name": "lookup_weather", + "toolUseId": "" + } + } + ], + "metadata": { + "usage": { + "inputTokens": 86, + "outputTokens": 15, + "totalTokens": 101 + } + }, + "role": "assistant" + }, + { + "content": [ + { + "toolResult": { + "content": [ + { + "json": { + "city": "Vienna", + "forecast": "sunny", + "marker": "STRANDS_TOOL_OK" + } + } + ], + "status": "success", + "toolUseId": "" + } + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -125,6 +194,34 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_DOCUMENT_OK.", + "role": "system" + }, + { + "content": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -204,6 +301,33 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_IMAGE_OK.", + "role": "system" + }, + { + "content": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -282,6 +406,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -351,6 +489,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly GRAPH_RESEARCH_DONE.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -425,6 +577,26 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_GRAPH_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + }, + { + "text": "[node: graph-researcher]" + }, + { + "text": "GRAPH_RESEARCH_DONE" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -545,6 +717,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -611,6 +797,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.txt index b142b80f2..89ceb214f 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-auto-hook.span-tree.txt @@ -39,6 +39,20 @@ span_tree: │ "tokens": 234 │ } │ ├── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -80,6 +94,61 @@ span_tree: │ │ "duration": "" │ │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "toolUse": { + │ "input": { + │ "city": "Vienna" + │ }, + │ "name": "lookup_weather", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 86, + │ "outputTokens": 15, + │ "totalTokens": 101 + │ } + │ }, + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "toolResult": { + │ "content": [ + │ { + │ "json": { + │ "city": "Vienna", + │ "forecast": "sunny", + │ "marker": "STRANDS_TOOL_OK" + │ } + │ } + │ ], + │ "status": "success", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -146,6 +215,34 @@ span_tree: │ "tokens": 252 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_DOCUMENT_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -211,6 +308,33 @@ span_tree: │ "tokens": 8530 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_IMAGE_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -259,6 +383,20 @@ span_tree: │ "tokens": 33 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -346,6 +484,20 @@ span_tree: │ │ "tokens": 39 │ │ } │ │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply exactly GRAPH_RESEARCH_DONE.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -416,6 +568,26 @@ span_tree: │ "tokens": 50 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_GRAPH_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ }, + │ { + │ "text": "[node: graph-researcher]" + │ }, + │ { + │ "text": "GRAPH_RESEARCH_DONE" + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -495,6 +667,20 @@ span_tree: │ "tokens": 229 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -549,6 +735,20 @@ span_tree: "tokens": 212 } └── Strands model: gpt-4o-mini-2024-07-18 [llm] + input: [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ] metadata: { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.json index e9d2b6a9b..a46268727 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.json @@ -15,6 +15,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -64,6 +78,61 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + }, + { + "content": [ + { + "toolUse": { + "input": { + "city": "Vienna" + }, + "name": "lookup_weather", + "toolUseId": "" + } + } + ], + "metadata": { + "usage": { + "inputTokens": 86, + "outputTokens": 15, + "totalTokens": 101 + } + }, + "role": "assistant" + }, + { + "content": [ + { + "toolResult": { + "content": [ + { + "json": { + "city": "Vienna", + "forecast": "sunny", + "marker": "STRANDS_TOOL_OK" + } + } + ], + "status": "success", + "toolUseId": "" + } + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -125,6 +194,34 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_DOCUMENT_OK.", + "role": "system" + }, + { + "content": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -204,6 +301,33 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_IMAGE_OK.", + "role": "system" + }, + { + "content": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -282,6 +406,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -351,6 +489,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly GRAPH_RESEARCH_DONE.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -425,6 +577,26 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_GRAPH_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + }, + { + "text": "[node: graph-researcher]" + }, + { + "text": "GRAPH_RESEARCH_DONE" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -545,6 +717,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -611,6 +797,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.txt index b142b80f2..89ceb214f 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-auto-hook.span-tree.txt @@ -39,6 +39,20 @@ span_tree: │ "tokens": 234 │ } │ ├── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -80,6 +94,61 @@ span_tree: │ │ "duration": "" │ │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "toolUse": { + │ "input": { + │ "city": "Vienna" + │ }, + │ "name": "lookup_weather", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 86, + │ "outputTokens": 15, + │ "totalTokens": 101 + │ } + │ }, + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "toolResult": { + │ "content": [ + │ { + │ "json": { + │ "city": "Vienna", + │ "forecast": "sunny", + │ "marker": "STRANDS_TOOL_OK" + │ } + │ } + │ ], + │ "status": "success", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -146,6 +215,34 @@ span_tree: │ "tokens": 252 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_DOCUMENT_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -211,6 +308,33 @@ span_tree: │ "tokens": 8530 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_IMAGE_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -259,6 +383,20 @@ span_tree: │ "tokens": 33 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -346,6 +484,20 @@ span_tree: │ │ "tokens": 39 │ │ } │ │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply exactly GRAPH_RESEARCH_DONE.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -416,6 +568,26 @@ span_tree: │ "tokens": 50 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_GRAPH_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ }, + │ { + │ "text": "[node: graph-researcher]" + │ }, + │ { + │ "text": "GRAPH_RESEARCH_DONE" + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -495,6 +667,20 @@ span_tree: │ "tokens": 229 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -549,6 +735,20 @@ span_tree: "tokens": 212 } └── Strands model: gpt-4o-mini-2024-07-18 [llm] + input: [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ] metadata: { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.json index 18b076066..bd7795797 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.json @@ -15,6 +15,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -64,6 +78,61 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + }, + { + "content": [ + { + "toolUse": { + "input": { + "city": "Vienna" + }, + "name": "lookup_weather", + "toolUseId": "" + } + } + ], + "metadata": { + "usage": { + "inputTokens": 86, + "outputTokens": 15, + "totalTokens": 101 + } + }, + "role": "assistant" + }, + { + "content": [ + { + "toolResult": { + "content": [ + { + "json": { + "city": "Vienna", + "forecast": "sunny", + "marker": "STRANDS_TOOL_OK" + } + } + ], + "status": "success", + "toolUseId": "" + } + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -125,6 +194,34 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_DOCUMENT_OK.", + "role": "system" + }, + { + "content": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -204,6 +301,33 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_IMAGE_OK.", + "role": "system" + }, + { + "content": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -282,6 +406,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -351,6 +489,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly GRAPH_RESEARCH_DONE.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -425,6 +577,26 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_GRAPH_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + }, + { + "text": "[node: graph-researcher]" + }, + { + "text": "GRAPH_RESEARCH_DONE" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -631,6 +803,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -697,6 +883,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.txt index ae7106d36..ed1a91c19 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-latest-wrapped.span-tree.txt @@ -39,6 +39,20 @@ span_tree: │ "tokens": 234 │ } │ ├── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -80,6 +94,61 @@ span_tree: │ │ "duration": "" │ │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "toolUse": { + │ "input": { + │ "city": "Vienna" + │ }, + │ "name": "lookup_weather", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 86, + │ "outputTokens": 15, + │ "totalTokens": 101 + │ } + │ }, + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "toolResult": { + │ "content": [ + │ { + │ "json": { + │ "city": "Vienna", + │ "forecast": "sunny", + │ "marker": "STRANDS_TOOL_OK" + │ } + │ } + │ ], + │ "status": "success", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -146,6 +215,34 @@ span_tree: │ "tokens": 252 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_DOCUMENT_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -211,6 +308,33 @@ span_tree: │ "tokens": 8530 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_IMAGE_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -259,6 +383,20 @@ span_tree: │ "tokens": 33 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -346,6 +484,20 @@ span_tree: │ │ "tokens": 39 │ │ } │ │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply exactly GRAPH_RESEARCH_DONE.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -416,6 +568,26 @@ span_tree: │ "tokens": 50 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_GRAPH_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ }, + │ { + │ "text": "[node: graph-researcher]" + │ }, + │ { + │ "text": "GRAPH_RESEARCH_DONE" + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -560,6 +732,20 @@ span_tree: │ "tokens": 229 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -614,6 +800,20 @@ span_tree: "tokens": 212 } └── Strands model: gpt-4o-mini-2024-07-18 [llm] + input: [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ] metadata: { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.json b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.json index 18b076066..bd7795797 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.json +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.json @@ -15,6 +15,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -64,6 +78,61 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + } + ], + "role": "user" + }, + { + "content": [ + { + "toolUse": { + "input": { + "city": "Vienna" + }, + "name": "lookup_weather", + "toolUseId": "" + } + } + ], + "metadata": { + "usage": { + "inputTokens": 86, + "outputTokens": 15, + "totalTokens": 101 + } + }, + "role": "assistant" + }, + { + "content": [ + { + "toolResult": { + "content": [ + { + "json": { + "city": "Vienna", + "forecast": "sunny", + "marker": "STRANDS_TOOL_OK" + } + } + ], + "status": "success", + "toolUseId": "" + } + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -125,6 +194,34 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_DOCUMENT_OK.", + "role": "system" + }, + { + "content": [ + { + "document": { + "format": "pdf", + "name": "strands-regression.pdf", + "source": { + "bytes": { + "content_type": "application/pdf", + "filename": "strands-regression.pdf", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_DOCUMENT_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -204,6 +301,33 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_IMAGE_OK.", + "role": "system" + }, + { + "content": [ + { + "image": { + "format": "png", + "source": { + "bytes": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + } + } + } + }, + { + "text": "Reply exactly STRANDS_IMAGE_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -282,6 +406,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -351,6 +489,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly GRAPH_RESEARCH_DONE.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -425,6 +577,26 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Reply exactly STRANDS_GRAPH_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + }, + { + "text": "[node: graph-researcher]" + }, + { + "text": "GRAPH_RESEARCH_DONE" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -631,6 +803,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", @@ -697,6 +883,20 @@ "name": "Strands model: gpt-4o-mini-2024-07-18", "type": "llm", "children": [], + "input": [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ], "metadata": { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.txt b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.txt index ae7106d36..ed1a91c19 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.txt +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/__snapshots__/strands-agent-sdk-v1-wrapped.span-tree.txt @@ -39,6 +39,20 @@ span_tree: │ "tokens": 234 │ } │ ├── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -80,6 +94,61 @@ span_tree: │ │ "duration": "" │ │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use lookup_weather for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK." + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": [ + │ { + │ "toolUse": { + │ "input": { + │ "city": "Vienna" + │ }, + │ "name": "lookup_weather", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "metadata": { + │ "usage": { + │ "inputTokens": 86, + │ "outputTokens": 15, + │ "totalTokens": 101 + │ } + │ }, + │ "role": "assistant" + │ }, + │ { + │ "content": [ + │ { + │ "toolResult": { + │ "content": [ + │ { + │ "json": { + │ "city": "Vienna", + │ "forecast": "sunny", + │ "marker": "STRANDS_TOOL_OK" + │ } + │ } + │ ], + │ "status": "success", + │ "toolUseId": "" + │ } + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -146,6 +215,34 @@ span_tree: │ "tokens": 252 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_DOCUMENT_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "document": { + │ "format": "pdf", + │ "name": "strands-regression.pdf", + │ "source": { + │ "bytes": { + │ "content_type": "application/pdf", + │ "filename": "strands-regression.pdf", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_DOCUMENT_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -211,6 +308,33 @@ span_tree: │ "tokens": 8530 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_IMAGE_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "image": { + │ "format": "png", + │ "source": { + │ "bytes": { + │ "content_type": "image/png", + │ "filename": "image.png", + │ "key": "", + │ "type": "braintrust_attachment" + │ } + │ } + │ } + │ }, + │ { + │ "text": "Reply exactly STRANDS_IMAGE_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -259,6 +383,20 @@ span_tree: │ "tokens": 33 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_AGENT_STREAM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Reply exactly STRANDS_AGENT_STREAM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -346,6 +484,20 @@ span_tree: │ │ "tokens": 39 │ │ } │ │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ │ input: [ + │ │ { + │ │ "content": "Reply exactly GRAPH_RESEARCH_DONE.", + │ │ "role": "system" + │ │ }, + │ │ { + │ │ "content": [ + │ │ { + │ │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ │ } + │ │ ], + │ │ "role": "user" + │ │ } + │ │ ] │ │ metadata: { │ │ "model": "gpt-4o-mini-2024-07-18", │ │ "provider": "openai", @@ -416,6 +568,26 @@ span_tree: │ "tokens": 50 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Reply exactly STRANDS_GRAPH_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Have the first node finish, then have the writer answer exactly STRANDS_GRAPH_OK." + │ }, + │ { + │ "text": "[node: graph-researcher]" + │ }, + │ { + │ "text": "GRAPH_RESEARCH_DONE" + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -560,6 +732,20 @@ span_tree: │ "tokens": 229 │ } │ └── Strands model: gpt-4o-mini-2024-07-18 [llm] + │ input: [ + │ { + │ "content": "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Route to the finisher, then finish exactly STRANDS_SWARM_OK." + │ } + │ ], + │ "role": "user" + │ } + │ ] │ metadata: { │ "model": "gpt-4o-mini-2024-07-18", │ "provider": "openai", @@ -614,6 +800,20 @@ span_tree: "tokens": 212 } └── Strands model: gpt-4o-mini-2024-07-18 [llm] + input: [ + { + "content": "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + "role": "system" + }, + { + "content": [ + { + "text": "finish with STRANDS_SWARM_OK" + } + ], + "role": "user" + } + ] metadata: { "model": "gpt-4o-mini-2024-07-18", "provider": "openai", diff --git a/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts b/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts index 3d89a3945..475e77297 100644 --- a/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts +++ b/e2e/scenarios/strands-agent-sdk-instrumentation/assertions.ts @@ -135,9 +135,7 @@ function snapshotFields(event: CapturedLogEvent) { return { span_attributes: fields.span_attributes, input: - event.span.type === "llm" || event.span.type === undefined - ? undefined - : normalizeValue(fields.input), + event.span.type === undefined ? undefined : normalizeValue(fields.input), output: event.span.type === "llm" || event.span.type === undefined ? undefined @@ -341,6 +339,54 @@ export function defineStrandsAgentSDKInstrumentationAssertions(options: { expect(streamModels.length).toBeGreaterThanOrEqual(1); }); + test( + "captures the system prompt in every model span input", + testConfig, + () => { + expect(setupError).toBeUndefined(); + const systemPromptsByAgentName = new Map([ + [ + "Agent: weather-agent", + "You are a deterministic test agent. Always call lookup_weather exactly once for Vienna, then answer exactly STRANDS_AGENT_TOOL_OK.", + ], + ["Agent: document-agent", "Reply exactly STRANDS_DOCUMENT_OK."], + ["Agent: image-agent", "Reply exactly STRANDS_IMAGE_OK."], + ["Agent: stream-agent", "Reply exactly STRANDS_AGENT_STREAM_OK."], + ["Agent: graph-researcher", "Reply exactly GRAPH_RESEARCH_DONE."], + ["Agent: graph-writer", "Reply exactly STRANDS_GRAPH_OK."], + [ + "Agent: swarm-router", + "Use the structured output schema. Hand off to swarm-finisher with message finish with STRANDS_SWARM_OK.", + ], + [ + "Agent: swarm-finisher", + "Use the structured output schema. Do not hand off. Set the final message to exactly STRANDS_SWARM_OK.", + ], + ]); + + for (const [agentName, systemPrompt] of systemPromptsByAgentName) { + const agents = latestEventsPerSpan(events).filter( + (event) => event.span.name === agentName, + ); + expect(agents.length, agentName).toBeGreaterThanOrEqual(1); + + for (const agent of agents) { + const modelSpans = findChildSpans( + events, + `Strands model: ${MODEL_NAME}`, + agent.span.id, + ); + expect(modelSpans.length, agentName).toBeGreaterThanOrEqual(1); + for (const modelSpan of modelSpans) { + expect( + Array.isArray(modelSpan.input) ? modelSpan.input[0] : undefined, + ).toEqual({ role: "system", content: systemPrompt }); + } + } + } + }, + ); + test("captures document bytes as attachments", testConfig, () => { expect(setupError).toBeUndefined(); const documentOperation = findLatestSpan( From 86db908229fcf67261e0fdc4f1b1d8ceac5697f0 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:23:43 +0000 Subject: [PATCH 3/4] Update PR #2311 --- .changeset/capture-strands-system-prompt.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/capture-strands-system-prompt.md diff --git a/.changeset/capture-strands-system-prompt.md b/.changeset/capture-strands-system-prompt.md new file mode 100644 index 000000000..a59b99478 --- /dev/null +++ b/.changeset/capture-strands-system-prompt.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix: Capture systemPrompt in Strands agent model span input From 9d86cb37731452cba8c511c72c9fa4ff26fc9835 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:37:46 +0000 Subject: [PATCH 4/4] Update PR #2311 --- .agents/skills/e2e-tests/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.agents/skills/e2e-tests/SKILL.md b/.agents/skills/e2e-tests/SKILL.md index ede026517..071d683d2 100644 --- a/.agents/skills/e2e-tests/SKILL.md +++ b/.agents/skills/e2e-tests/SKILL.md @@ -61,6 +61,7 @@ Cassettes mock provider HTTP responses (OpenAI, Anthropic, ...) so external-prov - Keep reusable logic in `e2e/helpers/`. Keep one-off fixtures and scenario-specific files inside the scenario directory. - Snapshot stable contracts, not raw noise. Use `normalizeForSnapshot(...)` before inline snapshots and `formatJsonFileSnapshot(...)` plus file snapshots for larger payloads or version matrices. - For span-tree snapshots, use `matchSpanTreeSnapshot(...)`. It writes and asserts paired `.span-tree.json` and `.span-tree.txt` files from the same normalized span tree. The JSON file is the structural contract that is easiest to parse mechanically; the TXT file is the ASCII tree that is easiest to review by eye. Keep them in sync by updating both through the e2e update/record commands, and never hand-edit only one side of the pair. +- For tracing-instrumentation span-tree snapshots, always capture `input`, `output`, `metadata`, `context`, `error`, and `metrics`. Normalize volatile values within these fields, but do not omit a field or selectively filter its semantic contents to reduce snapshot size or noise. - When a scenario family already has `assertions.ts`, keep version- or provider-specific test setup in `scenario.test.ts` and reuse the shared assertions file. - Keep the CI e2e summary up to date. If a scenario version matrix or `variantKey` changes, update `e2e/config/pr-comment-scenarios.json` in the same change and follow the established pattern used by other versioned scenarios: one summary row per version, not separate wrapped/auto rows unless that pattern already exists for the scenario family. - Run new or updated scenarios three times in a row before considering snapshots stable.