From b4df352076515521ddb7f5892e5ffb94654a800d Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 11 Dec 2025 00:29:44 +0100 Subject: [PATCH 01/36] feat(proxy): add Anthropic Messages API endpoint for Claude Code compatibility - Add /v1/messages endpoint with Anthropic-format request/response - Support both x-api-key and Bearer token authentication - Implement Anthropic <-> OpenAI format translation for messages, tools, and responses - Add streaming wrapper converting OpenAI SSE to Anthropic SSE events - Handle tool_use blocks with proper stop_reason detection - Fix NoneType iteration bug in tool_calls handling --- src/proxy_app/main.py | 690 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 689 insertions(+), 1 deletion(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 6eddf4e4e..f8eb2bf5b 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -99,7 +99,8 @@ from contextlib import asynccontextmanager from fastapi import FastAPI, Request, HTTPException, Depends from fastapi.middleware.cors import CORSMiddleware - from fastapi.responses import StreamingResponse + from fastapi.responses import StreamingResponse, JSONResponse + import uuid from fastapi.security import APIKeyHeader print(" → Loading core dependencies...") @@ -214,6 +215,112 @@ class EnrichedModelList(BaseModel): data: List[EnrichedModelCard] +# --- Anthropic API Models --- +class AnthropicTextBlock(BaseModel): + """Anthropic text content block.""" + + type: str = "text" + text: str + + +class AnthropicImageSource(BaseModel): + """Anthropic image source for base64 images.""" + + type: str = "base64" + media_type: str + data: str + + +class AnthropicImageBlock(BaseModel): + """Anthropic image content block.""" + + type: str = "image" + source: AnthropicImageSource + + +class AnthropicToolUseBlock(BaseModel): + """Anthropic tool use content block.""" + + type: str = "tool_use" + id: str + name: str + input: dict + + +class AnthropicToolResultBlock(BaseModel): + """Anthropic tool result content block.""" + + type: str = "tool_result" + tool_use_id: str + content: Union[str, List[Any]] + is_error: Optional[bool] = None + + +class AnthropicMessage(BaseModel): + """Anthropic message format.""" + + role: str + content: Union[ + str, + List[ + Union[ + AnthropicTextBlock, + AnthropicImageBlock, + AnthropicToolUseBlock, + AnthropicToolResultBlock, + dict, + ] + ], + ] + + +class AnthropicTool(BaseModel): + """Anthropic tool definition.""" + + name: str + description: Optional[str] = None + input_schema: dict + + +class AnthropicMessagesRequest(BaseModel): + """Anthropic Messages API request format.""" + + model: str + messages: List[AnthropicMessage] + max_tokens: int + system: Optional[Union[str, List[dict]]] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + stop_sequences: Optional[List[str]] = None + stream: Optional[bool] = False + tools: Optional[List[AnthropicTool]] = None + tool_choice: Optional[dict] = None + metadata: Optional[dict] = None + + +class AnthropicUsage(BaseModel): + """Anthropic usage statistics.""" + + input_tokens: int + output_tokens: int + cache_creation_input_tokens: Optional[int] = None + cache_read_input_tokens: Optional[int] = None + + +class AnthropicMessagesResponse(BaseModel): + """Anthropic Messages API response format.""" + + id: str + type: str = "message" + role: str = "assistant" + content: List[Union[AnthropicTextBlock, AnthropicToolUseBlock, dict]] + model: str + stop_reason: Optional[str] = None + stop_sequence: Optional[str] = None + usage: AnthropicUsage + + # Calculate total loading time _elapsed = time.time() - _start_time print( @@ -665,6 +772,433 @@ async def verify_api_key(auth: str = Depends(api_key_header)): return auth +# --- Anthropic API Key Header --- +anthropic_api_key_header = APIKeyHeader(name="x-api-key", auto_error=False) + + +async def verify_anthropic_api_key( + x_api_key: str = Depends(anthropic_api_key_header), + auth: str = Depends(api_key_header), +): + """ + Dependency to verify API key for Anthropic endpoints. + Accepts either x-api-key header (Anthropic style) or Authorization Bearer (OpenAI style). + """ + # Check x-api-key first (Anthropic style) + if x_api_key and x_api_key == PROXY_API_KEY: + return x_api_key + # Fall back to Bearer token (OpenAI style) + if auth and auth == f"Bearer {PROXY_API_KEY}": + return auth + raise HTTPException(status_code=401, detail="Invalid or missing API Key") + + +# --- Anthropic <-> OpenAI Format Translation --- +def anthropic_to_openai_messages( + anthropic_messages: List[dict], system: Optional[Union[str, List[dict]]] = None +) -> List[dict]: + """ + Convert Anthropic message format to OpenAI format. + + Key differences: + - Anthropic: system is a separate field, content can be string or list of blocks + - OpenAI: system is a message with role="system", content is usually string + """ + openai_messages = [] + + # Handle system message + if system: + if isinstance(system, str): + openai_messages.append({"role": "system", "content": system}) + elif isinstance(system, list): + # System can be list of text blocks in Anthropic format + system_text = " ".join( + block.get("text", "") + for block in system + if isinstance(block, dict) and block.get("type") == "text" + ) + if system_text: + openai_messages.append({"role": "system", "content": system_text}) + + for msg in anthropic_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if isinstance(content, str): + openai_messages.append({"role": role, "content": content}) + elif isinstance(content, list): + # Handle content blocks + openai_content = [] + tool_calls = [] + + for block in content: + if isinstance(block, dict): + block_type = block.get("type", "text") + + if block_type == "text": + openai_content.append( + {"type": "text", "text": block.get("text", "")} + ) + elif block_type == "image": + # Convert Anthropic image format to OpenAI + source = block.get("source", {}) + if source.get("type") == "base64": + openai_content.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{source.get('media_type', 'image/png')};base64,{source.get('data', '')}" + }, + } + ) + elif source.get("type") == "url": + openai_content.append( + { + "type": "image_url", + "image_url": {"url": source.get("url", "")}, + } + ) + elif block_type == "tool_use": + # Anthropic tool_use -> OpenAI tool_calls + tool_calls.append( + { + "id": block.get("id", ""), + "type": "function", + "function": { + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }, + } + ) + elif block_type == "tool_result": + # Tool results become separate messages in OpenAI format + tool_content = block.get("content", "") + if isinstance(tool_content, list): + tool_content = " ".join( + b.get("text", "") + for b in tool_content + if isinstance(b, dict) and b.get("type") == "text" + ) + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": str(tool_content), + } + ) + continue # Don't add to current message + + # Build the message + if tool_calls: + # Assistant message with tool calls + msg_dict = {"role": role} + if openai_content: + # If there's text content alongside tool calls + text_parts = [ + c.get("text", "") + for c in openai_content + if c.get("type") == "text" + ] + msg_dict["content"] = " ".join(text_parts) if text_parts else None + else: + msg_dict["content"] = None + msg_dict["tool_calls"] = tool_calls + openai_messages.append(msg_dict) + elif openai_content: + # Check if it's just text or mixed content + if len(openai_content) == 1 and openai_content[0].get("type") == "text": + openai_messages.append( + {"role": role, "content": openai_content[0].get("text", "")} + ) + else: + openai_messages.append({"role": role, "content": openai_content}) + + return openai_messages + + +def anthropic_to_openai_tools( + anthropic_tools: Optional[List[dict]], +) -> Optional[List[dict]]: + """Convert Anthropic tool definitions to OpenAI format.""" + if not anthropic_tools: + return None + + openai_tools = [] + for tool in anthropic_tools: + openai_tools.append( + { + "type": "function", + "function": { + "name": tool.get("name", ""), + "description": tool.get("description", ""), + "parameters": tool.get("input_schema", {}), + }, + } + ) + return openai_tools + + +def anthropic_to_openai_tool_choice( + anthropic_tool_choice: Optional[dict], +) -> Optional[Union[str, dict]]: + """Convert Anthropic tool_choice to OpenAI format.""" + if not anthropic_tool_choice: + return None + + choice_type = anthropic_tool_choice.get("type", "auto") + + if choice_type == "auto": + return "auto" + elif choice_type == "any": + return "required" + elif choice_type == "tool": + return { + "type": "function", + "function": {"name": anthropic_tool_choice.get("name", "")}, + } + elif choice_type == "none": + return "none" + + return "auto" + + +def openai_to_anthropic_response(openai_response: dict, original_model: str) -> dict: + """ + Convert OpenAI chat completion response to Anthropic Messages format. + """ + choice = openai_response.get("choices", [{}])[0] + message = choice.get("message", {}) + usage = openai_response.get("usage", {}) + + # Build content blocks + content_blocks = [] + + # Add text content if present + text_content = message.get("content") + if text_content: + content_blocks.append({"type": "text", "text": text_content}) + + # Add tool use blocks if present + tool_calls = message.get("tool_calls") or [] + for tc in tool_calls: + func = tc.get("function", {}) + try: + input_data = json.loads(func.get("arguments", "{}")) + except json.JSONDecodeError: + input_data = {} + + content_blocks.append( + { + "type": "tool_use", + "id": tc.get("id", f"toolu_{int(time.time())}"), + "name": func.get("name", ""), + "input": input_data, + } + ) + + # Map finish_reason to stop_reason + finish_reason = choice.get("finish_reason", "end_turn") + stop_reason_map = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "content_filter": "end_turn", + "function_call": "tool_use", + } + stop_reason = stop_reason_map.get(finish_reason, "end_turn") + + # Build usage + anthropic_usage = { + "input_tokens": usage.get("prompt_tokens", 0), + "output_tokens": usage.get("completion_tokens", 0), + } + + # Add cache tokens if present + if usage.get("prompt_tokens_details"): + details = usage["prompt_tokens_details"] + if details.get("cached_tokens"): + anthropic_usage["cache_read_input_tokens"] = details["cached_tokens"] + + return { + "id": openai_response.get("id", f"msg_{int(time.time())}"), + "type": "message", + "role": "assistant", + "content": content_blocks, + "model": original_model, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": anthropic_usage, + } + + +async def anthropic_streaming_wrapper( + request: Request, + openai_stream: AsyncGenerator[str, None], + original_model: str, + request_id: str, +) -> AsyncGenerator[str, None]: + """ + Convert OpenAI streaming format to Anthropic streaming format. + + Anthropic SSE events: + - message_start: Initial message metadata + - content_block_start: Start of a content block + - content_block_delta: Content chunk + - content_block_stop: End of a content block + - message_delta: Final message metadata (stop_reason, usage) + - message_stop: End of message + """ + message_started = False + content_block_started = False + current_block_index = 0 + accumulated_text = "" + tool_calls_by_index = {} # Track tool calls by their index + input_tokens = 0 + output_tokens = 0 + + try: + async for chunk_str in openai_stream: + if await request.is_disconnected(): + break + + if not chunk_str.strip() or not chunk_str.startswith("data:"): + continue + + data_content = chunk_str[len("data:") :].strip() + if data_content == "[DONE]": + # Close any open content blocks (text or tool_use) + if content_block_started or tool_calls_by_index: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + + # Determine stop_reason based on whether we had tool calls + stop_reason = "tool_use" if tool_calls_by_index else "end_turn" + + # Send message_delta with final info + yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "{stop_reason}", "stop_sequence": null}}, "usage": {{"output_tokens": {output_tokens}}}}}\n\n' + + # Send message_stop + yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' + break + + try: + chunk = json.loads(data_content) + except json.JSONDecodeError: + continue + + # Extract usage if present + if "usage" in chunk and chunk["usage"]: + input_tokens = chunk["usage"].get("prompt_tokens", input_tokens) + output_tokens = chunk["usage"].get("completion_tokens", output_tokens) + + # Send message_start on first chunk + if not message_started: + message_start = { + "type": "message_start", + "message": { + "id": request_id, + "type": "message", + "role": "assistant", + "content": [], + "model": original_model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": input_tokens, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" + message_started = True + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + finish_reason = choices[0].get("finish_reason") + + # Handle text content + content = delta.get("content") + if content: + if not content_block_started: + # Start a text content block + block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "text", "text": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + content_block_started = True + + # Send content delta + block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "text_delta", "text": content}, + } + yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + accumulated_text += content + + # Handle tool calls + tool_calls = delta.get("tool_calls", []) + for tc in tool_calls: + tc_index = tc.get("index", 0) + + if tc_index not in tool_calls_by_index: + # Close previous text block if open + if content_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + content_block_started = False + + # Start new tool use block + tool_calls_by_index[tc_index] = { + "id": tc.get("id", f"toolu_{tc_index}"), + "name": tc.get("function", {}).get("name", ""), + "arguments": "", + } + + block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": { + "type": "tool_use", + "id": tool_calls_by_index[tc_index]["id"], + "name": tool_calls_by_index[tc_index]["name"], + "input": {}, + }, + } + yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + + # Accumulate arguments + func = tc.get("function", {}) + if func.get("name"): + tool_calls_by_index[tc_index]["name"] = func["name"] + if func.get("arguments"): + tool_calls_by_index[tc_index]["arguments"] += func["arguments"] + + # Send partial JSON delta + block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": { + "type": "input_json_delta", + "partial_json": func["arguments"], + }, + } + yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + + # Note: We intentionally ignore finish_reason here. + # Block closing is handled when we receive [DONE] to avoid + # premature closes with providers that send finish_reason on each chunk. + + except Exception as e: + logging.error(f"Error in Anthropic streaming wrapper: {e}") + error_event = { + "type": "error", + "error": {"type": "api_error", "message": str(e)}, + } + yield f"event: error\ndata: {json.dumps(error_event)}\n\n" + + async def streaming_response_wrapper( request: Request, request_data: dict, @@ -967,6 +1501,160 @@ async def chat_completions( raise HTTPException(status_code=500, detail=str(e)) +# --- Anthropic Messages API Endpoint --- +@app.post("/v1/messages") +async def anthropic_messages( + request: Request, + body: AnthropicMessagesRequest, + client: RotatingClient = Depends(get_rotating_client), + _=Depends(verify_anthropic_api_key), +): + """ + Anthropic-compatible Messages API endpoint. + + Accepts requests in Anthropic's format and returns responses in Anthropic's format. + Internally translates to OpenAI format for processing via LiteLLM. + + This endpoint is compatible with Claude Code and other Anthropic API clients. + """ + request_id = f"msg_{uuid.uuid4().hex[:24]}" + original_model = body.model + + # Initialize logger if enabled + logger = DetailedLogger() if ENABLE_REQUEST_LOGGING else None + + try: + # Convert Anthropic request to OpenAI format + anthropic_request = body.model_dump(exclude_none=True) + + openai_messages = anthropic_to_openai_messages( + anthropic_request.get("messages", []), anthropic_request.get("system") + ) + + openai_tools = anthropic_to_openai_tools(anthropic_request.get("tools")) + openai_tool_choice = anthropic_to_openai_tool_choice( + anthropic_request.get("tool_choice") + ) + + # Build OpenAI-compatible request + openai_request = { + "model": body.model, + "messages": openai_messages, + "max_tokens": body.max_tokens, + "stream": body.stream or False, + } + + if body.temperature is not None: + openai_request["temperature"] = body.temperature + if body.top_p is not None: + openai_request["top_p"] = body.top_p + if body.stop_sequences: + openai_request["stop"] = body.stop_sequences + if openai_tools: + openai_request["tools"] = openai_tools + if openai_tool_choice: + openai_request["tool_choice"] = openai_tool_choice + + log_request_to_console( + url=str(request.url), + headers=dict(request.headers), + client_info=( + request.client.host if request.client else "unknown", + request.client.port if request.client else 0, + ), + request_data=openai_request, + ) + + if body.stream: + # Streaming response - acompletion returns a generator for streaming + response_generator = client.acompletion(request=request, **openai_request) + + return StreamingResponse( + anthropic_streaming_wrapper( + request, response_generator, original_model, request_id + ), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + else: + # Non-streaming response + response = await client.acompletion(request=request, **openai_request) + + # Convert OpenAI response to Anthropic format + openai_response = ( + response.model_dump() + if hasattr(response, "model_dump") + else dict(response) + ) + anthropic_response = openai_to_anthropic_response( + openai_response, original_model + ) + + # Override the ID with our request ID + anthropic_response["id"] = request_id + + if logger: + logger.log_final_response( + status_code=200, + headers=None, + body=anthropic_response, + ) + + return JSONResponse(content=anthropic_response) + + except ( + litellm.InvalidRequestError, + ValueError, + litellm.ContextWindowExceededError, + ) as e: + error_response = { + "type": "error", + "error": {"type": "invalid_request_error", "message": str(e)}, + } + raise HTTPException(status_code=400, detail=error_response) + except litellm.AuthenticationError as e: + error_response = { + "type": "error", + "error": {"type": "authentication_error", "message": str(e)}, + } + raise HTTPException(status_code=401, detail=error_response) + except litellm.RateLimitError as e: + error_response = { + "type": "error", + "error": {"type": "rate_limit_error", "message": str(e)}, + } + raise HTTPException(status_code=429, detail=error_response) + except (litellm.ServiceUnavailableError, litellm.APIConnectionError) as e: + error_response = { + "type": "error", + "error": {"type": "api_error", "message": str(e)}, + } + raise HTTPException(status_code=503, detail=error_response) + except litellm.Timeout as e: + error_response = { + "type": "error", + "error": {"type": "api_error", "message": f"Request timed out: {str(e)}"}, + } + raise HTTPException(status_code=504, detail=error_response) + except Exception as e: + logging.error(f"Anthropic messages endpoint error: {e}") + if logger: + logger.log_final_response( + status_code=500, + headers=None, + body={"error": str(e)}, + ) + error_response = { + "type": "error", + "error": {"type": "api_error", "message": str(e)}, + } + raise HTTPException(status_code=500, detail=error_response) + + @app.post("/v1/embeddings") async def embeddings( request: Request, From 7e229f4d93c2859c21df96ca2b601173bc5fcbff Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 12 Dec 2025 01:51:45 +0100 Subject: [PATCH 02/36] feat(anthropic): add extended thinking support to /v1/messages endpoint - Add AnthropicThinkingConfig model and thinking parameter to request - Translate Anthropic thinking config to reasoning_effort for providers - Handle reasoning_content in streaming wrapper (thinking_delta events) - Convert reasoning_content to thinking blocks in non-streaming responses --- src/proxy_app/main.py | 75 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index f8eb2bf5b..91017cb7e 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -282,6 +282,13 @@ class AnthropicTool(BaseModel): input_schema: dict +class AnthropicThinkingConfig(BaseModel): + """Anthropic thinking configuration.""" + + type: str # "enabled" or "disabled" + budget_tokens: Optional[int] = None + + class AnthropicMessagesRequest(BaseModel): """Anthropic Messages API request format.""" @@ -297,6 +304,7 @@ class AnthropicMessagesRequest(BaseModel): tools: Optional[List[AnthropicTool]] = None tool_choice: Optional[dict] = None metadata: Optional[dict] = None + thinking: Optional[AnthropicThinkingConfig] = None class AnthropicUsage(BaseModel): @@ -973,6 +981,15 @@ def openai_to_anthropic_response(openai_response: dict, original_model: str) -> # Build content blocks content_blocks = [] + # Add thinking content block if reasoning_content is present + reasoning_content = message.get("reasoning_content") + if reasoning_content: + content_blocks.append({ + "type": "thinking", + "thinking": reasoning_content, + "signature": "", # Signature is typically empty for proxied responses + }) + # Add text content if present text_content = message.get("content") if text_content: @@ -1050,8 +1067,10 @@ async def anthropic_streaming_wrapper( """ message_started = False content_block_started = False + thinking_block_started = False current_block_index = 0 accumulated_text = "" + accumulated_thinking = "" tool_calls_by_index = {} # Track tool calls by their index input_tokens = 0 output_tokens = 0 @@ -1066,8 +1085,8 @@ async def anthropic_streaming_wrapper( data_content = chunk_str[len("data:") :].strip() if data_content == "[DONE]": - # Close any open content blocks (text or tool_use) - if content_block_started or tool_calls_by_index: + # Close any open content blocks (thinking, text, or tool_use) + if thinking_block_started or content_block_started or tool_calls_by_index: yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' # Determine stop_reason based on whether we had tool calls @@ -1115,9 +1134,37 @@ async def anthropic_streaming_wrapper( delta = choices[0].get("delta", {}) finish_reason = choices[0].get("finish_reason") + # Handle reasoning/thinking content (from OpenAI-style reasoning_content) + reasoning_content = delta.get("reasoning_content") + if reasoning_content: + if not thinking_block_started: + # Start a thinking content block + block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "thinking", "thinking": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + thinking_block_started = True + + # Send thinking delta + block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "thinking_delta", "thinking": reasoning_content}, + } + yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + accumulated_thinking += reasoning_content + # Handle text content content = delta.get("content") if content: + # If we were in a thinking block, close it first + if thinking_block_started and not content_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + thinking_block_started = False + if not content_block_started: # Start a text content block block_start = { @@ -1143,6 +1190,12 @@ async def anthropic_streaming_wrapper( tc_index = tc.get("index", 0) if tc_index not in tool_calls_by_index: + # Close previous thinking block if open + if thinking_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + thinking_block_started = False + # Close previous text block if open if content_block_started: yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' @@ -1555,6 +1608,24 @@ async def anthropic_messages( if openai_tool_choice: openai_request["tool_choice"] = openai_tool_choice + # Handle Anthropic thinking config -> reasoning_effort translation + if body.thinking: + if body.thinking.type == "enabled": + # Map budget_tokens to reasoning_effort level + # Default to "medium" if enabled but budget not specified + budget = body.thinking.budget_tokens or 10000 + if budget >= 32000: + openai_request["reasoning_effort"] = "high" + openai_request["custom_reasoning_budget"] = True + elif budget >= 10000: + openai_request["reasoning_effort"] = "high" + elif budget >= 5000: + openai_request["reasoning_effort"] = "medium" + else: + openai_request["reasoning_effort"] = "low" + elif body.thinking.type == "disabled": + openai_request["reasoning_effort"] = "disable" + log_request_to_console( url=str(request.url), headers=dict(request.headers), From 7aea08eee94bc34060ee31691c32a72e735bd3e9 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 12 Dec 2025 02:03:37 +0100 Subject: [PATCH 03/36] feat(anthropic): force high thinking budget for Opus models by default When no thinking config is provided in the request, Opus models now automatically use reasoning_effort=high with custom_reasoning_budget=True. This ensures Opus 4.5 uses the full 32768 token thinking budget instead of the backend's auto mode (thinkingBudget: -1) which may use less. Opus always uses the -thinking variant regardless, but this change guarantees maximum thinking capacity for better reasoning quality. --- src/proxy_app/main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 91017cb7e..2cc61f07f 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1625,6 +1625,12 @@ async def anthropic_messages( openai_request["reasoning_effort"] = "low" elif body.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" + elif "opus" in body.model.lower(): + # Force high thinking for Opus models when no thinking config is provided + # Opus 4.5 always uses the -thinking variant, so we want maximum thinking budget + # Without this, the backend defaults to thinkingBudget: -1 (auto) instead of high + openai_request["reasoning_effort"] = "high" + openai_request["custom_reasoning_budget"] = True log_request_to_console( url=str(request.url), From 05d89a2a53ba610ef562d6e8ca0b2fe02e360c52 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Sat, 13 Dec 2025 12:11:23 +0100 Subject: [PATCH 04/36] fix: ensure max_tokens exceeds thinking budget and improve error handling - Add validation to ensure maxOutputTokens > thinkingBudget for Claude extended thinking (prevents 400 INVALID_ARGUMENT API errors) - Improve streaming error handling to send proper message_start and content blocks before error event for better client compatibility - Minor code formatting improvements --- src/proxy_app/main.py | 60 +++++++++++++++++-- .../providers/antigravity_provider.py | 22 +++++++ 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 2cc61f07f..f76369721 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -984,11 +984,13 @@ def openai_to_anthropic_response(openai_response: dict, original_model: str) -> # Add thinking content block if reasoning_content is present reasoning_content = message.get("reasoning_content") if reasoning_content: - content_blocks.append({ - "type": "thinking", - "thinking": reasoning_content, - "signature": "", # Signature is typically empty for proxied responses - }) + content_blocks.append( + { + "type": "thinking", + "thinking": reasoning_content, + "signature": "", # Signature is typically empty for proxied responses + } + ) # Add text content if present text_content = message.get("content") @@ -1086,7 +1088,11 @@ async def anthropic_streaming_wrapper( data_content = chunk_str[len("data:") :].strip() if data_content == "[DONE]": # Close any open content blocks (thinking, text, or tool_use) - if thinking_block_started or content_block_started or tool_calls_by_index: + if ( + thinking_block_started + or content_block_started + or tool_calls_by_index + ): yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' # Determine stop_reason based on whether we had tool calls @@ -1245,6 +1251,48 @@ async def anthropic_streaming_wrapper( except Exception as e: logging.error(f"Error in Anthropic streaming wrapper: {e}") + + # If we haven't sent message_start yet, send it now so the client can display the error + # Claude Code and other clients may ignore events that come before message_start + if not message_started: + message_start = { + "type": "message_start", + "message": { + "id": request_id, + "type": "message", + "role": "assistant", + "content": [], + "model": original_model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" + + # Send the error as a text content block so it's visible to the user + error_message = f"Error: {str(e)}" + error_block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "text", "text": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(error_block_start)}\n\n" + + error_block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "text_delta", "text": error_message}, + } + yield f"event: content_block_delta\ndata: {json.dumps(error_block_delta)}\n\n" + + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + + # Send message_delta and message_stop to properly close the stream + yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "end_turn", "stop_sequence": null}}, "usage": {{"output_tokens": 0}}}}\n\n' + yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' + + # Also send the formal error event for clients that handle it error_event = { "type": "error", "error": {"type": "api_error", "message": str(e)}, diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 4bd0b21c2..874e910a8 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -3537,6 +3537,28 @@ def _transform_to_antigravity_format( gen_config["maxOutputTokens"] = DEFAULT_MAX_OUTPUT_TOKENS # For non-Claude models without explicit max_tokens, don't set it + # CRITICAL: For Claude with extended thinking, max_tokens MUST be > thinking.budget_tokens + # Per Claude docs: https://docs.claude.com/en/docs/build-with-claude/extended-thinking + # If this constraint is violated, the API returns 400 INVALID_ARGUMENT + thinking_config = gen_config.get("thinkingConfig", {}) + thinking_budget = thinking_config.get("thinkingBudget", 0) + current_max_tokens = gen_config.get("maxOutputTokens") + + if ( + is_claude + and thinking_budget + and thinking_budget > 0 + and current_max_tokens is not None + ): + # Ensure max_tokens > thinkingBudget (add buffer for actual response content) + min_required_tokens = thinking_budget + 1024 # 1024 buffer for response + if current_max_tokens <= thinking_budget: + lib_logger.warning( + f"max_tokens ({current_max_tokens}) must be > thinkingBudget ({thinking_budget}). " + f"Adjusting to {min_required_tokens}" + ) + gen_config["maxOutputTokens"] = min_required_tokens + antigravity_payload["request"]["generationConfig"] = gen_config # Set toolConfig based on tool_choice parameter From e35f3f019812b03448e3618c5c00b36e7a0e05a3 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Sun, 14 Dec 2025 01:48:16 +0100 Subject: [PATCH 05/36] fix(anthropic): properly close all content blocks in streaming wrapper Track each tool_use block index separately and emit content_block_stop for all blocks (thinking, text, and each tool_use) when stream ends. Fixes Claude Code stopping mid-action due to malformed streaming events. --- src/proxy_app/main.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index f76369721..75e21a03f 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1074,6 +1074,7 @@ async def anthropic_streaming_wrapper( accumulated_text = "" accumulated_thinking = "" tool_calls_by_index = {} # Track tool calls by their index + tool_block_indices = {} # Track which block index each tool call uses input_tokens = 0 output_tokens = 0 @@ -1087,13 +1088,22 @@ async def anthropic_streaming_wrapper( data_content = chunk_str[len("data:") :].strip() if data_content == "[DONE]": - # Close any open content blocks (thinking, text, or tool_use) - if ( - thinking_block_started - or content_block_started - or tool_calls_by_index - ): + # Close any open thinking block + if thinking_block_started: yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + thinking_block_started = False + + # Close any open text block + if content_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + content_block_started = False + + # Close all open tool_use blocks + for tc_index in sorted(tool_block_indices.keys()): + block_idx = tool_block_indices[tc_index] + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {block_idx}}}\n\n' # Determine stop_reason based on whether we had tool calls stop_reason = "tool_use" if tool_calls_by_index else "end_turn" @@ -1214,6 +1224,8 @@ async def anthropic_streaming_wrapper( "name": tc.get("function", {}).get("name", ""), "arguments": "", } + # Track which block index this tool call uses + tool_block_indices[tc_index] = current_block_index block_start = { "type": "content_block_start", @@ -1226,6 +1238,8 @@ async def anthropic_streaming_wrapper( }, } yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + # Increment for the next block + current_block_index += 1 # Accumulate arguments func = tc.get("function", {}) @@ -1234,10 +1248,10 @@ async def anthropic_streaming_wrapper( if func.get("arguments"): tool_calls_by_index[tc_index]["arguments"] += func["arguments"] - # Send partial JSON delta + # Send partial JSON delta using the correct block index for this tool block_delta = { "type": "content_block_delta", - "index": current_block_index, + "index": tool_block_indices[tc_index], "delta": { "type": "input_json_delta", "partial_json": func["arguments"], From 4ec92ec9673b9bbfc323ab9adbd89e639f0e1a3c Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Sun, 14 Dec 2025 23:50:09 +0100 Subject: [PATCH 06/36] fix(anthropic): add missing uuid import for /v1/messages endpoint --- src/proxy_app/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 75e21a03f..d582ad7d8 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1,4 +1,5 @@ import time +import uuid # Phase 1: Minimal imports for arg parsing and TUI import asyncio From b70efdf65aa1dcd118be54399fd60aa41e2784da Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Mon, 15 Dec 2025 00:27:02 +0100 Subject: [PATCH 07/36] fix(anthropic): always set custom_reasoning_budget when thinking is enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fixed bug where budget_tokens between 10000-32000 would get ÷4 reduction - Now any explicit thinking request sets custom_reasoning_budget=True - Added logging to show thinking budget, effort level, and custom_budget flag - Simplified budget tier logic (removed redundant >= 32000 check) Before: 31999 tokens requested → 8192 tokens actual (÷4 applied) After: 31999 tokens requested → 32768 tokens actual (full "high" budget) --- src/proxy_app/main.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index d582ad7d8..ed0a37aa1 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1672,15 +1672,16 @@ async def anthropic_messages( openai_request["tool_choice"] = openai_tool_choice # Handle Anthropic thinking config -> reasoning_effort translation + thinking_budget_requested = None if body.thinking: if body.thinking.type == "enabled": # Map budget_tokens to reasoning_effort level - # Default to "medium" if enabled but budget not specified + # Always set custom_reasoning_budget=True when client explicitly requests thinking + # This prevents the ÷4 reduction in Antigravity provider budget = body.thinking.budget_tokens or 10000 - if budget >= 32000: - openai_request["reasoning_effort"] = "high" - openai_request["custom_reasoning_budget"] = True - elif budget >= 10000: + thinking_budget_requested = budget + openai_request["custom_reasoning_budget"] = True + if budget >= 10000: openai_request["reasoning_effort"] = "high" elif budget >= 5000: openai_request["reasoning_effort"] = "medium" @@ -1688,12 +1689,21 @@ async def anthropic_messages( openai_request["reasoning_effort"] = "low" elif body.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" + thinking_budget_requested = 0 elif "opus" in body.model.lower(): # Force high thinking for Opus models when no thinking config is provided # Opus 4.5 always uses the -thinking variant, so we want maximum thinking budget - # Without this, the backend defaults to thinkingBudget: -1 (auto) instead of high openai_request["reasoning_effort"] = "high" openai_request["custom_reasoning_budget"] = True + thinking_budget_requested = "auto (high)" + + # Log thinking config for debugging + if thinking_budget_requested is not None: + logging.info( + f"🧠 Thinking: requested={thinking_budget_requested}, " + f"effort={openai_request.get('reasoning_effort', 'none')}, " + f"custom_budget={openai_request.get('custom_reasoning_budget', False)}" + ) log_request_to_console( url=str(request.url), From 4bd879b3f60a2e58a96b52d8c27b69de6f2acf70 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Mon, 15 Dec 2025 00:31:00 +0100 Subject: [PATCH 08/36] feat(openai): auto-enable full thinking budget for Opus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using /v1/chat/completions with Opus and reasoning_effort="high" or "medium", automatically set custom_reasoning_budget=true to get full thinking tokens instead of the ÷4 reduced default. This makes the OpenAI endpoint behave consistently with the Anthropic endpoint for Opus models - if you're using Opus with high reasoning, you want the full thinking budget. Adds logging: "🧠 Thinking: auto-enabled custom_reasoning_budget for Opus" --- src/proxy_app/main.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index ed0a37aa1..6bd3976f1 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1547,6 +1547,20 @@ async def chat_completions( "custom_reasoning_budget" ) or generation_cfg.get("custom_reasoning_budget", False) + # Auto-enable full thinking budget for Opus with high reasoning effort + # Opus is THE reasoning model - if you're asking for "high", you want full budget + if ( + model + and "opus" in model.lower() + and reasoning_effort in ("high", "medium") + and not custom_reasoning_budget + ): + request_data["custom_reasoning_budget"] = True + custom_reasoning_budget = True + logging.info( + f"🧠 Thinking: auto-enabled custom_reasoning_budget for Opus (effort={reasoning_effort})" + ) + logging.getLogger("rotator_library").debug( f"Handling reasoning parameters: model={model}, reasoning_effort={reasoning_effort}, custom_reasoning_budget={custom_reasoning_budget}" ) From 758b4b53d84c7d751e01e314ddd98329a760bdce Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Mon, 15 Dec 2025 00:49:43 +0100 Subject: [PATCH 09/36] fix(anthropic): add missing JSONResponse import for non-streaming responses --- src/proxy_app/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 6bd3976f1..b0dce5279 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -101,7 +101,6 @@ from fastapi import FastAPI, Request, HTTPException, Depends from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse, JSONResponse - import uuid from fastapi.security import APIKeyHeader print(" → Loading core dependencies...") From f2d728849f6aee2277c5c3a15be7b1e56cc724d2 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Mon, 15 Dec 2025 21:55:10 +0100 Subject: [PATCH 10/36] fix(anthropic): ensure message_start is sent before message_stop in streaming Claude Code and other Anthropic SDK clients require message_start to be sent before any other SSE events. When a stream completed quickly without content chunks, the wrapper would send message_stop without message_start, causing clients to silently discard all output. --- src/proxy_app/main.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index b0dce5279..cc7e268b8 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1088,6 +1088,25 @@ async def anthropic_streaming_wrapper( data_content = chunk_str[len("data:") :].strip() if data_content == "[DONE]": + # CRITICAL: Send message_start if we haven't yet (e.g., empty response) + # Claude Code and other clients require message_start before message_stop + if not message_started: + message_start = { + "type": "message_start", + "message": { + "id": request_id, + "type": "message", + "role": "assistant", + "content": [], + "model": original_model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": input_tokens, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" + message_started = True + # Close any open thinking block if thinking_block_started: yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' From de88557392e40c148af4003c0e3f90382435ea8b Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Tue, 16 Dec 2025 01:00:17 +0100 Subject: [PATCH 11/36] feat: add /context endpoint for anthropic routes Signed-off-by: Moeeze Hassan --- src/proxy_app/main.py | 95 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index cc7e268b8..18104a66c 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -329,6 +329,24 @@ class AnthropicMessagesResponse(BaseModel): usage: AnthropicUsage +# --- Anthropic Count Tokens Models --- +class AnthropicCountTokensRequest(BaseModel): + """Anthropic count_tokens API request format.""" + + model: str + messages: List[AnthropicMessage] + system: Optional[Union[str, List[dict]]] = None + tools: Optional[List[AnthropicTool]] = None + tool_choice: Optional[dict] = None + thinking: Optional[AnthropicThinkingConfig] = None + + +class AnthropicCountTokensResponse(BaseModel): + """Anthropic count_tokens API response format.""" + + input_tokens: int + + # Calculate total loading time _elapsed = time.time() - _start_time print( @@ -1837,6 +1855,83 @@ async def anthropic_messages( raise HTTPException(status_code=500, detail=error_response) +# --- Anthropic Count Tokens Endpoint --- +@app.post("/v1/messages/count_tokens") +async def anthropic_count_tokens( + request: Request, + body: AnthropicCountTokensRequest, + client: RotatingClient = Depends(get_rotating_client), + _=Depends(verify_anthropic_api_key), +): + """ + Anthropic-compatible count_tokens endpoint. + + Counts the number of tokens that would be used by a Messages API request. + This is useful for estimating costs and managing context windows. + + Accepts requests in Anthropic's format and returns token count in Anthropic's format. + """ + try: + # Convert Anthropic request to OpenAI format for token counting + anthropic_request = body.model_dump(exclude_none=True) + + openai_messages = anthropic_to_openai_messages( + anthropic_request.get("messages", []), anthropic_request.get("system") + ) + + # Count tokens for messages + message_tokens = client.token_count( + model=body.model, + messages=openai_messages, + ) + + # Count tokens for tools if present + tool_tokens = 0 + if body.tools: + # Tools add tokens based on their definitions + # Convert to JSON string and count tokens for tool definitions + openai_tools = anthropic_to_openai_tools( + [tool.model_dump() for tool in body.tools] + ) + if openai_tools: + # Serialize tools to count their token contribution + tools_text = json.dumps(openai_tools) + tool_tokens = client.token_count( + model=body.model, + text=tools_text, + ) + + total_tokens = message_tokens + tool_tokens + + return JSONResponse( + content={"input_tokens": total_tokens} + ) + + except ( + litellm.InvalidRequestError, + ValueError, + litellm.ContextWindowExceededError, + ) as e: + error_response = { + "type": "error", + "error": {"type": "invalid_request_error", "message": str(e)}, + } + raise HTTPException(status_code=400, detail=error_response) + except litellm.AuthenticationError as e: + error_response = { + "type": "error", + "error": {"type": "authentication_error", "message": str(e)}, + } + raise HTTPException(status_code=401, detail=error_response) + except Exception as e: + logging.error(f"Anthropic count_tokens endpoint error: {e}") + error_response = { + "type": "error", + "error": {"type": "api_error", "message": str(e)}, + } + raise HTTPException(status_code=500, detail=error_response) + + @app.post("/v1/embeddings") async def embeddings( request: Request, From beed0bc2c26c23b52fce65be71e083ab45055ee7 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 19 Dec 2025 14:46:00 +0100 Subject: [PATCH 12/36] Revert "feat(openai): auto-enable full thinking budget for Opus" This reverts commit e80645e6191c6965f94b70fb0842f5689294a884. --- src/proxy_app/main.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 18104a66c..5f33a7ef9 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1583,20 +1583,6 @@ async def chat_completions( "custom_reasoning_budget" ) or generation_cfg.get("custom_reasoning_budget", False) - # Auto-enable full thinking budget for Opus with high reasoning effort - # Opus is THE reasoning model - if you're asking for "high", you want full budget - if ( - model - and "opus" in model.lower() - and reasoning_effort in ("high", "medium") - and not custom_reasoning_budget - ): - request_data["custom_reasoning_budget"] = True - custom_reasoning_budget = True - logging.info( - f"🧠 Thinking: auto-enabled custom_reasoning_budget for Opus (effort={reasoning_effort})" - ) - logging.getLogger("rotator_library").debug( f"Handling reasoning parameters: model={model}, reasoning_effort={reasoning_effort}, custom_reasoning_budget={custom_reasoning_budget}" ) From 2c93a68a7828f5a0d73486097b6452dbb67af636 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 19 Dec 2025 14:52:36 +0100 Subject: [PATCH 13/36] Revert "fix(anthropic): always set custom_reasoning_budget when thinking is enabled" This reverts commit 2ee549d997bedf1b4d77f1f70639d3767cb59d77. --- src/proxy_app/main.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 5f33a7ef9..277aa170a 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1708,16 +1708,15 @@ async def anthropic_messages( openai_request["tool_choice"] = openai_tool_choice # Handle Anthropic thinking config -> reasoning_effort translation - thinking_budget_requested = None if body.thinking: if body.thinking.type == "enabled": # Map budget_tokens to reasoning_effort level - # Always set custom_reasoning_budget=True when client explicitly requests thinking - # This prevents the ÷4 reduction in Antigravity provider + # Default to "medium" if enabled but budget not specified budget = body.thinking.budget_tokens or 10000 - thinking_budget_requested = budget - openai_request["custom_reasoning_budget"] = True - if budget >= 10000: + if budget >= 32000: + openai_request["reasoning_effort"] = "high" + openai_request["custom_reasoning_budget"] = True + elif budget >= 10000: openai_request["reasoning_effort"] = "high" elif budget >= 5000: openai_request["reasoning_effort"] = "medium" @@ -1725,21 +1724,12 @@ async def anthropic_messages( openai_request["reasoning_effort"] = "low" elif body.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" - thinking_budget_requested = 0 elif "opus" in body.model.lower(): # Force high thinking for Opus models when no thinking config is provided # Opus 4.5 always uses the -thinking variant, so we want maximum thinking budget + # Without this, the backend defaults to thinkingBudget: -1 (auto) instead of high openai_request["reasoning_effort"] = "high" openai_request["custom_reasoning_budget"] = True - thinking_budget_requested = "auto (high)" - - # Log thinking config for debugging - if thinking_budget_requested is not None: - logging.info( - f"🧠 Thinking: requested={thinking_budget_requested}, " - f"effort={openai_request.get('reasoning_effort', 'none')}, " - f"custom_budget={openai_request.get('custom_reasoning_budget', False)}" - ) log_request_to_console( url=str(request.url), From b19526cd76593ac0db1ae96f12a0018ee7490abc Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Sat, 20 Dec 2025 22:16:11 +0100 Subject: [PATCH 14/36] refactor: Move Anthropic translation layer to rotator_library - Create rotator_library/anthropic_compat module with models, translator, and streaming - Add anthropic_messages() and anthropic_count_tokens() methods to RotatingClient - Simplify main.py endpoints to use library methods - Remove ~762 lines of duplicate code from main.py - Fix: Use UUID instead of time.time() for tool/message IDs (avoids collisions) - Fix: Remove unused accumulated_text/accumulated_thinking variables - Fix: Map top_k parameter from Anthropic to OpenAI format --- src/proxy_app/main.py | 796 +----------------- src/rotator_library/__init__.py | 8 +- .../anthropic_compat/__init__.py | 67 ++ .../anthropic_compat/models.py | 144 ++++ .../anthropic_compat/streaming.py | 308 +++++++ .../anthropic_compat/translator.py | 363 ++++++++ src/rotator_library/client.py | 130 +++ 7 files changed, 1036 insertions(+), 780 deletions(-) create mode 100644 src/rotator_library/anthropic_compat/__init__.py create mode 100644 src/rotator_library/anthropic_compat/models.py create mode 100644 src/rotator_library/anthropic_compat/streaming.py create mode 100644 src/rotator_library/anthropic_compat/translator.py diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 277aa170a..16a64bd31 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -215,136 +215,11 @@ class EnrichedModelList(BaseModel): data: List[EnrichedModelCard] -# --- Anthropic API Models --- -class AnthropicTextBlock(BaseModel): - """Anthropic text content block.""" - - type: str = "text" - text: str - - -class AnthropicImageSource(BaseModel): - """Anthropic image source for base64 images.""" - - type: str = "base64" - media_type: str - data: str - - -class AnthropicImageBlock(BaseModel): - """Anthropic image content block.""" - - type: str = "image" - source: AnthropicImageSource - - -class AnthropicToolUseBlock(BaseModel): - """Anthropic tool use content block.""" - - type: str = "tool_use" - id: str - name: str - input: dict - - -class AnthropicToolResultBlock(BaseModel): - """Anthropic tool result content block.""" - - type: str = "tool_result" - tool_use_id: str - content: Union[str, List[Any]] - is_error: Optional[bool] = None - - -class AnthropicMessage(BaseModel): - """Anthropic message format.""" - - role: str - content: Union[ - str, - List[ - Union[ - AnthropicTextBlock, - AnthropicImageBlock, - AnthropicToolUseBlock, - AnthropicToolResultBlock, - dict, - ] - ], - ] - - -class AnthropicTool(BaseModel): - """Anthropic tool definition.""" - - name: str - description: Optional[str] = None - input_schema: dict - - -class AnthropicThinkingConfig(BaseModel): - """Anthropic thinking configuration.""" - - type: str # "enabled" or "disabled" - budget_tokens: Optional[int] = None - - -class AnthropicMessagesRequest(BaseModel): - """Anthropic Messages API request format.""" - - model: str - messages: List[AnthropicMessage] - max_tokens: int - system: Optional[Union[str, List[dict]]] = None - temperature: Optional[float] = None - top_p: Optional[float] = None - top_k: Optional[int] = None - stop_sequences: Optional[List[str]] = None - stream: Optional[bool] = False - tools: Optional[List[AnthropicTool]] = None - tool_choice: Optional[dict] = None - metadata: Optional[dict] = None - thinking: Optional[AnthropicThinkingConfig] = None - - -class AnthropicUsage(BaseModel): - """Anthropic usage statistics.""" - - input_tokens: int - output_tokens: int - cache_creation_input_tokens: Optional[int] = None - cache_read_input_tokens: Optional[int] = None - - -class AnthropicMessagesResponse(BaseModel): - """Anthropic Messages API response format.""" - - id: str - type: str = "message" - role: str = "assistant" - content: List[Union[AnthropicTextBlock, AnthropicToolUseBlock, dict]] - model: str - stop_reason: Optional[str] = None - stop_sequence: Optional[str] = None - usage: AnthropicUsage - - -# --- Anthropic Count Tokens Models --- -class AnthropicCountTokensRequest(BaseModel): - """Anthropic count_tokens API request format.""" - - model: str - messages: List[AnthropicMessage] - system: Optional[Union[str, List[dict]]] = None - tools: Optional[List[AnthropicTool]] = None - tool_choice: Optional[dict] = None - thinking: Optional[AnthropicThinkingConfig] = None - - -class AnthropicCountTokensResponse(BaseModel): - """Anthropic count_tokens API response format.""" - - input_tokens: int +# --- Anthropic API Models (imported from library) --- +from rotator_library.anthropic_compat import ( + AnthropicMessagesRequest, + AnthropicCountTokensRequest, +) # Calculate total loading time @@ -819,538 +694,6 @@ async def verify_anthropic_api_key( raise HTTPException(status_code=401, detail="Invalid or missing API Key") -# --- Anthropic <-> OpenAI Format Translation --- -def anthropic_to_openai_messages( - anthropic_messages: List[dict], system: Optional[Union[str, List[dict]]] = None -) -> List[dict]: - """ - Convert Anthropic message format to OpenAI format. - - Key differences: - - Anthropic: system is a separate field, content can be string or list of blocks - - OpenAI: system is a message with role="system", content is usually string - """ - openai_messages = [] - - # Handle system message - if system: - if isinstance(system, str): - openai_messages.append({"role": "system", "content": system}) - elif isinstance(system, list): - # System can be list of text blocks in Anthropic format - system_text = " ".join( - block.get("text", "") - for block in system - if isinstance(block, dict) and block.get("type") == "text" - ) - if system_text: - openai_messages.append({"role": "system", "content": system_text}) - - for msg in anthropic_messages: - role = msg.get("role", "user") - content = msg.get("content", "") - - if isinstance(content, str): - openai_messages.append({"role": role, "content": content}) - elif isinstance(content, list): - # Handle content blocks - openai_content = [] - tool_calls = [] - - for block in content: - if isinstance(block, dict): - block_type = block.get("type", "text") - - if block_type == "text": - openai_content.append( - {"type": "text", "text": block.get("text", "")} - ) - elif block_type == "image": - # Convert Anthropic image format to OpenAI - source = block.get("source", {}) - if source.get("type") == "base64": - openai_content.append( - { - "type": "image_url", - "image_url": { - "url": f"data:{source.get('media_type', 'image/png')};base64,{source.get('data', '')}" - }, - } - ) - elif source.get("type") == "url": - openai_content.append( - { - "type": "image_url", - "image_url": {"url": source.get("url", "")}, - } - ) - elif block_type == "tool_use": - # Anthropic tool_use -> OpenAI tool_calls - tool_calls.append( - { - "id": block.get("id", ""), - "type": "function", - "function": { - "name": block.get("name", ""), - "arguments": json.dumps(block.get("input", {})), - }, - } - ) - elif block_type == "tool_result": - # Tool results become separate messages in OpenAI format - tool_content = block.get("content", "") - if isinstance(tool_content, list): - tool_content = " ".join( - b.get("text", "") - for b in tool_content - if isinstance(b, dict) and b.get("type") == "text" - ) - openai_messages.append( - { - "role": "tool", - "tool_call_id": block.get("tool_use_id", ""), - "content": str(tool_content), - } - ) - continue # Don't add to current message - - # Build the message - if tool_calls: - # Assistant message with tool calls - msg_dict = {"role": role} - if openai_content: - # If there's text content alongside tool calls - text_parts = [ - c.get("text", "") - for c in openai_content - if c.get("type") == "text" - ] - msg_dict["content"] = " ".join(text_parts) if text_parts else None - else: - msg_dict["content"] = None - msg_dict["tool_calls"] = tool_calls - openai_messages.append(msg_dict) - elif openai_content: - # Check if it's just text or mixed content - if len(openai_content) == 1 and openai_content[0].get("type") == "text": - openai_messages.append( - {"role": role, "content": openai_content[0].get("text", "")} - ) - else: - openai_messages.append({"role": role, "content": openai_content}) - - return openai_messages - - -def anthropic_to_openai_tools( - anthropic_tools: Optional[List[dict]], -) -> Optional[List[dict]]: - """Convert Anthropic tool definitions to OpenAI format.""" - if not anthropic_tools: - return None - - openai_tools = [] - for tool in anthropic_tools: - openai_tools.append( - { - "type": "function", - "function": { - "name": tool.get("name", ""), - "description": tool.get("description", ""), - "parameters": tool.get("input_schema", {}), - }, - } - ) - return openai_tools - - -def anthropic_to_openai_tool_choice( - anthropic_tool_choice: Optional[dict], -) -> Optional[Union[str, dict]]: - """Convert Anthropic tool_choice to OpenAI format.""" - if not anthropic_tool_choice: - return None - - choice_type = anthropic_tool_choice.get("type", "auto") - - if choice_type == "auto": - return "auto" - elif choice_type == "any": - return "required" - elif choice_type == "tool": - return { - "type": "function", - "function": {"name": anthropic_tool_choice.get("name", "")}, - } - elif choice_type == "none": - return "none" - - return "auto" - - -def openai_to_anthropic_response(openai_response: dict, original_model: str) -> dict: - """ - Convert OpenAI chat completion response to Anthropic Messages format. - """ - choice = openai_response.get("choices", [{}])[0] - message = choice.get("message", {}) - usage = openai_response.get("usage", {}) - - # Build content blocks - content_blocks = [] - - # Add thinking content block if reasoning_content is present - reasoning_content = message.get("reasoning_content") - if reasoning_content: - content_blocks.append( - { - "type": "thinking", - "thinking": reasoning_content, - "signature": "", # Signature is typically empty for proxied responses - } - ) - - # Add text content if present - text_content = message.get("content") - if text_content: - content_blocks.append({"type": "text", "text": text_content}) - - # Add tool use blocks if present - tool_calls = message.get("tool_calls") or [] - for tc in tool_calls: - func = tc.get("function", {}) - try: - input_data = json.loads(func.get("arguments", "{}")) - except json.JSONDecodeError: - input_data = {} - - content_blocks.append( - { - "type": "tool_use", - "id": tc.get("id", f"toolu_{int(time.time())}"), - "name": func.get("name", ""), - "input": input_data, - } - ) - - # Map finish_reason to stop_reason - finish_reason = choice.get("finish_reason", "end_turn") - stop_reason_map = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - "content_filter": "end_turn", - "function_call": "tool_use", - } - stop_reason = stop_reason_map.get(finish_reason, "end_turn") - - # Build usage - anthropic_usage = { - "input_tokens": usage.get("prompt_tokens", 0), - "output_tokens": usage.get("completion_tokens", 0), - } - - # Add cache tokens if present - if usage.get("prompt_tokens_details"): - details = usage["prompt_tokens_details"] - if details.get("cached_tokens"): - anthropic_usage["cache_read_input_tokens"] = details["cached_tokens"] - - return { - "id": openai_response.get("id", f"msg_{int(time.time())}"), - "type": "message", - "role": "assistant", - "content": content_blocks, - "model": original_model, - "stop_reason": stop_reason, - "stop_sequence": None, - "usage": anthropic_usage, - } - - -async def anthropic_streaming_wrapper( - request: Request, - openai_stream: AsyncGenerator[str, None], - original_model: str, - request_id: str, -) -> AsyncGenerator[str, None]: - """ - Convert OpenAI streaming format to Anthropic streaming format. - - Anthropic SSE events: - - message_start: Initial message metadata - - content_block_start: Start of a content block - - content_block_delta: Content chunk - - content_block_stop: End of a content block - - message_delta: Final message metadata (stop_reason, usage) - - message_stop: End of message - """ - message_started = False - content_block_started = False - thinking_block_started = False - current_block_index = 0 - accumulated_text = "" - accumulated_thinking = "" - tool_calls_by_index = {} # Track tool calls by their index - tool_block_indices = {} # Track which block index each tool call uses - input_tokens = 0 - output_tokens = 0 - - try: - async for chunk_str in openai_stream: - if await request.is_disconnected(): - break - - if not chunk_str.strip() or not chunk_str.startswith("data:"): - continue - - data_content = chunk_str[len("data:") :].strip() - if data_content == "[DONE]": - # CRITICAL: Send message_start if we haven't yet (e.g., empty response) - # Claude Code and other clients require message_start before message_stop - if not message_started: - message_start = { - "type": "message_start", - "message": { - "id": request_id, - "type": "message", - "role": "assistant", - "content": [], - "model": original_model, - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": input_tokens, "output_tokens": 0}, - }, - } - yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" - message_started = True - - # Close any open thinking block - if thinking_block_started: - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' - current_block_index += 1 - thinking_block_started = False - - # Close any open text block - if content_block_started: - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' - current_block_index += 1 - content_block_started = False - - # Close all open tool_use blocks - for tc_index in sorted(tool_block_indices.keys()): - block_idx = tool_block_indices[tc_index] - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {block_idx}}}\n\n' - - # Determine stop_reason based on whether we had tool calls - stop_reason = "tool_use" if tool_calls_by_index else "end_turn" - - # Send message_delta with final info - yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "{stop_reason}", "stop_sequence": null}}, "usage": {{"output_tokens": {output_tokens}}}}}\n\n' - - # Send message_stop - yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' - break - - try: - chunk = json.loads(data_content) - except json.JSONDecodeError: - continue - - # Extract usage if present - if "usage" in chunk and chunk["usage"]: - input_tokens = chunk["usage"].get("prompt_tokens", input_tokens) - output_tokens = chunk["usage"].get("completion_tokens", output_tokens) - - # Send message_start on first chunk - if not message_started: - message_start = { - "type": "message_start", - "message": { - "id": request_id, - "type": "message", - "role": "assistant", - "content": [], - "model": original_model, - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": input_tokens, "output_tokens": 0}, - }, - } - yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" - message_started = True - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - finish_reason = choices[0].get("finish_reason") - - # Handle reasoning/thinking content (from OpenAI-style reasoning_content) - reasoning_content = delta.get("reasoning_content") - if reasoning_content: - if not thinking_block_started: - # Start a thinking content block - block_start = { - "type": "content_block_start", - "index": current_block_index, - "content_block": {"type": "thinking", "thinking": ""}, - } - yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" - thinking_block_started = True - - # Send thinking delta - block_delta = { - "type": "content_block_delta", - "index": current_block_index, - "delta": {"type": "thinking_delta", "thinking": reasoning_content}, - } - yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" - accumulated_thinking += reasoning_content - - # Handle text content - content = delta.get("content") - if content: - # If we were in a thinking block, close it first - if thinking_block_started and not content_block_started: - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' - current_block_index += 1 - thinking_block_started = False - - if not content_block_started: - # Start a text content block - block_start = { - "type": "content_block_start", - "index": current_block_index, - "content_block": {"type": "text", "text": ""}, - } - yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" - content_block_started = True - - # Send content delta - block_delta = { - "type": "content_block_delta", - "index": current_block_index, - "delta": {"type": "text_delta", "text": content}, - } - yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" - accumulated_text += content - - # Handle tool calls - tool_calls = delta.get("tool_calls", []) - for tc in tool_calls: - tc_index = tc.get("index", 0) - - if tc_index not in tool_calls_by_index: - # Close previous thinking block if open - if thinking_block_started: - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' - current_block_index += 1 - thinking_block_started = False - - # Close previous text block if open - if content_block_started: - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' - current_block_index += 1 - content_block_started = False - - # Start new tool use block - tool_calls_by_index[tc_index] = { - "id": tc.get("id", f"toolu_{tc_index}"), - "name": tc.get("function", {}).get("name", ""), - "arguments": "", - } - # Track which block index this tool call uses - tool_block_indices[tc_index] = current_block_index - - block_start = { - "type": "content_block_start", - "index": current_block_index, - "content_block": { - "type": "tool_use", - "id": tool_calls_by_index[tc_index]["id"], - "name": tool_calls_by_index[tc_index]["name"], - "input": {}, - }, - } - yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" - # Increment for the next block - current_block_index += 1 - - # Accumulate arguments - func = tc.get("function", {}) - if func.get("name"): - tool_calls_by_index[tc_index]["name"] = func["name"] - if func.get("arguments"): - tool_calls_by_index[tc_index]["arguments"] += func["arguments"] - - # Send partial JSON delta using the correct block index for this tool - block_delta = { - "type": "content_block_delta", - "index": tool_block_indices[tc_index], - "delta": { - "type": "input_json_delta", - "partial_json": func["arguments"], - }, - } - yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" - - # Note: We intentionally ignore finish_reason here. - # Block closing is handled when we receive [DONE] to avoid - # premature closes with providers that send finish_reason on each chunk. - - except Exception as e: - logging.error(f"Error in Anthropic streaming wrapper: {e}") - - # If we haven't sent message_start yet, send it now so the client can display the error - # Claude Code and other clients may ignore events that come before message_start - if not message_started: - message_start = { - "type": "message_start", - "message": { - "id": request_id, - "type": "message", - "role": "assistant", - "content": [], - "model": original_model, - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": 0, "output_tokens": 0}, - }, - } - yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" - - # Send the error as a text content block so it's visible to the user - error_message = f"Error: {str(e)}" - error_block_start = { - "type": "content_block_start", - "index": current_block_index, - "content_block": {"type": "text", "text": ""}, - } - yield f"event: content_block_start\ndata: {json.dumps(error_block_start)}\n\n" - - error_block_delta = { - "type": "content_block_delta", - "index": current_block_index, - "delta": {"type": "text_delta", "text": error_message}, - } - yield f"event: content_block_delta\ndata: {json.dumps(error_block_delta)}\n\n" - - yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' - - # Send message_delta and message_stop to properly close the stream - yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "end_turn", "stop_sequence": null}}, "usage": {{"output_tokens": 0}}}}\n\n' - yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' - - # Also send the formal error event for clients that handle it - error_event = { - "type": "error", - "error": {"type": "api_error", "message": str(e)}, - } - yield f"event: error\ndata: {json.dumps(error_event)}\n\n" - - async def streaming_response_wrapper( request: Request, request_data: dict, @@ -1669,68 +1012,11 @@ async def anthropic_messages( This endpoint is compatible with Claude Code and other Anthropic API clients. """ - request_id = f"msg_{uuid.uuid4().hex[:24]}" - original_model = body.model - # Initialize logger if enabled logger = DetailedLogger() if ENABLE_REQUEST_LOGGING else None try: - # Convert Anthropic request to OpenAI format - anthropic_request = body.model_dump(exclude_none=True) - - openai_messages = anthropic_to_openai_messages( - anthropic_request.get("messages", []), anthropic_request.get("system") - ) - - openai_tools = anthropic_to_openai_tools(anthropic_request.get("tools")) - openai_tool_choice = anthropic_to_openai_tool_choice( - anthropic_request.get("tool_choice") - ) - - # Build OpenAI-compatible request - openai_request = { - "model": body.model, - "messages": openai_messages, - "max_tokens": body.max_tokens, - "stream": body.stream or False, - } - - if body.temperature is not None: - openai_request["temperature"] = body.temperature - if body.top_p is not None: - openai_request["top_p"] = body.top_p - if body.stop_sequences: - openai_request["stop"] = body.stop_sequences - if openai_tools: - openai_request["tools"] = openai_tools - if openai_tool_choice: - openai_request["tool_choice"] = openai_tool_choice - - # Handle Anthropic thinking config -> reasoning_effort translation - if body.thinking: - if body.thinking.type == "enabled": - # Map budget_tokens to reasoning_effort level - # Default to "medium" if enabled but budget not specified - budget = body.thinking.budget_tokens or 10000 - if budget >= 32000: - openai_request["reasoning_effort"] = "high" - openai_request["custom_reasoning_budget"] = True - elif budget >= 10000: - openai_request["reasoning_effort"] = "high" - elif budget >= 5000: - openai_request["reasoning_effort"] = "medium" - else: - openai_request["reasoning_effort"] = "low" - elif body.thinking.type == "disabled": - openai_request["reasoning_effort"] = "disable" - elif "opus" in body.model.lower(): - # Force high thinking for Opus models when no thinking config is provided - # Opus 4.5 always uses the -thinking variant, so we want maximum thinking budget - # Without this, the backend defaults to thinkingBudget: -1 (auto) instead of high - openai_request["reasoning_effort"] = "high" - openai_request["custom_reasoning_budget"] = True - + # Log the request to console log_request_to_console( url=str(request.url), headers=dict(request.headers), @@ -1738,17 +1024,16 @@ async def anthropic_messages( request.client.host if request.client else "unknown", request.client.port if request.client else 0, ), - request_data=openai_request, + request_data=body.model_dump(exclude_none=True), ) - if body.stream: - # Streaming response - acompletion returns a generator for streaming - response_generator = client.acompletion(request=request, **openai_request) + # Use the library method to handle the request + result = await client.anthropic_messages(body, raw_request=request) + if body.stream: + # Streaming response return StreamingResponse( - anthropic_streaming_wrapper( - request, response_generator, original_model, request_id - ), + result, media_type="text/event-stream", headers={ "Cache-Control": "no-cache", @@ -1758,29 +1043,13 @@ async def anthropic_messages( ) else: # Non-streaming response - response = await client.acompletion(request=request, **openai_request) - - # Convert OpenAI response to Anthropic format - openai_response = ( - response.model_dump() - if hasattr(response, "model_dump") - else dict(response) - ) - anthropic_response = openai_to_anthropic_response( - openai_response, original_model - ) - - # Override the ID with our request ID - anthropic_response["id"] = request_id - if logger: logger.log_final_response( status_code=200, headers=None, - body=anthropic_response, + body=result, ) - - return JSONResponse(content=anthropic_response) + return JSONResponse(content=result) except ( litellm.InvalidRequestError, @@ -1848,40 +1117,9 @@ async def anthropic_count_tokens( Accepts requests in Anthropic's format and returns token count in Anthropic's format. """ try: - # Convert Anthropic request to OpenAI format for token counting - anthropic_request = body.model_dump(exclude_none=True) - - openai_messages = anthropic_to_openai_messages( - anthropic_request.get("messages", []), anthropic_request.get("system") - ) - - # Count tokens for messages - message_tokens = client.token_count( - model=body.model, - messages=openai_messages, - ) - - # Count tokens for tools if present - tool_tokens = 0 - if body.tools: - # Tools add tokens based on their definitions - # Convert to JSON string and count tokens for tool definitions - openai_tools = anthropic_to_openai_tools( - [tool.model_dump() for tool in body.tools] - ) - if openai_tools: - # Serialize tools to count their token contribution - tools_text = json.dumps(openai_tools) - tool_tokens = client.token_count( - model=body.model, - text=tools_text, - ) - - total_tokens = message_tokens + tool_tokens - - return JSONResponse( - content={"input_tokens": total_tokens} - ) + # Use the library method to handle the request + result = await client.anthropic_count_tokens(body) + return JSONResponse(content=result) except ( litellm.InvalidRequestError, diff --git a/src/rotator_library/__init__.py b/src/rotator_library/__init__.py index 7944443fc..b05e47078 100644 --- a/src/rotator_library/__init__.py +++ b/src/rotator_library/__init__.py @@ -8,6 +8,7 @@ from .providers import PROVIDER_PLUGINS from .providers.provider_interface import ProviderInterface from .model_info_service import ModelInfoService, ModelInfo, ModelMetadata + from . import anthropic_compat __all__ = [ "RotatingClient", @@ -15,11 +16,12 @@ "ModelInfoService", "ModelInfo", "ModelMetadata", + "anthropic_compat", ] def __getattr__(name): - """Lazy-load PROVIDER_PLUGINS and ModelInfoService to speed up module import.""" + """Lazy-load PROVIDER_PLUGINS, ModelInfoService, and anthropic_compat to speed up module import.""" if name == "PROVIDER_PLUGINS": from .providers import PROVIDER_PLUGINS @@ -36,4 +38,8 @@ def __getattr__(name): from .model_info_service import ModelMetadata return ModelMetadata + if name == "anthropic_compat": + from . import anthropic_compat + + return anthropic_compat raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/rotator_library/anthropic_compat/__init__.py b/src/rotator_library/anthropic_compat/__init__.py new file mode 100644 index 000000000..8572ac79d --- /dev/null +++ b/src/rotator_library/anthropic_compat/__init__.py @@ -0,0 +1,67 @@ +""" +Anthropic API compatibility module for rotator_library. + +This module provides format translation between Anthropic's Messages API +and OpenAI's Chat Completions API, enabling any OpenAI-compatible provider +to work with Anthropic clients like Claude Code. + +Usage: + from rotator_library.anthropic_compat import ( + AnthropicMessagesRequest, + AnthropicMessagesResponse, + translate_anthropic_request, + openai_to_anthropic_response, + anthropic_streaming_wrapper, + ) +""" + +from .models import ( + AnthropicTextBlock, + AnthropicImageSource, + AnthropicImageBlock, + AnthropicToolUseBlock, + AnthropicToolResultBlock, + AnthropicMessage, + AnthropicTool, + AnthropicThinkingConfig, + AnthropicMessagesRequest, + AnthropicUsage, + AnthropicMessagesResponse, + AnthropicCountTokensRequest, + AnthropicCountTokensResponse, +) + +from .translator import ( + anthropic_to_openai_messages, + anthropic_to_openai_tools, + anthropic_to_openai_tool_choice, + openai_to_anthropic_response, + translate_anthropic_request, +) + +from .streaming import anthropic_streaming_wrapper + +__all__ = [ + # Models + "AnthropicTextBlock", + "AnthropicImageSource", + "AnthropicImageBlock", + "AnthropicToolUseBlock", + "AnthropicToolResultBlock", + "AnthropicMessage", + "AnthropicTool", + "AnthropicThinkingConfig", + "AnthropicMessagesRequest", + "AnthropicUsage", + "AnthropicMessagesResponse", + "AnthropicCountTokensRequest", + "AnthropicCountTokensResponse", + # Translator functions + "anthropic_to_openai_messages", + "anthropic_to_openai_tools", + "anthropic_to_openai_tool_choice", + "openai_to_anthropic_response", + "translate_anthropic_request", + # Streaming + "anthropic_streaming_wrapper", +] diff --git a/src/rotator_library/anthropic_compat/models.py b/src/rotator_library/anthropic_compat/models.py new file mode 100644 index 000000000..c579f2e2c --- /dev/null +++ b/src/rotator_library/anthropic_compat/models.py @@ -0,0 +1,144 @@ +""" +Pydantic models for the Anthropic Messages API. + +These models define the request and response formats for Anthropic's Messages API, +enabling compatibility with Claude Code and other Anthropic API clients. +""" + +from typing import Any, List, Optional, Union +from pydantic import BaseModel + + +# --- Content Blocks --- +class AnthropicTextBlock(BaseModel): + """Anthropic text content block.""" + + type: str = "text" + text: str + + +class AnthropicImageSource(BaseModel): + """Anthropic image source for base64 images.""" + + type: str = "base64" + media_type: str + data: str + + +class AnthropicImageBlock(BaseModel): + """Anthropic image content block.""" + + type: str = "image" + source: AnthropicImageSource + + +class AnthropicToolUseBlock(BaseModel): + """Anthropic tool use content block.""" + + type: str = "tool_use" + id: str + name: str + input: dict + + +class AnthropicToolResultBlock(BaseModel): + """Anthropic tool result content block.""" + + type: str = "tool_result" + tool_use_id: str + content: Union[str, List[Any]] + is_error: Optional[bool] = None + + +# --- Message and Tool Definitions --- +class AnthropicMessage(BaseModel): + """Anthropic message format.""" + + role: str + content: Union[ + str, + List[ + Union[ + AnthropicTextBlock, + AnthropicImageBlock, + AnthropicToolUseBlock, + AnthropicToolResultBlock, + dict, + ] + ], + ] + + +class AnthropicTool(BaseModel): + """Anthropic tool definition.""" + + name: str + description: Optional[str] = None + input_schema: dict + + +class AnthropicThinkingConfig(BaseModel): + """Anthropic thinking configuration.""" + + type: str # "enabled" or "disabled" + budget_tokens: Optional[int] = None + + +# --- Messages Request --- +class AnthropicMessagesRequest(BaseModel): + """Anthropic Messages API request format.""" + + model: str + messages: List[AnthropicMessage] + max_tokens: int + system: Optional[Union[str, List[dict]]] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + stop_sequences: Optional[List[str]] = None + stream: Optional[bool] = False + tools: Optional[List[AnthropicTool]] = None + tool_choice: Optional[dict] = None + metadata: Optional[dict] = None + thinking: Optional[AnthropicThinkingConfig] = None + + +# --- Messages Response --- +class AnthropicUsage(BaseModel): + """Anthropic usage statistics.""" + + input_tokens: int + output_tokens: int + cache_creation_input_tokens: Optional[int] = None + cache_read_input_tokens: Optional[int] = None + + +class AnthropicMessagesResponse(BaseModel): + """Anthropic Messages API response format.""" + + id: str + type: str = "message" + role: str = "assistant" + content: List[Union[AnthropicTextBlock, AnthropicToolUseBlock, dict]] + model: str + stop_reason: Optional[str] = None + stop_sequence: Optional[str] = None + usage: AnthropicUsage + + +# --- Count Tokens --- +class AnthropicCountTokensRequest(BaseModel): + """Anthropic count_tokens API request format.""" + + model: str + messages: List[AnthropicMessage] + system: Optional[Union[str, List[dict]]] = None + tools: Optional[List[AnthropicTool]] = None + tool_choice: Optional[dict] = None + thinking: Optional[AnthropicThinkingConfig] = None + + +class AnthropicCountTokensResponse(BaseModel): + """Anthropic count_tokens API response format.""" + + input_tokens: int diff --git a/src/rotator_library/anthropic_compat/streaming.py b/src/rotator_library/anthropic_compat/streaming.py new file mode 100644 index 000000000..5ceb71455 --- /dev/null +++ b/src/rotator_library/anthropic_compat/streaming.py @@ -0,0 +1,308 @@ +""" +Streaming wrapper for converting OpenAI streaming format to Anthropic streaming format. + +This module provides a framework-agnostic streaming wrapper that converts +OpenAI SSE (Server-Sent Events) format to Anthropic's streaming format. +""" + +import json +import logging +import uuid +from typing import AsyncGenerator, Callable, Optional, Awaitable + +logger = logging.getLogger("rotator_library.anthropic_compat") + + +async def anthropic_streaming_wrapper( + openai_stream: AsyncGenerator[str, None], + original_model: str, + request_id: Optional[str] = None, + is_disconnected: Optional[Callable[[], Awaitable[bool]]] = None, +) -> AsyncGenerator[str, None]: + """ + Convert OpenAI streaming format to Anthropic streaming format. + + This is a framework-agnostic wrapper that can be used with any async web framework. + Instead of taking a FastAPI Request object, it accepts an optional callback function + to check for client disconnection. + + Anthropic SSE events: + - message_start: Initial message metadata + - content_block_start: Start of a content block + - content_block_delta: Content chunk + - content_block_stop: End of a content block + - message_delta: Final message metadata (stop_reason, usage) + - message_stop: End of message + + Args: + openai_stream: AsyncGenerator yielding OpenAI SSE format strings + original_model: The model name to include in responses + request_id: Optional request ID (auto-generated if not provided) + is_disconnected: Optional async callback that returns True if client disconnected + + Yields: + SSE format strings in Anthropic's streaming format + """ + if request_id is None: + request_id = f"msg_{uuid.uuid4().hex[:24]}" + + message_started = False + content_block_started = False + thinking_block_started = False + current_block_index = 0 + tool_calls_by_index = {} # Track tool calls by their index + tool_block_indices = {} # Track which block index each tool call uses + input_tokens = 0 + output_tokens = 0 + + try: + async for chunk_str in openai_stream: + # Check for client disconnection if callback provided + if is_disconnected is not None and await is_disconnected(): + break + + if not chunk_str.strip() or not chunk_str.startswith("data:"): + continue + + data_content = chunk_str[len("data:") :].strip() + if data_content == "[DONE]": + # CRITICAL: Send message_start if we haven't yet (e.g., empty response) + # Claude Code and other clients require message_start before message_stop + if not message_started: + message_start = { + "type": "message_start", + "message": { + "id": request_id, + "type": "message", + "role": "assistant", + "content": [], + "model": original_model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": input_tokens, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" + message_started = True + + # Close any open thinking block + if thinking_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + thinking_block_started = False + + # Close any open text block + if content_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + content_block_started = False + + # Close all open tool_use blocks + for tc_index in sorted(tool_block_indices.keys()): + block_idx = tool_block_indices[tc_index] + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {block_idx}}}\n\n' + + # Determine stop_reason based on whether we had tool calls + stop_reason = "tool_use" if tool_calls_by_index else "end_turn" + + # Send message_delta with final info + yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "{stop_reason}", "stop_sequence": null}}, "usage": {{"output_tokens": {output_tokens}}}}}\n\n' + + # Send message_stop + yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' + break + + try: + chunk = json.loads(data_content) + except json.JSONDecodeError: + continue + + # Extract usage if present + if "usage" in chunk and chunk["usage"]: + input_tokens = chunk["usage"].get("prompt_tokens", input_tokens) + output_tokens = chunk["usage"].get("completion_tokens", output_tokens) + + # Send message_start on first chunk + if not message_started: + message_start = { + "type": "message_start", + "message": { + "id": request_id, + "type": "message", + "role": "assistant", + "content": [], + "model": original_model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": input_tokens, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" + message_started = True + + choices = chunk.get("choices", []) + if not choices: + continue + + delta = choices[0].get("delta", {}) + + # Handle reasoning/thinking content (from OpenAI-style reasoning_content) + reasoning_content = delta.get("reasoning_content") + if reasoning_content: + if not thinking_block_started: + # Start a thinking content block + block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "thinking", "thinking": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + thinking_block_started = True + + # Send thinking delta + block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "thinking_delta", "thinking": reasoning_content}, + } + yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + + # Handle text content + content = delta.get("content") + if content: + # If we were in a thinking block, close it first + if thinking_block_started and not content_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + thinking_block_started = False + + if not content_block_started: + # Start a text content block + block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "text", "text": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + content_block_started = True + + # Send content delta + block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "text_delta", "text": content}, + } + yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + + # Handle tool calls + tool_calls = delta.get("tool_calls", []) + for tc in tool_calls: + tc_index = tc.get("index", 0) + + if tc_index not in tool_calls_by_index: + # Close previous thinking block if open + if thinking_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + thinking_block_started = False + + # Close previous text block if open + if content_block_started: + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + current_block_index += 1 + content_block_started = False + + # Start new tool use block + tool_calls_by_index[tc_index] = { + "id": tc.get("id", f"toolu_{uuid.uuid4().hex[:12]}"), + "name": tc.get("function", {}).get("name", ""), + "arguments": "", + } + # Track which block index this tool call uses + tool_block_indices[tc_index] = current_block_index + + block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": { + "type": "tool_use", + "id": tool_calls_by_index[tc_index]["id"], + "name": tool_calls_by_index[tc_index]["name"], + "input": {}, + }, + } + yield f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n" + # Increment for the next block + current_block_index += 1 + + # Accumulate arguments + func = tc.get("function", {}) + if func.get("name"): + tool_calls_by_index[tc_index]["name"] = func["name"] + if func.get("arguments"): + tool_calls_by_index[tc_index]["arguments"] += func["arguments"] + + # Send partial JSON delta using the correct block index for this tool + block_delta = { + "type": "content_block_delta", + "index": tool_block_indices[tc_index], + "delta": { + "type": "input_json_delta", + "partial_json": func["arguments"], + }, + } + yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + + # Note: We intentionally ignore finish_reason here. + # Block closing is handled when we receive [DONE] to avoid + # premature closes with providers that send finish_reason on each chunk. + + except Exception as e: + logger.error(f"Error in Anthropic streaming wrapper: {e}") + + # If we haven't sent message_start yet, send it now so the client can display the error + # Claude Code and other clients may ignore events that come before message_start + if not message_started: + message_start = { + "type": "message_start", + "message": { + "id": request_id, + "type": "message", + "role": "assistant", + "content": [], + "model": original_model, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 0, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" + + # Send the error as a text content block so it's visible to the user + error_message = f"Error: {str(e)}" + error_block_start = { + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "text", "text": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(error_block_start)}\n\n" + + error_block_delta = { + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "text_delta", "text": error_message}, + } + yield f"event: content_block_delta\ndata: {json.dumps(error_block_delta)}\n\n" + + yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + + # Send message_delta and message_stop to properly close the stream + yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "end_turn", "stop_sequence": null}}, "usage": {{"output_tokens": 0}}}}\n\n' + yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' + + # Also send the formal error event for clients that handle it + error_event = { + "type": "error", + "error": {"type": "api_error", "message": str(e)}, + } + yield f"event: error\ndata: {json.dumps(error_event)}\n\n" diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py new file mode 100644 index 000000000..451abfab5 --- /dev/null +++ b/src/rotator_library/anthropic_compat/translator.py @@ -0,0 +1,363 @@ +""" +Format translation functions between Anthropic and OpenAI API formats. + +This module provides functions to convert requests and responses between +Anthropic's Messages API format and OpenAI's Chat Completions API format. +This enables any OpenAI-compatible provider to work with Anthropic clients. +""" + +import json +import uuid +from typing import Any, Dict, List, Optional, Union + +from .models import AnthropicMessagesRequest + + +def anthropic_to_openai_messages( + anthropic_messages: List[dict], system: Optional[Union[str, List[dict]]] = None +) -> List[dict]: + """ + Convert Anthropic message format to OpenAI format. + + Key differences: + - Anthropic: system is a separate field, content can be string or list of blocks + - OpenAI: system is a message with role="system", content is usually string + + Args: + anthropic_messages: List of messages in Anthropic format + system: Optional system message (string or list of text blocks) + + Returns: + List of messages in OpenAI format + """ + openai_messages = [] + + # Handle system message + if system: + if isinstance(system, str): + openai_messages.append({"role": "system", "content": system}) + elif isinstance(system, list): + # System can be list of text blocks in Anthropic format + system_text = " ".join( + block.get("text", "") + for block in system + if isinstance(block, dict) and block.get("type") == "text" + ) + if system_text: + openai_messages.append({"role": "system", "content": system_text}) + + for msg in anthropic_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + if isinstance(content, str): + openai_messages.append({"role": role, "content": content}) + elif isinstance(content, list): + # Handle content blocks + openai_content = [] + tool_calls = [] + + for block in content: + if isinstance(block, dict): + block_type = block.get("type", "text") + + if block_type == "text": + openai_content.append( + {"type": "text", "text": block.get("text", "")} + ) + elif block_type == "image": + # Convert Anthropic image format to OpenAI + source = block.get("source", {}) + if source.get("type") == "base64": + openai_content.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{source.get('media_type', 'image/png')};base64,{source.get('data', '')}" + }, + } + ) + elif source.get("type") == "url": + openai_content.append( + { + "type": "image_url", + "image_url": {"url": source.get("url", "")}, + } + ) + elif block_type == "tool_use": + # Anthropic tool_use -> OpenAI tool_calls + tool_calls.append( + { + "id": block.get("id", ""), + "type": "function", + "function": { + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }, + } + ) + elif block_type == "tool_result": + # Tool results become separate messages in OpenAI format + tool_content = block.get("content", "") + if isinstance(tool_content, list): + tool_content = " ".join( + b.get("text", "") + for b in tool_content + if isinstance(b, dict) and b.get("type") == "text" + ) + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": str(tool_content), + } + ) + continue # Don't add to current message + + # Build the message + if tool_calls: + # Assistant message with tool calls + msg_dict = {"role": role} + if openai_content: + # If there's text content alongside tool calls + text_parts = [ + c.get("text", "") + for c in openai_content + if c.get("type") == "text" + ] + msg_dict["content"] = " ".join(text_parts) if text_parts else None + else: + msg_dict["content"] = None + msg_dict["tool_calls"] = tool_calls + openai_messages.append(msg_dict) + elif openai_content: + # Check if it's just text or mixed content + if len(openai_content) == 1 and openai_content[0].get("type") == "text": + openai_messages.append( + {"role": role, "content": openai_content[0].get("text", "")} + ) + else: + openai_messages.append({"role": role, "content": openai_content}) + + return openai_messages + + +def anthropic_to_openai_tools( + anthropic_tools: Optional[List[dict]], +) -> Optional[List[dict]]: + """ + Convert Anthropic tool definitions to OpenAI format. + + Args: + anthropic_tools: List of tools in Anthropic format + + Returns: + List of tools in OpenAI format, or None if no tools provided + """ + if not anthropic_tools: + return None + + openai_tools = [] + for tool in anthropic_tools: + openai_tools.append( + { + "type": "function", + "function": { + "name": tool.get("name", ""), + "description": tool.get("description", ""), + "parameters": tool.get("input_schema", {}), + }, + } + ) + return openai_tools + + +def anthropic_to_openai_tool_choice( + anthropic_tool_choice: Optional[dict], +) -> Optional[Union[str, dict]]: + """ + Convert Anthropic tool_choice to OpenAI format. + + Args: + anthropic_tool_choice: Tool choice in Anthropic format + + Returns: + Tool choice in OpenAI format + """ + if not anthropic_tool_choice: + return None + + choice_type = anthropic_tool_choice.get("type", "auto") + + if choice_type == "auto": + return "auto" + elif choice_type == "any": + return "required" + elif choice_type == "tool": + return { + "type": "function", + "function": {"name": anthropic_tool_choice.get("name", "")}, + } + elif choice_type == "none": + return "none" + + return "auto" + + +def openai_to_anthropic_response(openai_response: dict, original_model: str) -> dict: + """ + Convert OpenAI chat completion response to Anthropic Messages format. + + Args: + openai_response: Response from OpenAI-compatible API + original_model: The model name requested by the client + + Returns: + Response in Anthropic Messages format + """ + choice = openai_response.get("choices", [{}])[0] + message = choice.get("message", {}) + usage = openai_response.get("usage", {}) + + # Build content blocks + content_blocks = [] + + # Add thinking content block if reasoning_content is present + reasoning_content = message.get("reasoning_content") + if reasoning_content: + content_blocks.append( + { + "type": "thinking", + "thinking": reasoning_content, + "signature": "", # Signature is typically empty for proxied responses + } + ) + + # Add text content if present + text_content = message.get("content") + if text_content: + content_blocks.append({"type": "text", "text": text_content}) + + # Add tool use blocks if present + tool_calls = message.get("tool_calls") or [] + for tc in tool_calls: + func = tc.get("function", {}) + try: + input_data = json.loads(func.get("arguments", "{}")) + except json.JSONDecodeError: + input_data = {} + + content_blocks.append( + { + "type": "tool_use", + "id": tc.get("id", f"toolu_{uuid.uuid4().hex[:12]}"), + "name": func.get("name", ""), + "input": input_data, + } + ) + + # Map finish_reason to stop_reason + finish_reason = choice.get("finish_reason", "end_turn") + stop_reason_map = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "content_filter": "end_turn", + "function_call": "tool_use", + } + stop_reason = stop_reason_map.get(finish_reason, "end_turn") + + # Build usage + anthropic_usage = { + "input_tokens": usage.get("prompt_tokens", 0), + "output_tokens": usage.get("completion_tokens", 0), + } + + # Add cache tokens if present + if usage.get("prompt_tokens_details"): + details = usage["prompt_tokens_details"] + if details.get("cached_tokens"): + anthropic_usage["cache_read_input_tokens"] = details["cached_tokens"] + + return { + "id": openai_response.get("id", f"msg_{uuid.uuid4().hex[:24]}"), + "type": "message", + "role": "assistant", + "content": content_blocks, + "model": original_model, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": anthropic_usage, + } + + +def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, Any]: + """ + Translate a complete Anthropic Messages API request to OpenAI format. + + This is a high-level function that handles all aspects of request translation, + including messages, tools, tool_choice, and thinking configuration. + + Args: + request: An AnthropicMessagesRequest object + + Returns: + Dictionary containing the OpenAI-compatible request parameters + """ + anthropic_request = request.model_dump(exclude_none=True) + + openai_messages = anthropic_to_openai_messages( + anthropic_request.get("messages", []), anthropic_request.get("system") + ) + + openai_tools = anthropic_to_openai_tools(anthropic_request.get("tools")) + openai_tool_choice = anthropic_to_openai_tool_choice( + anthropic_request.get("tool_choice") + ) + + # Build OpenAI-compatible request + openai_request = { + "model": request.model, + "messages": openai_messages, + "max_tokens": request.max_tokens, + "stream": request.stream or False, + } + + if request.temperature is not None: + openai_request["temperature"] = request.temperature + if request.top_p is not None: + openai_request["top_p"] = request.top_p + if request.top_k is not None: + openai_request["top_k"] = request.top_k + if request.stop_sequences: + openai_request["stop"] = request.stop_sequences + if openai_tools: + openai_request["tools"] = openai_tools + if openai_tool_choice: + openai_request["tool_choice"] = openai_tool_choice + + # Handle Anthropic thinking config -> reasoning_effort translation + if request.thinking: + if request.thinking.type == "enabled": + # Map budget_tokens to reasoning_effort level + # Default to "medium" if enabled but budget not specified + budget = request.thinking.budget_tokens or 10000 + if budget >= 32000: + openai_request["reasoning_effort"] = "high" + openai_request["custom_reasoning_budget"] = True + elif budget >= 10000: + openai_request["reasoning_effort"] = "high" + elif budget >= 5000: + openai_request["reasoning_effort"] = "medium" + else: + openai_request["reasoning_effort"] = "low" + elif request.thinking.type == "disabled": + openai_request["reasoning_effort"] = "disable" + elif "opus" in request.model.lower(): + # Force high thinking for Opus models when no thinking config is provided + # Opus 4.5 always uses the -thinking variant, so we want maximum thinking budget + # Without this, the backend defaults to thinkingBudget: -1 (auto) instead of high + openai_request["reasoning_effort"] = "high" + openai_request["custom_reasoning_budget"] = True + + return openai_request diff --git a/src/rotator_library/client.py b/src/rotator_library/client.py index 49d61795f..f06313afe 100644 --- a/src/rotator_library/client.py +++ b/src/rotator_library/client.py @@ -3017,3 +3017,133 @@ async def force_refresh_quota( result["duration_ms"] = int((time.time() - start_time) * 1000) return result + + # --- Anthropic API Compatibility Methods --- + + async def anthropic_messages( + self, + request: "AnthropicMessagesRequest", + raw_request: Optional[Any] = None, + pre_request_callback: Optional[callable] = None, + ) -> Any: + """ + Handle Anthropic Messages API requests. + + This method accepts requests in Anthropic's format, translates them to + OpenAI format internally, processes them through the existing acompletion + method, and returns responses in Anthropic's format. + + Args: + request: An AnthropicMessagesRequest object + raw_request: Optional raw request object for disconnect checks + pre_request_callback: Optional async callback before each API request + + Returns: + For non-streaming: dict in Anthropic Messages format + For streaming: AsyncGenerator yielding Anthropic SSE format strings + """ + from .anthropic_compat import ( + translate_anthropic_request, + openai_to_anthropic_response, + anthropic_streaming_wrapper, + ) + import uuid + + request_id = f"msg_{uuid.uuid4().hex[:24]}" + original_model = request.model + + # Translate Anthropic request to OpenAI format + openai_request = translate_anthropic_request(request) + + if request.stream: + # Streaming response + response_generator = self.acompletion( + request=raw_request, + pre_request_callback=pre_request_callback, + **openai_request, + ) + + # Create disconnect checker if raw_request provided + is_disconnected = None + if raw_request is not None and hasattr(raw_request, "is_disconnected"): + is_disconnected = raw_request.is_disconnected + + # Return the streaming wrapper + return anthropic_streaming_wrapper( + openai_stream=response_generator, + original_model=original_model, + request_id=request_id, + is_disconnected=is_disconnected, + ) + else: + # Non-streaming response + response = await self.acompletion( + request=raw_request, + pre_request_callback=pre_request_callback, + **openai_request, + ) + + # Convert OpenAI response to Anthropic format + openai_response = ( + response.model_dump() if hasattr(response, "model_dump") else dict(response) + ) + anthropic_response = openai_to_anthropic_response(openai_response, original_model) + + # Override the ID with our request ID + anthropic_response["id"] = request_id + + return anthropic_response + + async def anthropic_count_tokens( + self, + request: "AnthropicCountTokensRequest", + ) -> dict: + """ + Handle Anthropic count_tokens API requests. + + Counts the number of tokens that would be used by a Messages API request. + This is useful for estimating costs and managing context windows. + + Args: + request: An AnthropicCountTokensRequest object + + Returns: + Dict with input_tokens count in Anthropic format + """ + from .anthropic_compat import ( + anthropic_to_openai_messages, + anthropic_to_openai_tools, + ) + import json + + anthropic_request = request.model_dump(exclude_none=True) + + openai_messages = anthropic_to_openai_messages( + anthropic_request.get("messages", []), anthropic_request.get("system") + ) + + # Count tokens for messages + message_tokens = self.token_count( + model=request.model, + messages=openai_messages, + ) + + # Count tokens for tools if present + tool_tokens = 0 + if request.tools: + # Tools add tokens based on their definitions + # Convert to JSON string and count tokens for tool definitions + openai_tools = anthropic_to_openai_tools( + [tool.model_dump() for tool in request.tools] + ) + if openai_tools: + # Serialize tools to count their token contribution + tools_text = json.dumps(openai_tools) + tool_tokens = self.token_count( + model=request.model, + text=tools_text, + ) + + total_tokens = message_tokens + tool_tokens + + return {"input_tokens": total_tokens} From d91f98bcb57dcd16ab523ae0461a3158d17e4796 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Sat, 20 Dec 2025 22:18:56 +0100 Subject: [PATCH 15/36] fix(anthropic): improve model detection and document thinking budget - Add comment explaining metadata parameter is intentionally not mapped (OpenAI doesn't have an equivalent field) - Use safer regex pattern matching for Opus model detection (avoids false positives like "magnum-opus-model") - Document reasoning budget thresholds and // 4 reduction behavior - Conserve thinking tokens for Opus auto-detection (use // 4 like other models) Only set custom_reasoning_budget=True when user explicitly requests 32000+ tokens --- .../anthropic_compat/translator.py | 59 ++++++++++++++++--- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 451abfab5..54f077d42 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -336,28 +336,73 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, if openai_tool_choice: openai_request["tool_choice"] = openai_tool_choice + # Note: request.metadata is intentionally not mapped. + # OpenAI's API doesn't have an equivalent field for client-side metadata. + # The metadata is typically used by Anthropic clients for tracking purposes + # and doesn't affect the model's behavior. + # Handle Anthropic thinking config -> reasoning_effort translation + # The provider (antigravity_provider.py) applies a // 4 reduction to thinking budget + # unless custom_reasoning_budget is True. This conserves thinking tokens. + # + # Reasoning budget thresholds map to provider budgets: + # - Claude "high" = 32768 tokens (but // 4 = 8192 unless custom_reasoning_budget) + # - Claude "medium" = 16384 tokens (// 4 = 4096) + # - Claude "low" = 8192 tokens (// 4 = 2048) + # + # We only set custom_reasoning_budget=True when user explicitly requests + # a large budget (32000+), indicating they want full thinking capacity. if request.thinking: if request.thinking.type == "enabled": - # Map budget_tokens to reasoning_effort level - # Default to "medium" if enabled but budget not specified budget = request.thinking.budget_tokens or 10000 if budget >= 32000: + # User explicitly wants full thinking capacity openai_request["reasoning_effort"] = "high" openai_request["custom_reasoning_budget"] = True elif budget >= 10000: openai_request["reasoning_effort"] = "high" + # custom_reasoning_budget defaults to False, so // 4 applies elif budget >= 5000: openai_request["reasoning_effort"] = "medium" else: openai_request["reasoning_effort"] = "low" elif request.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" - elif "opus" in request.model.lower(): - # Force high thinking for Opus models when no thinking config is provided - # Opus 4.5 always uses the -thinking variant, so we want maximum thinking budget - # Without this, the backend defaults to thinkingBudget: -1 (auto) instead of high + elif _is_opus_model(request.model): + # Enable thinking for Opus models when no thinking config is provided + # Use "high" effort but NOT custom_reasoning_budget, so // 4 applies + # This gives 8192 thinking tokens (32768 // 4) which is reasonable for most tasks + # Users who want full capacity can explicitly set thinking.budget_tokens >= 32000 openai_request["reasoning_effort"] = "high" - openai_request["custom_reasoning_budget"] = True + # Note: NOT setting custom_reasoning_budget here to conserve tokens return openai_request + + +def _is_opus_model(model_name: str) -> bool: + """ + Check if a model name refers to a Claude Opus model. + + Uses specific pattern matching to avoid false positives with model names + that might contain "opus" as part of another word. + + Args: + model_name: The model name to check + + Returns: + True if the model is a Claude Opus model, False otherwise + """ + import re + + model_lower = model_name.lower() + # Match Claude Opus models specifically: + # - "claude-opus-4-5", "claude-4-opus", "claude_opus" + # - "opus-4", "opus-4.5", "opus4" (standalone with version) + # - "antigravity/claude-opus-4-5" + # Avoid matching things like "magnum-opus" or other non-Claude models + opus_patterns = [ + r'claude[-_]?opus', # "claude-opus", "claude_opus", "claudeopus" + r'opus[-_]?\d', # "opus-4", "opus_4", "opus4" (with version number) + r'\d[-_]?opus(?:[-_]|$)', # "4-opus", "4_opus" at word boundary + ] + return any(re.search(pattern, model_lower) for pattern in opus_patterns) From 16c889f367669321b74161d578411f99b5da5aaf Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Tue, 23 Dec 2025 00:55:16 +0100 Subject: [PATCH 16/36] fix(anthropic): handle images in tool results for Claude Code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool results with images (e.g., from Read tool) were being dropped during Anthropic→OpenAI translation, and not properly converted to Gemini format. - translator.py: Extract image blocks from tool_result content and convert to OpenAI image_url format - antigravity_provider.py: Handle multimodal tool responses by converting image_url to Gemini inlineData format --- .../anthropic_compat/translator.py | 92 ++++++++++++++++--- .../providers/antigravity_provider.py | 57 +++++++++++- 2 files changed, 134 insertions(+), 15 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 54f077d42..70fa1cfba 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -98,20 +98,88 @@ def anthropic_to_openai_messages( ) elif block_type == "tool_result": # Tool results become separate messages in OpenAI format + # Content can be string, or list of text/image blocks tool_content = block.get("content", "") - if isinstance(tool_content, list): - tool_content = " ".join( - b.get("text", "") - for b in tool_content - if isinstance(b, dict) and b.get("type") == "text" + if isinstance(tool_content, str): + # Simple string content + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": tool_content, + } + ) + elif isinstance(tool_content, list): + # List of content blocks - may include text and images + tool_content_parts = [] + for b in tool_content: + if not isinstance(b, dict): + continue + b_type = b.get("type", "") + if b_type == "text": + tool_content_parts.append( + {"type": "text", "text": b.get("text", "")} + ) + elif b_type == "image": + # Convert Anthropic image format to OpenAI format + source = b.get("source", {}) + if source.get("type") == "base64": + tool_content_parts.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{source.get('media_type', 'image/png')};base64,{source.get('data', '')}" + }, + } + ) + elif source.get("type") == "url": + tool_content_parts.append( + { + "type": "image_url", + "image_url": {"url": source.get("url", "")}, + } + ) + + # If we only have text parts, join them as a string for compatibility + # Otherwise use the array format for multimodal content + if all(p.get("type") == "text" for p in tool_content_parts): + combined_text = " ".join( + p.get("text", "") for p in tool_content_parts + ) + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": combined_text, + } + ) + elif tool_content_parts: + # Multimodal content (includes images) + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": tool_content_parts, + } + ) + else: + # Empty content + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": "", + } + ) + else: + # Fallback for unexpected content type + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.get("tool_use_id", ""), + "content": str(tool_content) if tool_content else "", + } ) - openai_messages.append( - { - "role": "tool", - "tool_call_id": block.get("tool_use_id", ""), - "content": str(tool_content), - } - ) continue # Don't add to current message # Build the message diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 874e910a8..83b585af9 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -2438,7 +2438,12 @@ def _get_cached_thinking( def _transform_tool_message( self, msg: Dict[str, Any], model: str, tool_id_to_name: Dict[str, str] ) -> List[Dict[str, Any]]: - """Transform tool response message.""" + """Transform tool response message. + + Handles both text-only and multimodal (text + images) tool responses. + For multimodal responses, images are converted to inlineData format + and returned as separate parts alongside the functionResponse. + """ tool_id = msg.get("tool_call_id", "") func_name = tool_id_to_name.get(tool_id, "unknown_function") content = msg.get("content", "{}") @@ -2449,14 +2454,60 @@ def _transform_tool_message( f"[ID Mismatch] Tool response has ID '{tool_id}' which was not found in tool_id_to_name map. " f"Available IDs: {list(tool_id_to_name.keys())}" ) - # else: - # lib_logger.debug(f"[ID Mapping] Tool response matched: id={tool_id}, name={func_name}") # Add prefix for Gemini 3 (and rename problematic tools) if self._is_gemini_3(model) and self._enable_gemini3_tool_fix: func_name = GEMINI3_TOOL_RENAMES.get(func_name, func_name) func_name = f"{self._gemini3_tool_prefix}{func_name}" + # Handle multimodal content (array with text and images) + if isinstance(content, list): + text_parts = [] + image_parts = [] + + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type", "") + + if item_type == "text": + text_parts.append(item.get("text", "")) + elif item_type == "image_url": + # Convert OpenAI image_url format to Gemini inlineData + image_url = item.get("image_url", {}).get("url", "") + if image_url.startswith("data:"): + try: + # Parse: data:image/png;base64,iVBORw0KG... + header, data = image_url.split(",", 1) + mime_type = header.split(":")[1].split(";")[0] + image_parts.append({ + "inlineData": { + "mimeType": mime_type, + "data": data, + } + }) + except Exception as e: + lib_logger.warning(f"Failed to parse image data URL in tool response: {e}") + + # Build the result parts + parts = [] + + # Add function response with text content + text_result = " ".join(text_parts) if text_parts else "" + parts.append({ + "functionResponse": { + "name": func_name, + "response": {"result": text_result if text_result else "Image content provided"}, + "id": tool_id, + } + }) + + # Add image parts separately (Gemini handles these as additional parts) + parts.extend(image_parts) + + return parts + + # Handle string content (text-only) try: parsed_content = json.loads(content) except (json.JSONDecodeError, TypeError): From 545d0d5b6f292aa6a42e74270fc9f227205fb127 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Wed, 31 Dec 2025 20:23:01 +0100 Subject: [PATCH 17/36] fix(anthropic): force Claude thinking budget and interleaved hint - Force default Claude thinking budget to 31999 when thinking is enabled - Inject interleaved thinking hint for Claude tool calls - Log request headers and raw/unwrapped Claude responses for debugging - Preserve thinking signatures across Anthropic compat translation - Improve thinking signature validation/filtering in Antigravity provider Signed-off-by: Moeeze Hassan --- .../anthropic_compat/translator.py | 53 ++- .../providers/antigravity_provider.py | 368 ++++++++++++++++-- 2 files changed, 376 insertions(+), 45 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 70fa1cfba..574b12503 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -12,6 +12,8 @@ from .models import AnthropicMessagesRequest +MIN_THINKING_SIGNATURE_LENGTH = 100 + def anthropic_to_openai_messages( anthropic_messages: List[dict], system: Optional[Union[str, List[dict]]] = None @@ -56,6 +58,8 @@ def anthropic_to_openai_messages( # Handle content blocks openai_content = [] tool_calls = [] + reasoning_content = "" + thinking_signature = "" for block in content: if isinstance(block, dict): @@ -84,6 +88,17 @@ def anthropic_to_openai_messages( "image_url": {"url": source.get("url", "")}, } ) + elif block_type == "thinking": + signature = block.get("signature", "") + if signature and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH: + thinking_text = block.get("thinking", "") + if thinking_text: + reasoning_content += thinking_text + thinking_signature = signature + elif block_type == "redacted_thinking": + signature = block.get("signature", "") + if signature and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH: + thinking_signature = signature elif block_type == "tool_use": # Anthropic tool_use -> OpenAI tool_calls tool_calls.append( @@ -196,16 +211,37 @@ def anthropic_to_openai_messages( msg_dict["content"] = " ".join(text_parts) if text_parts else None else: msg_dict["content"] = None + if reasoning_content: + msg_dict["reasoning_content"] = reasoning_content + if thinking_signature: + msg_dict["thinking_signature"] = thinking_signature msg_dict["tool_calls"] = tool_calls openai_messages.append(msg_dict) elif openai_content: # Check if it's just text or mixed content if len(openai_content) == 1 and openai_content[0].get("type") == "text": - openai_messages.append( - {"role": role, "content": openai_content[0].get("text", "")} - ) + msg_dict = { + "role": role, + "content": openai_content[0].get("text", ""), + } + if reasoning_content: + msg_dict["reasoning_content"] = reasoning_content + if thinking_signature: + msg_dict["thinking_signature"] = thinking_signature + openai_messages.append(msg_dict) else: - openai_messages.append({"role": role, "content": openai_content}) + msg_dict = {"role": role, "content": openai_content} + if reasoning_content: + msg_dict["reasoning_content"] = reasoning_content + if thinking_signature: + msg_dict["thinking_signature"] = thinking_signature + openai_messages.append(msg_dict) + elif reasoning_content: + msg_dict = {"role": role, "content": ""} + msg_dict["reasoning_content"] = reasoning_content + if thinking_signature: + msg_dict["thinking_signature"] = thinking_signature + openai_messages.append(msg_dict) return openai_messages @@ -293,11 +329,18 @@ def openai_to_anthropic_response(openai_response: dict, original_model: str) -> # Add thinking content block if reasoning_content is present reasoning_content = message.get("reasoning_content") if reasoning_content: + thinking_signature = message.get("thinking_signature", "") + signature = ( + thinking_signature + if thinking_signature + and len(thinking_signature) >= MIN_THINKING_SIGNATURE_LENGTH + else "" + ) content_blocks.append( { "type": "thinking", "thinking": reasoning_content, - "signature": "", # Signature is typically empty for proxied responses + "signature": signature, } ) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 83b585af9..3189fbdcc 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -138,6 +138,14 @@ def _env_int(key: str, default: int) -> int: # When Gemini 3 returns MALFORMED_FUNCTION_CALL (invalid JSON syntax in tool args), # inject corrective messages and retry up to this many times MALFORMED_CALL_MAX_RETRIES = max(1, _env_int("ANTIGRAVITY_MALFORMED_CALL_RETRIES", 2)) + +# Claude thinking signatures must be long enough to be valid +MIN_THINKING_SIGNATURE_LENGTH = 100 +CLAUDE_FORCED_THINKING_BUDGET = 31999 + + +def _is_valid_thinking_signature(signature): + return isinstance(signature, str) and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH MALFORMED_CALL_RETRY_DELAY = _env_int("ANTIGRAVITY_MALFORMED_CALL_DELAY", 1) # Model alias mappings (internal ↔ public) @@ -281,6 +289,9 @@ def _get_claude_thinking_cache_file(): # Parallel tool usage encouragement instruction DEFAULT_PARALLEL_TOOL_INSTRUCTION = """When multiple independent operations are needed, prefer making parallel tool calls in a single response rather than sequential calls across multiple responses. This reduces round-trips and improves efficiency. Only use sequential calls when one tool's output is required as input for another.""" +# Claude interleaved thinking hint (encourages thinking after tool results) +DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """Interleaved thinking is enabled. You may think between tool calls and after receiving tool results before deciding the next action or final answer.""" + # ============================================================================= # HELPER FUNCTIONS @@ -685,6 +696,10 @@ def log_response_chunk(self, chunk: str) -> None: """Append a raw chunk to the response stream log.""" self._append_text("response_stream.log", chunk) + def log_unwrapped_stream_chunk(self, chunk: Dict[str, Any]) -> None: + """Append an unwrapped response chunk as JSON.""" + self._append_text("response_stream_unwrapped.log", json.dumps(chunk)) + def log_error(self, error_message: str) -> None: """Log an error message.""" self._append_text( @@ -705,6 +720,17 @@ def log_final_response(self, response: Dict[str, Any]) -> None: """Log the final response.""" self._write_json("final_response.json", response) + def log_request_headers(self, headers: Dict[str, str]) -> None: + """Log sanitized request headers (no auth tokens).""" + sanitized = dict(headers or {}) + if "Authorization" in sanitized: + sanitized["Authorization"] = "***" + self._write_json("request_headers.json", sanitized) + + def log_raw_response(self, response: Dict[str, Any], filename: str) -> None: + """Log raw response payload.""" + self._write_json(filename, response) + def log_malformed_autofix( self, tool_name: str, raw_args: str, fixed_json: str ) -> None: @@ -1114,6 +1140,13 @@ def __init__(self): self._claude_system_instruction = os.getenv( "ANTIGRAVITY_CLAUDE_SYSTEM_INSTRUCTION", DEFAULT_CLAUDE_SYSTEM_INSTRUCTION ) + self._enable_claude_interleaved_hint = _env_bool( + "ANTIGRAVITY_ENABLE_CLAUDE_INTERLEAVED_HINT", True + ) + self._claude_interleaved_hint = os.getenv( + "ANTIGRAVITY_CLAUDE_INTERLEAVED_HINT", + DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT, + ) # Parallel tool usage instruction configuration self._enable_parallel_tool_instruction_claude = _env_bool( @@ -1139,7 +1172,8 @@ def _log_config(self) -> None: f"gemini3_fix={self._enable_gemini3_tool_fix}, gemini3_strict_schema={self._gemini3_enforce_strict_schema}, " f"claude_fix={self._enable_claude_tool_fix}, thinking_sanitization={self._enable_thinking_sanitization}, " f"parallel_tool_claude={self._enable_parallel_tool_instruction_claude}, " - f"parallel_tool_gemini3={self._enable_parallel_tool_instruction_gemini3}" + f"parallel_tool_gemini3={self._enable_parallel_tool_instruction_gemini3}, " + f"claude_interleaved_hint={self._enable_claude_interleaved_hint}" ) def _get_antigravity_headers(self) -> Dict[str, str]: @@ -1586,7 +1620,18 @@ def _message_has_thinking(self, msg: Dict[str, Any]) -> bool: """ parts = msg.get("parts", []) for part in parts: - if isinstance(part, dict) and part.get("thought") is True: + if not isinstance(part, dict): + continue + + is_thought = part.get("thought") is True or part.get("type") in ( + "thinking", + "redacted_thinking", + ) + if not is_thought: + continue + + signature = part.get("thoughtSignature") or part.get("signature") + if _is_valid_thinking_signature(signature): return True return False @@ -1595,6 +1640,52 @@ def _message_has_tool_calls(self, msg: Dict[str, Any]) -> bool: parts = msg.get("parts", []) return any(isinstance(p, dict) and "functionCall" in p for p in parts) + def _filter_unsigned_thinking_blocks(self, messages): + """ + Drop thinking parts without valid signatures to avoid Claude rejections. + + Handles GEMINI format: role "model", "parts" with thought/thoughtSignature. + """ + for msg in messages: + if msg.get("role") != "model": + continue + + parts = msg.get("parts", []) + if not parts: + continue + + filtered = [] + removed = False + for part in parts: + if not isinstance(part, dict): + filtered.append(part) + continue + + is_thought = part.get("thought") is True or part.get("type") in ( + "thinking", + "redacted_thinking", + ) + if is_thought: + signature = part.get("thoughtSignature") or part.get("signature") + if _is_valid_thinking_signature(signature): + filtered.append(part) + else: + removed = True + continue + + filtered.append(part) + + if removed: + has_function_calls = any( + isinstance(p, dict) and "functionCall" in p for p in filtered + ) + if not filtered: + msg["parts"] = [{"text": ""}] if not has_function_calls else [] + else: + msg["parts"] = filtered + + return messages + def _sanitize_thinking_for_claude( self, messages: List[Dict[str, Any]], thinking_enabled: bool ) -> Tuple[List[Dict[str, Any]], bool]: @@ -1624,6 +1715,7 @@ def _sanitize_thinking_for_claude( - force_disable_thinking: If True, thinking must be disabled for this request """ messages = copy.deepcopy(messages) + messages = self._filter_unsigned_thinking_blocks(messages) state = self._analyze_conversation_state(messages) lib_logger.debug( @@ -1801,7 +1893,13 @@ def _strip_all_thinking_blocks( filtered = [ p for p in parts - if not (isinstance(p, dict) and p.get("thought") is True) + if not ( + isinstance(p, dict) + and ( + p.get("thought") is True + or p.get("type") in ("thinking", "redacted_thinking") + ) + ) ] # Check if there are still functionCalls remaining @@ -1838,7 +1936,13 @@ def _strip_old_turn_thinking( filtered = [ p for p in parts - if not (isinstance(p, dict) and p.get("thought") is True) + if not ( + isinstance(p, dict) + and ( + p.get("thought") is True + or p.get("type") in ("thinking", "redacted_thinking") + ) + ) ] has_function_calls = any( @@ -1881,7 +1985,13 @@ def _preserve_turn_start_thinking( filtered = [ p for p in parts - if not (isinstance(p, dict) and p.get("thought") is True) + if not ( + isinstance(p, dict) + and ( + p.get("thought") is True + or p.get("type") in ("thinking", "redacted_thinking") + ) + ) ] has_function_calls = any( @@ -1922,6 +2032,7 @@ def _looks_like_compacted_thinking_turn(self, msg: Dict[str, Any]) -> bool: and "text" in p and p.get("text", "").strip() and not p.get("thought") # Exclude thinking text + and p.get("type") not in ("thinking", "redacted_thinking") for p in parts ) @@ -1987,7 +2098,7 @@ def _try_recover_thinking_from_cache( thinking_text = thinking_data.get("thinking_text", "") signature = thinking_data.get("thought_signature", "") - if not thinking_text or not signature: + if not thinking_text or not _is_valid_thinking_signature(signature): lib_logger.debug( "[Thinking Sanitization] Cached thinking missing text or signature" ) @@ -2144,11 +2255,22 @@ def _get_thinking_config( # Gemini 2.5 & Claude: Integer thinkingBudget if not reasoning_effort: + if is_claude: + return { + "thinkingBudget": CLAUDE_FORCED_THINKING_BUDGET, + "include_thoughts": True, + } return {"thinkingBudget": -1, "include_thoughts": True} # Auto if reasoning_effort == "disable": return {"thinkingBudget": 0, "include_thoughts": False} + if is_claude: + return { + "thinkingBudget": CLAUDE_FORCED_THINKING_BUDGET, + "include_thoughts": True, + } + # Model-specific budgets if "gemini-2.5-pro" in model or is_claude: budgets = {"low": 8192, "medium": 16384, "high": 32768} @@ -2298,12 +2420,16 @@ def _transform_assistant_message( "text": reasoning_content, "thought": True, } - # Try to get signature from cache + # Prefer signature provided by the message, fall back to cache + cached_sig = msg.get("thinking_signature") or msg.get("thought_signature") + if cached_sig and not _is_valid_thinking_signature(cached_sig): + cached_sig = None + + # Try to get signature from cache if not provided cache_key = self._generate_thinking_cache_key( content if isinstance(content, str) else "", tool_calls ) - cached_sig = None - if cache_key: + if not cached_sig and cache_key: cached_json = self._thinking_cache.retrieve(cache_key) if cached_json: try: @@ -2312,7 +2438,7 @@ def _transform_assistant_message( except json.JSONDecodeError: pass - if cached_sig: + if cached_sig and _is_valid_thinking_signature(cached_sig): thinking_part["thoughtSignature"] = cached_sig parts.append(thinking_part) lib_logger.debug( @@ -2422,14 +2548,18 @@ def _get_cached_thinking( thinking_text = thinking_data.get("thinking_text", "") sig = thinking_data.get("thought_signature", "") - if thinking_text: + if thinking_text and _is_valid_thinking_signature(sig): thinking_part = { "text": thinking_text, "thought": True, - "thoughtSignature": sig or "skip_thought_signature_validator", + "thoughtSignature": sig, } parts.append(thinking_part) lib_logger.debug(f"Injected {len(thinking_text)} chars of thinking") + elif thinking_text: + lib_logger.debug( + "[Thinking Cache] Dropping cached thinking with invalid signature" + ) except json.JSONDecodeError: lib_logger.warning(f"Failed to parse cached thinking: {cache_key}") @@ -3592,7 +3722,9 @@ def _transform_to_antigravity_format( # Per Claude docs: https://docs.claude.com/en/docs/build-with-claude/extended-thinking # If this constraint is violated, the API returns 400 INVALID_ARGUMENT thinking_config = gen_config.get("thinkingConfig", {}) - thinking_budget = thinking_config.get("thinkingBudget", 0) + thinking_budget = thinking_config.get( + "thinkingBudget", thinking_config.get("thinking_budget", 0) + ) current_max_tokens = gen_config.get("maxOutputTokens") if ( @@ -3629,6 +3761,21 @@ def _transform_to_antigravity_format( del thinking_config["thinkingLevel"] thinking_config["thinkingBudget"] = -1 + # Claude expects snake_case thinkingConfig fields + if is_claude: + thinking_config = gen_config.get("thinkingConfig", {}) + if thinking_config: + if "includeThoughts" in thinking_config and "include_thoughts" not in thinking_config: + thinking_config["include_thoughts"] = thinking_config.pop("includeThoughts") + + if "thinkingBudget" in thinking_config: + budget = thinking_config.pop("thinkingBudget") + if budget != -1: + thinking_config["thinking_budget"] = budget + + if thinking_config.get("thinking_budget") == -1: + thinking_config.pop("thinking_budget", None) + # Ensure first function call in each model message has a thoughtSignature for Gemini 3 # Per Gemini docs: Only the FIRST parallel function call gets a signature if internal_model.startswith("gemini-3-"): @@ -3679,6 +3826,16 @@ def _unwrap_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """Extract Gemini response from Antigravity envelope.""" return response.get("response", response) + def _get_candidate_parts(self, candidate): + content = candidate.get("content", {}) + if isinstance(content, dict): + parts = content.get("parts", []) + if isinstance(parts, list): + return parts + if isinstance(content, list): + return content + return [] + def _gemini_to_openai_chunk( self, chunk: Dict[str, Any], @@ -3698,7 +3855,7 @@ def _gemini_to_openai_chunk( return {} candidate = candidates[0] - content_parts = candidate.get("content", {}).get("parts", []) + content_parts = self._get_candidate_parts(candidate) text_content = "" reasoning_content = "" @@ -3707,32 +3864,53 @@ def _gemini_to_openai_chunk( tool_idx = accumulator.get("tool_idx", 0) if accumulator else 0 for part in content_parts: - has_func = "functionCall" in part - has_text = "text" in part - has_sig = bool(part.get("thoughtSignature")) + if not isinstance(part, dict): + continue + + part_type = part.get("type") + signature = part.get("thoughtSignature") or part.get("signature") + has_sig = bool(signature) is_thought = ( part.get("thought") is True or str(part.get("thought")).lower() == "true" + or part_type in ("thinking", "redacted_thinking") ) + text_value = None + if "text" in part: + text_value = part.get("text", "") + elif part_type == "thinking": + text_value = part.get("thinking", "") + elif part_type == "text": + text_value = part.get("text", "") + + has_func = "functionCall" in part + is_tool_use = part_type == "tool_use" + # Accumulate signature for Claude caching if has_sig and is_thought and accumulator is not None: - accumulator["thought_signature"] = part["thoughtSignature"] + if not self._is_claude(model) or _is_valid_thinking_signature(signature): + accumulator["thought_signature"] = signature # Skip standalone signature parts - if has_sig and not has_func and (not has_text or not part.get("text")): + if ( + has_sig + and not has_func + and not is_tool_use + and not text_value + and not is_thought + ): continue - if has_text: - text = part["text"] + if text_value is not None: if is_thought: - reasoning_content += text + reasoning_content += text_value if accumulator is not None: - accumulator["reasoning_content"] += text + accumulator["reasoning_content"] += text_value else: - text_content += text + text_content += text_value if accumulator is not None: - accumulator["text_content"] += text + accumulator["text_content"] += text_value if has_func: # Get tool_schemas from accumulator for schema-aware parsing @@ -3743,10 +3921,14 @@ def _gemini_to_openai_chunk( # Store signature for each tool call (needed for parallel tool calls) if has_sig: - self._handle_tool_signature(tool_call, part["thoughtSignature"]) + self._handle_tool_signature(tool_call, signature) tool_calls.append(tool_call) tool_idx += 1 + elif is_tool_use: + tool_call = self._extract_tool_use(part, tool_idx, accumulator) + tool_calls.append(tool_call) + tool_idx += 1 # Build delta delta = {} @@ -3803,7 +3985,7 @@ def _gemini_to_openai_non_streaming( return {} candidate = candidates[0] - content_parts = candidate.get("content", {}).get("parts", []) + content_parts = self._get_candidate_parts(candidate) text_content = "" reasoning_content = "" @@ -3811,25 +3993,47 @@ def _gemini_to_openai_non_streaming( thought_sig = "" for part in content_parts: - has_func = "functionCall" in part - has_text = "text" in part - has_sig = bool(part.get("thoughtSignature")) + if not isinstance(part, dict): + continue + + part_type = part.get("type") + signature = part.get("thoughtSignature") or part.get("signature") + has_sig = bool(signature) is_thought = ( part.get("thought") is True or str(part.get("thought")).lower() == "true" + or part_type in ("thinking", "redacted_thinking") ) if has_sig and is_thought: - thought_sig = part["thoughtSignature"] + if not self._is_claude(model) or _is_valid_thinking_signature(signature): + thought_sig = signature + + text_value = None + if "text" in part: + text_value = part.get("text", "") + elif part_type == "thinking": + text_value = part.get("thinking", "") + elif part_type == "text": + text_value = part.get("text", "") + + has_func = "functionCall" in part + is_tool_use = part_type == "tool_use" - if has_sig and not has_func and (not has_text or not part.get("text")): + if ( + has_sig + and not has_func + and not is_tool_use + and not text_value + and not is_thought + ): continue - if has_text: + if text_value is not None: if is_thought: - reasoning_content += part["text"] + reasoning_content += text_value else: - text_content += part["text"] + text_content += text_value if has_func: tool_call = self._extract_tool_call( @@ -3838,9 +4042,12 @@ def _gemini_to_openai_non_streaming( # Store signature for each tool call (needed for parallel tool calls) if has_sig: - self._handle_tool_signature(tool_call, part["thoughtSignature"]) + self._handle_tool_signature(tool_call, signature) tool_calls.append(tool_call) + elif is_tool_use: + tool_call = self._extract_tool_use(part, len(tool_calls)) + tool_calls.append(tool_call) # Cache Claude thinking if ( @@ -3860,6 +4067,8 @@ def _gemini_to_openai_non_streaming( message["content"] = "" if reasoning_content: message["reasoning_content"] = reasoning_content + if thought_sig and _is_valid_thinking_signature(thought_sig): + message["thinking_signature"] = thought_sig if tool_calls: message["tool_calls"] = tool_calls message.pop("content", None) @@ -3914,6 +4123,28 @@ def _build_tool_schema_map( return schema_map + def _extract_tool_use(self, part, index, accumulator=None): + tool_id = part.get("id") or f"call_{uuid.uuid4().hex[:24]}" + tool_name = part.get("name", "") + tool_input = part.get("input", {}) + + try: + args = json.dumps(tool_input) + except TypeError: + args = json.dumps({}) + + tool_call = { + "id": tool_id, + "type": "function", + "index": index, + "function": {"name": tool_name, "arguments": args}, + } + + if accumulator is not None: + accumulator["tool_calls"].append(tool_call) + + return tool_call + def _extract_tool_call( self, part: Dict[str, Any], @@ -4010,6 +4241,12 @@ def _cache_thinking( self, reasoning: str, signature: str, text: str, tool_calls: List[Dict] ) -> None: """Cache Claude thinking content.""" + if not _is_valid_thinking_signature(signature): + lib_logger.debug( + "[Thinking Cache] Skipping cache due to invalid signature" + ) + return + cache_key = self._generate_thinking_cache_key(text, tool_calls) if not cache_key: return @@ -4121,10 +4358,14 @@ async def acompletion( # Thinking is enabled if reasoning_effort is set (and not "disable") for Claude thinking_enabled = False if self._is_claude(model): - # For Claude, thinking is enabled when reasoning_effort is provided and not "disable" - thinking_enabled = ( - reasoning_effort is not None and reasoning_effort != "disable" - ) + if reasoning_effort is not None: + # For Claude, thinking is enabled when reasoning_effort is provided and not "disable" + thinking_enabled = reasoning_effort != "disable" + else: + # Opus always thinks, and -thinking variants should be treated as enabled + thinking_enabled = model.startswith("claude-opus-") or model.endswith( + "-thinking" + ) # Transform messages to Gemini format FIRST # This restores thinking from cache if reasoning_content was stripped by client @@ -4176,6 +4417,17 @@ async def acompletion( gemini_payload, self._parallel_tool_instruction ) + # Add interleaved thinking hint for Claude thinking models with tools + if ( + tools + and self._is_claude(model) + and thinking_enabled + and self._enable_claude_interleaved_hint + ): + self._append_system_instruction( + gemini_payload, self._claude_interleaved_hint + ) + # Add generation config gen_config = {} if top_p is not None: @@ -4259,6 +4511,15 @@ async def acompletion( **ANTIGRAVITY_HEADERS, } + if self._is_claude(model) and thinking_enabled: + headers["anthropic-beta"] = "interleaved-thinking-2025-05-14" + lib_logger.debug( + f"[Antigravity] Added anthropic-beta header for Claude thinking model: {payload.get('model')}" + ) + + if file_logger: + file_logger.log_request_headers(headers) + # Track malformed call retries (separate from empty response retries) malformed_retry_count = 0 # Keep a mutable reference to gemini_contents for retry injection @@ -4526,6 +4787,28 @@ def _inject_tool_hardening_instruction( "parts": [instruction_part], } + def _append_system_instruction(self, payload, instruction_text): + """Append a system instruction without reordering earlier instructions.""" + if not instruction_text: + return + + instruction_part = {"text": instruction_text} + + if "system_instruction" in payload: + existing = payload["system_instruction"] + if isinstance(existing, dict) and "parts" in existing: + existing["parts"].append(instruction_part) + else: + payload["system_instruction"] = { + "role": "user", + "parts": [{"text": str(existing)}, instruction_part], + } + else: + payload["system_instruction"] = { + "role": "user", + "parts": [instruction_part], + } + async def _handle_non_streaming( self, client: httpx.AsyncClient, @@ -4547,6 +4830,8 @@ async def _handle_non_streaming( data = response.json() if file_logger: file_logger.log_final_response(data) + if self._is_claude(model): + file_logger.log_raw_response(data, "claude_raw_response.json") gemini_response = self._unwrap_response(data) @@ -4639,6 +4924,9 @@ async def _handle_streaming( if not accumulator.get("response_id"): accumulator["response_id"] = gemini_chunk.get("responseId") + if file_logger and self._is_claude(model): + file_logger.log_unwrapped_stream_chunk(gemini_chunk) + # Check for MALFORMED_FUNCTION_CALL malformed_msg = self._check_for_malformed_call(gemini_chunk) if malformed_msg: From 765df7ad543d1c2744ac7d91db4ccec9145b6318 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Wed, 31 Dec 2025 21:05:09 +0100 Subject: [PATCH 18/36] fix(anthropic): read thinking budget from client request Pass through the exact budget_tokens value from the Anthropic request instead of using a hardcoded constant. This allows Claude Code and other clients to control the thinking budget directly. Changes: - translator.py: Pass thinking_budget from request.thinking.budget_tokens - antigravity_provider.py: Accept and use thinking_budget parameter in _get_thinking_config(), falling back to default if not provided Signed-off-by: Moeeze Hassan --- .../anthropic_compat/translator.py | 32 +++++-------------- .../providers/antigravity_provider.py | 23 ++++++++++--- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 574b12503..3686799be 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -453,39 +453,23 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, # and doesn't affect the model's behavior. # Handle Anthropic thinking config -> reasoning_effort translation - # The provider (antigravity_provider.py) applies a // 4 reduction to thinking budget - # unless custom_reasoning_budget is True. This conserves thinking tokens. - # - # Reasoning budget thresholds map to provider budgets: - # - Claude "high" = 32768 tokens (but // 4 = 8192 unless custom_reasoning_budget) - # - Claude "medium" = 16384 tokens (// 4 = 4096) - # - Claude "low" = 8192 tokens (// 4 = 2048) - # - # We only set custom_reasoning_budget=True when user explicitly requests - # a large budget (32000+), indicating they want full thinking capacity. + # Pass through the exact budget_tokens value when specified, allowing the + # provider to use the client's requested thinking budget directly. if request.thinking: if request.thinking.type == "enabled": - budget = request.thinking.budget_tokens or 10000 - if budget >= 32000: - # User explicitly wants full thinking capacity + budget = request.thinking.budget_tokens + if budget: + # Pass the exact budget through for the provider to use openai_request["reasoning_effort"] = "high" - openai_request["custom_reasoning_budget"] = True - elif budget >= 10000: - openai_request["reasoning_effort"] = "high" - # custom_reasoning_budget defaults to False, so // 4 applies - elif budget >= 5000: - openai_request["reasoning_effort"] = "medium" + openai_request["thinking_budget"] = budget else: - openai_request["reasoning_effort"] = "low" + # No specific budget requested, use high effort + openai_request["reasoning_effort"] = "high" elif request.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" elif _is_opus_model(request.model): # Enable thinking for Opus models when no thinking config is provided - # Use "high" effort but NOT custom_reasoning_budget, so // 4 applies - # This gives 8192 thinking tokens (32768 // 4) which is reasonable for most tasks - # Users who want full capacity can explicitly set thinking.budget_tokens >= 32000 openai_request["reasoning_effort"] = "high" - # Note: NOT setting custom_reasoning_budget here to conserve tokens return openai_request diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 3189fbdcc..86f321621 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -2216,7 +2216,11 @@ def _close_tool_loop_for_thinking( # ========================================================================= def _get_thinking_config( - self, reasoning_effort: Optional[str], model: str, custom_budget: bool = False + self, + reasoning_effort: Optional[str], + model: str, + custom_budget: bool = False, + thinking_budget: Optional[int] = None, ) -> Optional[Dict[str, Any]]: """ Map reasoning_effort to thinking configuration. @@ -2224,6 +2228,12 @@ def _get_thinking_config( - Gemini 2.5 & Claude: thinkingBudget (integer tokens) - Gemini 3 Pro: thinkingLevel (string: "low"/"high") - Gemini 3 Flash: thinkingLevel (string: "minimal"/"low"/"medium"/"high") + + Args: + reasoning_effort: The reasoning effort level (low/medium/high/disable) + model: The model name + custom_budget: Whether to use the full budget without reduction + thinking_budget: Exact thinking budget from client (takes precedence for Claude) """ internal = self._alias_to_internal(model) is_gemini_25 = "gemini-2.5" in model @@ -2256,8 +2266,10 @@ def _get_thinking_config( # Gemini 2.5 & Claude: Integer thinkingBudget if not reasoning_effort: if is_claude: + # Use client-provided budget if available, otherwise use default + budget = thinking_budget if thinking_budget else CLAUDE_FORCED_THINKING_BUDGET return { - "thinkingBudget": CLAUDE_FORCED_THINKING_BUDGET, + "thinkingBudget": budget, "include_thoughts": True, } return {"thinkingBudget": -1, "include_thoughts": True} # Auto @@ -2266,8 +2278,10 @@ def _get_thinking_config( return {"thinkingBudget": 0, "include_thoughts": False} if is_claude: + # Use client-provided budget if available, otherwise use default + budget = thinking_budget if thinking_budget else CLAUDE_FORCED_THINKING_BUDGET return { - "thinkingBudget": CLAUDE_FORCED_THINKING_BUDGET, + "thinkingBudget": budget, "include_thoughts": True, } @@ -4349,6 +4363,7 @@ async def acompletion( temperature = kwargs.get("temperature") max_tokens = kwargs.get("max_tokens") custom_budget = kwargs.get("custom_reasoning_budget", False) + thinking_budget = kwargs.get("thinking_budget") # Exact budget from client enable_logging = kwargs.pop("enable_request_logging", False) # Create logger @@ -4441,7 +4456,7 @@ async def acompletion( gen_config["temperature"] = 1.0 thinking_config = self._get_thinking_config( - reasoning_effort, model, custom_budget + reasoning_effort, model, custom_budget, thinking_budget ) if thinking_config: gen_config.setdefault("thinkingConfig", {}).update(thinking_config) From 5af1f10cad6617a8e5a37ae24beb6cf10fa72e5d Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 1 Jan 2026 10:01:29 +0100 Subject: [PATCH 19/36] fix(anthropic): handle thinking toggle for text-only assistant messages When thinking is enabled but the last assistant message has no thinking block AND no tool calls (simple text response), Claude API rejects with "Expected thinking but found text". Add synthetic user message to start a fresh turn, allowing thinking to be generated naturally. Signed-off-by: Moeeze Hassan --- .../providers/antigravity_provider.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 86f321621..5e15092a1 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -1869,6 +1869,26 @@ def _sanitize_thinking_for_claude( "This is likely from context compression or non-thinking model. " "New response will include thinking naturally." ) + elif not state["turn_has_thinking"]: + # CASE: Last assistant message has NO tool calls AND NO thinking + # This happens when: + # 1. Previous turn was made without thinking enabled + # 2. A simple text response without any tool use + # + # Per Claude docs: "the final assistant message must start with a thinking block" + # If we're enabling thinking now, we MUST close the turn and start fresh, + # otherwise Claude API rejects with: + # "Expected `thinking` or `redacted_thinking`, but found `text`" + lib_logger.info( + "[Thinking Sanitization] Last model message has no thinking and no tool calls. " + "Adding synthetic user message to start fresh thinking turn." + ) + synthetic_user = { + "role": "user", + "parts": [{"text": "[Continue]"}], + } + messages.append(synthetic_user) + return self._strip_all_thinking_blocks(messages), False # Strip thinking from old turns, let new response add thinking naturally return self._strip_old_turn_thinking( From 0bb8a521f0ec48215d0a08f1ed1ae019a8919717 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 1 Jan 2026 11:28:36 +0100 Subject: [PATCH 20/36] fix(anthropic): strengthen interleaved thinking hint Require a thinking block before each tool call and after tool results for Claude interleaved thinking. Signed-off-by: Moeeze Hassan --- src/rotator_library/providers/antigravity_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 5e15092a1..f397667df 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -290,7 +290,7 @@ def _get_claude_thinking_cache_file(): DEFAULT_PARALLEL_TOOL_INSTRUCTION = """When multiple independent operations are needed, prefer making parallel tool calls in a single response rather than sequential calls across multiple responses. This reduces round-trips and improves efficiency. Only use sequential calls when one tool's output is required as input for another.""" # Claude interleaved thinking hint (encourages thinking after tool results) -DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """Interleaved thinking is enabled. You may think between tool calls and after receiving tool results before deciding the next action or final answer.""" +DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """Interleaved thinking is enabled. Always emit a thinking block before each tool call and after each tool result, even if brief, before deciding the next action or final answer.""" # ============================================================================= From 991a8e301b8905be43029bc88a30050e1840e562 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 1 Jan 2026 11:56:43 +0100 Subject: [PATCH 21/36] fix(antigravity): remove unreachable is_claude condition in thinking config Claude models always return early before reaching the model-specific budgets section, making the `or is_claude` condition dead code. --- src/rotator_library/providers/antigravity_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index f397667df..a3d5dad67 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -2305,8 +2305,8 @@ def _get_thinking_config( "include_thoughts": True, } - # Model-specific budgets - if "gemini-2.5-pro" in model or is_claude: + # Model-specific budgets (Claude already returned above) + if "gemini-2.5-pro" in model: budgets = {"low": 8192, "medium": 16384, "high": 32768} elif "gemini-2.5-flash" in model: budgets = {"low": 6144, "medium": 12288, "high": 24576} From 354ac17bc4e3d808e6dc38b73b88e5d92c37dbc7 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 1 Jan 2026 11:57:19 +0100 Subject: [PATCH 22/36] fix(antigravity): add debug logging for non-data URL images Logs a debug message when skipping non-data URL images, helping developers troubleshoot why images may not appear in requests. --- src/rotator_library/providers/antigravity_provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index a3d5dad67..0cc9783cb 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -2424,6 +2424,7 @@ def _parse_image_url(self, image_url: Dict[str, Any]) -> Optional[Dict[str, Any] """Parse image URL into Gemini inlineData format.""" url = image_url.get("url", "") if not url.startswith("data:"): + lib_logger.debug(f"Skipping non-data URL image: {url[:100]}...") return None try: From b81ca57bf2c300493f88e8198f0f1706a5418a64 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 2 Jan 2026 02:14:06 +0100 Subject: [PATCH 23/36] fix(anthropic): correct cache token handling in usage responses Google's promptTokenCount INCLUDES cached tokens, but Anthropic's input_tokens EXCLUDES cached tokens. This fix: - Extract cachedContentTokenCount from Google's usageMetadata - Subtract cached tokens from input_tokens in responses - Include cache_read_input_tokens and cache_creation_input_tokens - Apply fix to both streaming and non-streaming responses --- .../anthropic_compat/streaming.py | 51 ++++++++++++++++--- .../anthropic_compat/translator.py | 19 +++++-- .../providers/antigravity_provider.py | 19 ++++++- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/rotator_library/anthropic_compat/streaming.py b/src/rotator_library/anthropic_compat/streaming.py index 5ceb71455..e3ab84abe 100644 --- a/src/rotator_library/anthropic_compat/streaming.py +++ b/src/rotator_library/anthropic_compat/streaming.py @@ -54,6 +54,7 @@ async def anthropic_streaming_wrapper( tool_block_indices = {} # Track which block index each tool call uses input_tokens = 0 output_tokens = 0 + cached_tokens = 0 # Track cached tokens for proper Anthropic format try: async for chunk_str in openai_stream: @@ -69,6 +70,12 @@ async def anthropic_streaming_wrapper( # CRITICAL: Send message_start if we haven't yet (e.g., empty response) # Claude Code and other clients require message_start before message_stop if not message_started: + # Build usage with cached tokens properly handled + usage_dict = {"input_tokens": input_tokens - cached_tokens, "output_tokens": 0} + if cached_tokens > 0: + usage_dict["cache_read_input_tokens"] = cached_tokens + usage_dict["cache_creation_input_tokens"] = 0 + message_start = { "type": "message_start", "message": { @@ -79,7 +86,7 @@ async def anthropic_streaming_wrapper( "model": original_model, "stop_reason": None, "stop_sequence": None, - "usage": {"input_tokens": input_tokens, "output_tokens": 0}, + "usage": usage_dict, }, } yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" @@ -105,8 +112,14 @@ async def anthropic_streaming_wrapper( # Determine stop_reason based on whether we had tool calls stop_reason = "tool_use" if tool_calls_by_index else "end_turn" + # Build final usage dict with cached tokens + final_usage = {"output_tokens": output_tokens} + if cached_tokens > 0: + final_usage["cache_read_input_tokens"] = cached_tokens + final_usage["cache_creation_input_tokens"] = 0 + # Send message_delta with final info - yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "{stop_reason}", "stop_sequence": null}}, "usage": {{"output_tokens": {output_tokens}}}}}\n\n' + yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "{stop_reason}", "stop_sequence": null}}, "usage": {json.dumps(final_usage)}}}\n\n' # Send message_stop yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' @@ -118,12 +131,24 @@ async def anthropic_streaming_wrapper( continue # Extract usage if present + # Note: Google's promptTokenCount INCLUDES cached tokens, but Anthropic's + # input_tokens EXCLUDES cached tokens. We extract cached tokens and subtract. if "usage" in chunk and chunk["usage"]: - input_tokens = chunk["usage"].get("prompt_tokens", input_tokens) - output_tokens = chunk["usage"].get("completion_tokens", output_tokens) + usage = chunk["usage"] + input_tokens = usage.get("prompt_tokens", input_tokens) + output_tokens = usage.get("completion_tokens", output_tokens) + # Extract cached tokens from prompt_tokens_details + if usage.get("prompt_tokens_details"): + cached_tokens = usage["prompt_tokens_details"].get("cached_tokens", cached_tokens) # Send message_start on first chunk if not message_started: + # Build usage with cached tokens properly handled for Anthropic format + usage_dict = {"input_tokens": input_tokens - cached_tokens, "output_tokens": 0} + if cached_tokens > 0: + usage_dict["cache_read_input_tokens"] = cached_tokens + usage_dict["cache_creation_input_tokens"] = 0 + message_start = { "type": "message_start", "message": { @@ -134,7 +159,7 @@ async def anthropic_streaming_wrapper( "model": original_model, "stop_reason": None, "stop_sequence": None, - "usage": {"input_tokens": input_tokens, "output_tokens": 0}, + "usage": usage_dict, }, } yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" @@ -263,6 +288,12 @@ async def anthropic_streaming_wrapper( # If we haven't sent message_start yet, send it now so the client can display the error # Claude Code and other clients may ignore events that come before message_start if not message_started: + # Build usage with cached tokens properly handled + usage_dict = {"input_tokens": input_tokens - cached_tokens, "output_tokens": 0} + if cached_tokens > 0: + usage_dict["cache_read_input_tokens"] = cached_tokens + usage_dict["cache_creation_input_tokens"] = 0 + message_start = { "type": "message_start", "message": { @@ -273,7 +304,7 @@ async def anthropic_streaming_wrapper( "model": original_model, "stop_reason": None, "stop_sequence": None, - "usage": {"input_tokens": 0, "output_tokens": 0}, + "usage": usage_dict, }, } yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" @@ -296,8 +327,14 @@ async def anthropic_streaming_wrapper( yield f'event: content_block_stop\ndata: {{"type": "content_block_stop", "index": {current_block_index}}}\n\n' + # Build final usage with cached tokens + final_usage = {"output_tokens": 0} + if cached_tokens > 0: + final_usage["cache_read_input_tokens"] = cached_tokens + final_usage["cache_creation_input_tokens"] = 0 + # Send message_delta and message_stop to properly close the stream - yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "end_turn", "stop_sequence": null}}, "usage": {{"output_tokens": 0}}}}\n\n' + yield f'event: message_delta\ndata: {{"type": "message_delta", "delta": {{"stop_reason": "end_turn", "stop_sequence": null}}, "usage": {json.dumps(final_usage)}}}\n\n' yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' # Also send the formal error event for clients that handle it diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 3686799be..44caf5f8c 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -379,16 +379,25 @@ def openai_to_anthropic_response(openai_response: dict, original_model: str) -> stop_reason = stop_reason_map.get(finish_reason, "end_turn") # Build usage + # Note: Google's promptTokenCount INCLUDES cached tokens, but Anthropic's + # input_tokens EXCLUDES cached tokens. We need to subtract cached tokens. + prompt_tokens = usage.get("prompt_tokens", 0) + cached_tokens = 0 + + # Extract cached tokens if present + if usage.get("prompt_tokens_details"): + details = usage["prompt_tokens_details"] + cached_tokens = details.get("cached_tokens", 0) + anthropic_usage = { - "input_tokens": usage.get("prompt_tokens", 0), + "input_tokens": prompt_tokens - cached_tokens, # Subtract cached tokens "output_tokens": usage.get("completion_tokens", 0), } # Add cache tokens if present - if usage.get("prompt_tokens_details"): - details = usage["prompt_tokens_details"] - if details.get("cached_tokens"): - anthropic_usage["cache_read_input_tokens"] = details["cached_tokens"] + if cached_tokens > 0: + anthropic_usage["cache_read_input_tokens"] = cached_tokens + anthropic_usage["cache_creation_input_tokens"] = 0 return { "id": openai_response.get("id", f"msg_{uuid.uuid4().hex[:24]}"), diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 0cc9783cb..3d52090c1 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -4253,13 +4253,20 @@ def _map_finish_reason( return "tool_calls" if has_tool_calls else reason def _build_usage(self, metadata: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Build usage dict from Gemini usage metadata.""" + """Build usage dict from Gemini usage metadata. + + Note: Google's promptTokenCount INCLUDES cached tokens, but Anthropic's + input_tokens EXCLUDES cached tokens. We pass cached tokens through in + OpenAI format (prompt_tokens_details.cached_tokens) so the translator + can correctly subtract them when converting to Anthropic format. + """ if not metadata: return None prompt = metadata.get("promptTokenCount", 0) thoughts = metadata.get("thoughtsTokenCount", 0) completion = metadata.get("candidatesTokenCount", 0) + cached = metadata.get("cachedContentTokenCount", 0) usage = { "prompt_tokens": prompt + thoughts, @@ -4267,6 +4274,16 @@ def _build_usage(self, metadata: Dict[str, Any]) -> Optional[Dict[str, Any]]: "total_tokens": metadata.get("totalTokenCount", 0), } + # Build prompt_tokens_details for cached and reasoning tokens + prompt_details = {} + if cached > 0: + prompt_details["cached_tokens"] = cached + if thoughts > 0: + prompt_details["reasoning_tokens"] = thoughts + + if prompt_details: + usage["prompt_tokens_details"] = prompt_details + if thoughts > 0: usage["completion_tokens_details"] = {"reasoning_tokens": thoughts} From 97ef2d11614b66329300d7c71949d5646862970a Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 2 Jan 2026 02:31:30 +0100 Subject: [PATCH 24/36] feat(anthropic): add 5 translation improvements from reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Session ID for Prompt Caching (High Priority) - Derive stable session ID from first user message hash - Enables prompt caching continuity across conversation turns - Falls back to random ID if no user message found 2. Content Reordering (Medium Priority) - Reorder assistant content blocks: thinking → text → tool_use - Matches Anthropic's expected ordering - Sanitizes thinking blocks by removing cache_control 3. Document/PDF Handling (Low Priority) - Support for 'document' type content blocks - Converts base64/URL documents to OpenAI image_url format - Default media type: application/pdf 4. Gemini Output Token Cap (Low Priority) - Add GEMINI_MAX_OUTPUT_TOKENS constant (16384) - Cap maxOutputTokens for non-Claude models - Prevents errors from exceeding Gemini limits 5. Schema Sanitization Improvements (Low Priority) - Add _score_schema_option() for smarter anyOf/oneOf selection - Add _merge_all_of() to properly merge allOf schemas - Add description hints when flattening union types - Select best option (objects > arrays > primitives > null) --- .../anthropic_compat/translator.py | 76 ++++ .../providers/antigravity_provider.py | 324 +++++++++++++++++- 2 files changed, 387 insertions(+), 13 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 44caf5f8c..dfbea758e 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -15,6 +15,57 @@ MIN_THINKING_SIGNATURE_LENGTH = 100 +def _reorder_assistant_content(content: List[dict]) -> List[dict]: + """ + Reorder assistant message content blocks to ensure correct order: + 1. Thinking blocks come first (required when thinking is enabled) + 2. Text blocks come in the middle (filtering out empty ones) + 3. Tool_use blocks come at the end (required before tool_result) + + This matches Anthropic's expected ordering and prevents API errors. + """ + if not isinstance(content, list) or len(content) <= 1: + return content + + thinking_blocks = [] + text_blocks = [] + tool_use_blocks = [] + other_blocks = [] + + for block in content: + if not isinstance(block, dict): + other_blocks.append(block) + continue + + block_type = block.get("type", "") + + if block_type in ("thinking", "redacted_thinking"): + # Sanitize thinking blocks - remove cache_control and other extra fields + sanitized = { + "type": block_type, + "thinking": block.get("thinking", ""), + } + if block.get("signature"): + sanitized["signature"] = block["signature"] + thinking_blocks.append(sanitized) + + elif block_type == "tool_use": + tool_use_blocks.append(block) + + elif block_type == "text": + # Only keep text blocks with meaningful content + text = block.get("text", "") + if text and text.strip(): + text_blocks.append(block) + + else: + # Other block types (images, documents, etc.) go in the text position + other_blocks.append(block) + + # Reorder: thinking → other → text → tool_use + return thinking_blocks + other_blocks + text_blocks + tool_use_blocks + + def anthropic_to_openai_messages( anthropic_messages: List[dict], system: Optional[Union[str, List[dict]]] = None ) -> List[dict]: @@ -55,6 +106,11 @@ def anthropic_to_openai_messages( if isinstance(content, str): openai_messages.append({"role": role, "content": content}) elif isinstance(content, list): + # Reorder assistant content blocks to ensure correct order: + # thinking → text → tool_use + if role == "assistant": + content = _reorder_assistant_content(content) + # Handle content blocks openai_content = [] tool_calls = [] @@ -88,6 +144,26 @@ def anthropic_to_openai_messages( "image_url": {"url": source.get("url", "")}, } ) + elif block_type == "document": + # Convert Anthropic document format (e.g. PDF) to OpenAI + # Documents are treated similarly to images with appropriate mime type + source = block.get("source", {}) + if source.get("type") == "base64": + openai_content.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{source.get('media_type', 'application/pdf')};base64,{source.get('data', '')}" + }, + } + ) + elif source.get("type") == "url": + openai_content.append( + { + "type": "image_url", + "image_url": {"url": source.get("url", "")}, + } + ) elif block_type == "thinking": signature = block.get("signature", "") if signature and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH: diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 3d52090c1..a991a92de 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -128,6 +128,10 @@ def _env_int(key: str, default: int) -> int: # Default max output tokens (including thinking) - can be overridden per request DEFAULT_MAX_OUTPUT_TOKENS = 64000 +# Gemini max output tokens cap - Gemini models have a 16K output limit +# See: https://ai.google.dev/gemini-api/docs/models +GEMINI_MAX_OUTPUT_TOKENS = 16384 + # Empty response retry configuration # When Antigravity returns an empty response (no content, no tool calls), # automatically retry up to this many attempts before giving up (minimum 1) @@ -303,12 +307,118 @@ def _generate_request_id() -> str: return f"agent-{uuid.uuid4()}" +def _derive_session_id(messages: List[Dict[str, Any]]) -> str: + """ + Derive a stable session ID from the first user message in the conversation. + + This ensures the same conversation uses the same session ID across turns, + enabling prompt caching (cache is scoped to session + organization). + + Args: + messages: List of Anthropic-format messages + + Returns: + A stable session ID (32 hex characters) derived from first user message, + or a random fallback if no user message found. + """ + import hashlib + + for msg in messages: + if msg.get("role") == "user": + content = msg.get("content", "") + + # Handle string content + if isinstance(content, str): + text_content = content + # Handle array content (extract text blocks) + elif isinstance(content, list): + text_parts = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + if text: + text_parts.append(text) + text_content = "\n".join(text_parts) + else: + text_content = "" + + if text_content: + # Hash the content with SHA256, return first 32 hex chars + hash_digest = hashlib.sha256(text_content.encode()).hexdigest() + return hash_digest[:32] + + # Fallback to random ID if no user message found + return f"-{random.randint(1_000_000_000_000_000_000, 9_999_999_999_999_999_999)}" + + def _generate_session_id() -> str: - """Generate Antigravity session ID: -{random_number}""" + """Generate Antigravity session ID: -{random_number} (legacy fallback)""" n = random.randint(1_000_000_000_000_000_000, 9_999_999_999_999_999_999) return f"-{n}" +def _reorder_assistant_content(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Reorder assistant message content blocks to ensure correct order: + 1. Thinking blocks come first (required when thinking is enabled) + 2. Text blocks come in the middle (filtering out empty ones) + 3. Tool_use blocks come at the end (required before tool_result) + + This matches Anthropic's expected ordering and prevents API errors. + + Args: + content: List of content blocks from an assistant message + + Returns: + Reordered content blocks + """ + if not isinstance(content, list): + return content + + # Single element - just return as-is (but could sanitize thinking if needed) + if len(content) <= 1: + return content + + thinking_blocks = [] + text_blocks = [] + tool_use_blocks = [] + other_blocks = [] + + for block in content: + if not isinstance(block, dict): + other_blocks.append(block) + continue + + block_type = block.get("type", "") + + if block_type in ("thinking", "redacted_thinking"): + # Sanitize thinking blocks - remove cache_control and other extra fields + sanitized = { + "type": block_type, + "thinking": block.get("thinking", ""), + } + # Preserve signature if present + if block.get("signature"): + sanitized["signature"] = block["signature"] + thinking_blocks.append(sanitized) + + elif block_type == "tool_use": + tool_use_blocks.append(block) + + elif block_type == "text": + # Only keep text blocks with meaningful content + text = block.get("text", "") + if text and text.strip(): + text_blocks.append(block) + + else: + # Other block types (images, etc.) go in the text position + other_blocks.append(block) + + # Reorder: thinking → other → text → tool_use + return thinking_blocks + other_blocks + text_blocks + tool_use_blocks + + def _generate_project_id() -> str: """Generate fake project ID: {adj}-{noun}-{random}""" adjectives = ["useful", "bright", "swift", "calm", "bold"] @@ -508,6 +618,115 @@ def resolve(node, seen=()): return resolve(schema) +def _score_schema_option(schema: dict) -> int: + """ + Score a schema option for anyOf/oneOf selection. + Higher scores = more preferred schemas. + + Returns: + Score (0-3): object with properties=3, array=2, other non-null type=1, null/no type=0 + """ + if not isinstance(schema, dict): + return 0 + + # Score 3: Object types with properties (most informative) + if schema.get("type") == "object" or "properties" in schema: + return 3 + + # Score 2: Array types with items + if schema.get("type") == "array" or "items" in schema: + return 2 + + # Score 1: Any other non-null type + if schema.get("type") and schema.get("type") != "null": + return 1 + + # Score 0: Null or no type + return 0 + + +def _merge_all_of(schema: Any) -> Any: + """ + Merge all schemas in an allOf array into a single schema. + Properties and required arrays are merged; other fields use first occurrence. + """ + if not isinstance(schema, dict): + return schema + + # Process allOf if present + if "allOf" in schema and isinstance(schema["allOf"], list) and schema["allOf"]: + merged_properties = {} + merged_required = set() + other_fields = {} + + for sub_schema in schema["allOf"]: + if not isinstance(sub_schema, dict): + continue + + # Recursively merge nested allOf first + sub_schema = _merge_all_of(sub_schema) + + # Merge properties (later overrides earlier) + if "properties" in sub_schema and isinstance(sub_schema["properties"], dict): + for key, value in sub_schema["properties"].items(): + merged_properties[key] = value + + # Union required arrays + if "required" in sub_schema and isinstance(sub_schema["required"], list): + for req in sub_schema["required"]: + merged_required.add(req) + + # Copy other fields (first occurrence wins) + for key, value in sub_schema.items(): + if key not in ("properties", "required", "allOf") and key not in other_fields: + other_fields[key] = value + + # Build result without allOf + result = {} + + # Apply other fields first + for key, value in other_fields.items(): + if key not in schema or key == "allOf": + result[key] = value + + # Copy non-allOf fields from parent schema (parent takes precedence) + for key, value in schema.items(): + if key != "allOf": + if key == "properties" and isinstance(value, dict): + # Merge parent properties with allOf properties + result["properties"] = {**merged_properties, **value} + elif key == "required" and isinstance(value, list): + # Merge parent required with allOf required + result["required"] = list(merged_required.union(value)) + else: + result[key] = value + + # Add merged properties if not already present + if merged_properties and "properties" not in result: + result["properties"] = merged_properties + + # Add merged required if not already present + if merged_required and "required" not in result: + result["required"] = list(merged_required) + + schema = result + + # Recursively process properties + if "properties" in schema and isinstance(schema["properties"], dict): + schema["properties"] = { + key: _merge_all_of(value) for key, value in schema["properties"].items() + } + + # Recursively process items + if "items" in schema: + if isinstance(schema["items"], list): + schema["items"] = [_merge_all_of(item) for item in schema["items"]] + elif isinstance(schema["items"], dict): + schema["items"] = _merge_all_of(schema["items"]) + + return schema + + def _clean_claude_schema(schema: Any, for_gemini: bool = False) -> Any: """ Recursively clean JSON Schema for Antigravity/Google's Proto-based API. @@ -571,19 +790,76 @@ def _clean_claude_schema(schema: Any, for_gemini: bool = False) -> Any: "default", } - # Handle 'anyOf' by taking the first option (Claude doesn't support anyOf) - # Gemini supports anyOf/oneOf, so pass through for Gemini + # Handle 'anyOf' by selecting the best option based on scoring + # Claude doesn't support anyOf, Gemini does - so only flatten for Claude if not for_gemini: if "anyOf" in schema and isinstance(schema["anyOf"], list) and schema["anyOf"]: - first_option = _clean_claude_schema(schema["anyOf"][0], for_gemini) - if isinstance(first_option, dict): - return first_option + options = schema["anyOf"] + # Find the best option using scoring + best_option = None + best_score = -1 + type_names = [] + + for option in options: + if not isinstance(option, dict): + continue + # Collect type names for hint + type_name = option.get("type") or ("object" if "properties" in option else None) + if type_name and type_name != "null": + type_names.append(type_name) + # Score and track best + score = _score_schema_option(option) + if score > best_score: + best_score = score + best_option = option + + if best_option: + cleaned_option = _clean_claude_schema(best_option, for_gemini) + if isinstance(cleaned_option, dict): + # Add hint if multiple types existed + if len(type_names) > 1: + hint = f"one of: {', '.join(type_names)}" + if "description" in cleaned_option: + cleaned_option["description"] = f"{cleaned_option['description']} ({hint})" + else: + cleaned_option["description"] = hint + return cleaned_option - # Handle 'oneOf' similarly + # Handle 'oneOf' similarly with scoring if "oneOf" in schema and isinstance(schema["oneOf"], list) and schema["oneOf"]: - first_option = _clean_claude_schema(schema["oneOf"][0], for_gemini) - if isinstance(first_option, dict): - return first_option + options = schema["oneOf"] + best_option = None + best_score = -1 + type_names = [] + + for option in options: + if not isinstance(option, dict): + continue + type_name = option.get("type") or ("object" if "properties" in option else None) + if type_name and type_name != "null": + type_names.append(type_name) + score = _score_schema_option(option) + if score > best_score: + best_score = score + best_option = option + + if best_option: + cleaned_option = _clean_claude_schema(best_option, for_gemini) + if isinstance(cleaned_option, dict): + if len(type_names) > 1: + hint = f"one of: {', '.join(type_names)}" + if "description" in cleaned_option: + cleaned_option["description"] = f"{cleaned_option['description']} ({hint})" + else: + cleaned_option["description"] = hint + return cleaned_option + + # Handle 'allOf' by merging all schemas together using the helper function + if "allOf" in schema and isinstance(schema["allOf"], list) and schema["allOf"]: + # Use the dedicated merge function + merged_schema = _merge_all_of(schema) + # Then clean the merged result + return _clean_claude_schema(merged_schema, for_gemini) cleaned = {} # Handle 'const' by converting to 'enum' with single value (Claude only) @@ -3683,6 +3959,7 @@ def _transform_to_antigravity_format( max_tokens: Optional[int] = None, reasoning_effort: Optional[str] = None, tool_choice: Optional[Union[str, Dict[str, Any]]] = None, + original_messages: Optional[List[Dict[str, Any]]] = None, ) -> Dict[str, Any]: """ Transform Gemini CLI payload to complete Antigravity format. @@ -3692,6 +3969,7 @@ def _transform_to_antigravity_format( model: Model name (public alias) max_tokens: Max output tokens (including thinking) reasoning_effort: Reasoning effort level (determines -thinking variant for Claude) + original_messages: Original Anthropic-format messages for session ID derivation """ internal_model = self._alias_to_internal(model) @@ -3731,8 +4009,13 @@ def _transform_to_antigravity_format( "request": copy.deepcopy(gemini_payload), } - # Add session ID - antigravity_payload["request"]["sessionId"] = _generate_session_id() + # Add session ID - derive from first user message for prompt caching continuity + if original_messages: + antigravity_payload["request"]["sessionId"] = _derive_session_id( + original_messages + ) + else: + antigravity_payload["request"]["sessionId"] = _generate_session_id() # Add default safety settings to prevent content filtering # Only add if not already present in the payload @@ -3777,6 +4060,16 @@ def _transform_to_antigravity_format( ) gen_config["maxOutputTokens"] = min_required_tokens + # Cap maxOutputTokens for Gemini models to their limit (16K) + # Gemini models have a lower output limit than Claude + if not is_claude and gen_config.get("maxOutputTokens"): + current_max = gen_config["maxOutputTokens"] + if current_max > GEMINI_MAX_OUTPUT_TOKENS: + lib_logger.debug( + f"Capping maxOutputTokens from {current_max} to {GEMINI_MAX_OUTPUT_TOKENS} for Gemini model" + ) + gen_config["maxOutputTokens"] = GEMINI_MAX_OUTPUT_TOKENS + antigravity_payload["request"]["generationConfig"] = gen_config # Set toolConfig based on tool_choice parameter @@ -4539,7 +4832,8 @@ async def acompletion( # Transform to Antigravity format with real project ID payload = self._transform_to_antigravity_format( - gemini_payload, model, project_id, max_tokens, reasoning_effort, tool_choice + gemini_payload, model, project_id, max_tokens, reasoning_effort, tool_choice, + original_messages=messages ) file_logger.log_request(payload) @@ -4599,6 +4893,7 @@ async def acompletion( max_tokens, reasoning_effort, tool_choice, + original_messages=messages, ) else: # Non-streaming: empty response, bare 429, and malformed call retry @@ -4724,6 +5019,7 @@ async def acompletion( max_tokens, reasoning_effort, tool_choice, + original_messages=messages, ) # Log the retry request in the same folder @@ -5050,6 +5346,7 @@ async def _streaming_with_retry( max_tokens: Optional[int] = None, reasoning_effort: Optional[str] = None, tool_choice: Optional[Union[str, Dict[str, Any]]] = None, + original_messages: Optional[List[Dict[str, Any]]] = None, ) -> AsyncGenerator[litellm.ModelResponse, None]: """ Wrapper around _handle_streaming that retries on empty responses, bare 429s, @@ -5197,6 +5494,7 @@ async def _streaming_with_retry( max_tokens, reasoning_effort, tool_choice, + original_messages=original_messages, ) # Log the retry request in the same folder From dc19691b8d30866af102bde42668f88b9d2ab1a7 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Fri, 2 Jan 2026 09:14:43 +0100 Subject: [PATCH 25/36] fix(antigravity): make interleaved thinking hint more explicit Use structured format with CRITICAL prefix and bullet points to reduce skipped thinking blocks between tool calls. Signed-off-by: Moeeze Hassan --- src/rotator_library/providers/antigravity_provider.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index a991a92de..12e93d94c 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -294,7 +294,10 @@ def _get_claude_thinking_cache_file(): DEFAULT_PARALLEL_TOOL_INSTRUCTION = """When multiple independent operations are needed, prefer making parallel tool calls in a single response rather than sequential calls across multiple responses. This reduces round-trips and improves efficiency. Only use sequential calls when one tool's output is required as input for another.""" # Claude interleaved thinking hint (encourages thinking after tool results) -DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """Interleaved thinking is enabled. Always emit a thinking block before each tool call and after each tool result, even if brief, before deciding the next action or final answer.""" +DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """CRITICAL: Interleaved thinking is required. Emit a thinking block: +- Before every tool call (to reason about what you're doing) +- After every tool result (to analyze the result before proceeding) +Never skip thinking, even for simple or sequential tool calls.""" # ============================================================================= From 5a8258ca891ac14b9ea5f2943d80351b10cfee1f Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Mon, 5 Jan 2026 15:48:57 +0100 Subject: [PATCH 26/36] fix(antigravity): reject requests exceeding Claude's 64K max_tokens limit Instead of silently capping max_tokens, raise a ValueError so Claude Code sees the error and can adjust its request. Fixes 400 INVALID_ARGUMENT errors when clients send max_tokens > 64000 for Claude models. --- .../providers/antigravity_provider.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 12e93d94c..e5f429b25 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -132,6 +132,10 @@ def _env_int(key: str, default: int) -> int: # See: https://ai.google.dev/gemini-api/docs/models GEMINI_MAX_OUTPUT_TOKENS = 16384 +# Claude max output tokens cap - Claude models have a 64K output limit +# See: https://docs.anthropic.com/en/docs/about-claude/models +CLAUDE_MAX_OUTPUT_TOKENS = 64000 + # Empty response retry configuration # When Antigravity returns an empty response (no content, no tool calls), # automatically retry up to this many attempts before giving up (minimum 1) @@ -4073,6 +4077,16 @@ def _transform_to_antigravity_format( ) gen_config["maxOutputTokens"] = GEMINI_MAX_OUTPUT_TOKENS + # Reject requests that exceed Claude's max_tokens limit (64K) + # Let the client see the error so it can adjust its request + if is_claude and gen_config.get("maxOutputTokens"): + current_max = gen_config["maxOutputTokens"] + if current_max > CLAUDE_MAX_OUTPUT_TOKENS: + raise ValueError( + f"max_tokens: {current_max} > {CLAUDE_MAX_OUTPUT_TOKENS}, " + f"which is the maximum allowed number of output tokens for {model}" + ) + antigravity_payload["request"]["generationConfig"] = gen_config # Set toolConfig based on tool_choice parameter From bbc1060dfcb11dcdff99f80612d062caa468a8e9 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Mon, 5 Jan 2026 17:40:54 +0100 Subject: [PATCH 27/36] experimental: try to be more explicit about must think instruction Signed-off-by: Moeeze Hassan --- .../providers/antigravity_provider.py | 161 +++++++++++++----- 1 file changed, 122 insertions(+), 39 deletions(-) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index e5f429b25..f370ca4a9 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -29,6 +29,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Dict, @@ -36,20 +37,19 @@ Optional, Tuple, Union, - TYPE_CHECKING, ) import httpx import litellm -from .provider_interface import ProviderInterface, UsageResetConfigDef, QuotaGroupMap +from ..error_handler import EmptyResponseError, TransientQuotaError +from ..model_definitions import ModelDefinitions +from ..timeout_config import TimeoutConfig +from ..utils.paths import get_cache_dir, get_logs_dir from .antigravity_auth_base import AntigravityAuthBase from .provider_cache import ProviderCache +from .provider_interface import ProviderInterface, QuotaGroupMap, UsageResetConfigDef from .utilities.antigravity_quota_tracker import AntigravityQuotaTracker -from ..model_definitions import ModelDefinitions -from ..timeout_config import TimeoutConfig -from ..error_handler import EmptyResponseError, TransientQuotaError -from ..utils.paths import get_logs_dir, get_cache_dir if TYPE_CHECKING: from ..usage_manager import UsageManager @@ -153,7 +153,11 @@ def _env_int(key: str, default: int) -> int: def _is_valid_thinking_signature(signature): - return isinstance(signature, str) and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH + return ( + isinstance(signature, str) and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH + ) + + MALFORMED_CALL_RETRY_DELAY = _env_int("ANTIGRAVITY_MALFORMED_CALL_DELAY", 1) # Model alias mappings (internal ↔ public) @@ -263,7 +267,7 @@ def _get_claude_thinking_cache_file(): ## COMMON FAILURE PATTERNS TO AVOID - Using 'path' when schema says 'filePath' (or vice versa) -- Using 'content' when schema says 'text' (or vice versa) +- Using 'content' when schema says 'text' (or vice versa) - Providing {"file": "..."} when schema wants [{"path": "...", "line_ranges": [...]}] - Omitting required nested fields in array items - Adding 'additionalProperties' that the schema doesn't define @@ -298,11 +302,16 @@ def _get_claude_thinking_cache_file(): DEFAULT_PARALLEL_TOOL_INSTRUCTION = """When multiple independent operations are needed, prefer making parallel tool calls in a single response rather than sequential calls across multiple responses. This reduces round-trips and improves efficiency. Only use sequential calls when one tool's output is required as input for another.""" # Claude interleaved thinking hint (encourages thinking after tool results) -DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """CRITICAL: Interleaved thinking is required. Emit a thinking block: +DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """CRITICAL: Interleaved thinking is required and IS UNCOMPROMISINGLY A MUST DO. Emit a thinking block: - Before every tool call (to reason about what you're doing) - After every tool result (to analyze the result before proceeding) Never skip thinking, even for simple or sequential tool calls.""" +# Short reminder appended to tool results to reinforce interleaved thinking +DEFAULT_CLAUDE_TOOL_RESULT_THINKING_REMINDER = """ +CRITICAL: Interleaved thinking is required. You MUST emit a thinking block NOW to analyze this tool result before proceeding with any response or tool call. +""" + # ============================================================================= # HELPER FUNCTIONS @@ -674,7 +683,9 @@ def _merge_all_of(schema: Any) -> Any: sub_schema = _merge_all_of(sub_schema) # Merge properties (later overrides earlier) - if "properties" in sub_schema and isinstance(sub_schema["properties"], dict): + if "properties" in sub_schema and isinstance( + sub_schema["properties"], dict + ): for key, value in sub_schema["properties"].items(): merged_properties[key] = value @@ -685,7 +696,10 @@ def _merge_all_of(schema: Any) -> Any: # Copy other fields (first occurrence wins) for key, value in sub_schema.items(): - if key not in ("properties", "required", "allOf") and key not in other_fields: + if ( + key not in ("properties", "required", "allOf") + and key not in other_fields + ): other_fields[key] = value # Build result without allOf @@ -811,7 +825,9 @@ def _clean_claude_schema(schema: Any, for_gemini: bool = False) -> Any: if not isinstance(option, dict): continue # Collect type names for hint - type_name = option.get("type") or ("object" if "properties" in option else None) + type_name = option.get("type") or ( + "object" if "properties" in option else None + ) if type_name and type_name != "null": type_names.append(type_name) # Score and track best @@ -827,7 +843,9 @@ def _clean_claude_schema(schema: Any, for_gemini: bool = False) -> Any: if len(type_names) > 1: hint = f"one of: {', '.join(type_names)}" if "description" in cleaned_option: - cleaned_option["description"] = f"{cleaned_option['description']} ({hint})" + cleaned_option["description"] = ( + f"{cleaned_option['description']} ({hint})" + ) else: cleaned_option["description"] = hint return cleaned_option @@ -842,7 +860,9 @@ def _clean_claude_schema(schema: Any, for_gemini: bool = False) -> Any: for option in options: if not isinstance(option, dict): continue - type_name = option.get("type") or ("object" if "properties" in option else None) + type_name = option.get("type") or ( + "object" if "properties" in option else None + ) if type_name and type_name != "null": type_names.append(type_name) score = _score_schema_option(option) @@ -856,7 +876,9 @@ def _clean_claude_schema(schema: Any, for_gemini: bool = False) -> Any: if len(type_names) > 1: hint = f"one of: {', '.join(type_names)}" if "description" in cleaned_option: - cleaned_option["description"] = f"{cleaned_option['description']} ({hint})" + cleaned_option["description"] = ( + f"{cleaned_option['description']} ({hint})" + ) else: cleaned_option["description"] = hint return cleaned_option @@ -1430,6 +1452,13 @@ def __init__(self): "ANTIGRAVITY_CLAUDE_INTERLEAVED_HINT", DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT, ) + self._enable_claude_tool_result_reminder = _env_bool( + "ANTIGRAVITY_ENABLE_CLAUDE_TOOL_RESULT_REMINDER", True + ) + self._claude_tool_result_reminder = os.getenv( + "ANTIGRAVITY_CLAUDE_TOOL_RESULT_REMINDER", + DEFAULT_CLAUDE_TOOL_RESULT_THINKING_REMINDER, + ) # Parallel tool usage instruction configuration self._enable_parallel_tool_instruction_claude = _env_bool( @@ -1456,7 +1485,8 @@ def _log_config(self) -> None: f"claude_fix={self._enable_claude_tool_fix}, thinking_sanitization={self._enable_thinking_sanitization}, " f"parallel_tool_claude={self._enable_parallel_tool_instruction_claude}, " f"parallel_tool_gemini3={self._enable_parallel_tool_instruction_gemini3}, " - f"claude_interleaved_hint={self._enable_claude_interleaved_hint}" + f"claude_interleaved_hint={self._enable_claude_interleaved_hint}, " + f"claude_tool_result_reminder={self._enable_claude_tool_result_reminder}" ) def _get_antigravity_headers(self) -> Dict[str, str]: @@ -2570,7 +2600,11 @@ def _get_thinking_config( if not reasoning_effort: if is_claude: # Use client-provided budget if available, otherwise use default - budget = thinking_budget if thinking_budget else CLAUDE_FORCED_THINKING_BUDGET + budget = ( + thinking_budget + if thinking_budget + else CLAUDE_FORCED_THINKING_BUDGET + ) return { "thinkingBudget": budget, "include_thoughts": True, @@ -2582,7 +2616,9 @@ def _get_thinking_config( if is_claude: # Use client-provided budget if available, otherwise use default - budget = thinking_budget if thinking_budget else CLAUDE_FORCED_THINKING_BUDGET + budget = ( + thinking_budget if thinking_budget else CLAUDE_FORCED_THINKING_BUDGET + ) return { "thinkingBudget": budget, "include_thoughts": True, @@ -2908,6 +2944,13 @@ def _transform_tool_message( func_name = GEMINI3_TOOL_RENAMES.get(func_name, func_name) func_name = f"{self._gemini3_tool_prefix}{func_name}" + # Determine if we should add thinking reminder for Claude + should_add_reminder = ( + self._is_claude(model) + and self._enable_claude_tool_result_reminder + and self._claude_tool_result_reminder + ) + # Handle multimodal content (array with text and images) if isinstance(content, list): text_parts = [] @@ -2928,27 +2971,41 @@ def _transform_tool_message( # Parse: data:image/png;base64,iVBORw0KG... header, data = image_url.split(",", 1) mime_type = header.split(":")[1].split(";")[0] - image_parts.append({ - "inlineData": { - "mimeType": mime_type, - "data": data, + image_parts.append( + { + "inlineData": { + "mimeType": mime_type, + "data": data, + } } - }) + ) except Exception as e: - lib_logger.warning(f"Failed to parse image data URL in tool response: {e}") + lib_logger.warning( + f"Failed to parse image data URL in tool response: {e}" + ) # Build the result parts parts = [] # Add function response with text content text_result = " ".join(text_parts) if text_parts else "" - parts.append({ - "functionResponse": { - "name": func_name, - "response": {"result": text_result if text_result else "Image content provided"}, - "id": tool_id, + result_content = ( + text_result if text_result else "Image content provided" + ) + + # Append thinking reminder for Claude + if should_add_reminder: + result_content = f"{result_content}\n\n{self._claude_tool_result_reminder}" + + parts.append( + { + "functionResponse": { + "name": func_name, + "response": {"result": result_content}, + "id": tool_id, + } } - }) + ) # Add image parts separately (Gemini handles these as additional parts) parts.extend(image_parts) @@ -2961,6 +3018,14 @@ def _transform_tool_message( except (json.JSONDecodeError, TypeError): parsed_content = content + # Append thinking reminder for Claude (for string/parsed content) + if should_add_reminder: + if isinstance(parsed_content, str): + parsed_content = f"{parsed_content}\n\n{self._claude_tool_result_reminder}" + elif isinstance(parsed_content, dict): + # For dict results, add as a separate key + parsed_content["_system_reminder"] = self._claude_tool_result_reminder + return [ { "functionResponse": { @@ -4110,8 +4175,13 @@ def _transform_to_antigravity_format( if is_claude: thinking_config = gen_config.get("thinkingConfig", {}) if thinking_config: - if "includeThoughts" in thinking_config and "include_thoughts" not in thinking_config: - thinking_config["include_thoughts"] = thinking_config.pop("includeThoughts") + if ( + "includeThoughts" in thinking_config + and "include_thoughts" not in thinking_config + ): + thinking_config["include_thoughts"] = thinking_config.pop( + "includeThoughts" + ) if "thinkingBudget" in thinking_config: budget = thinking_config.pop("thinkingBudget") @@ -4234,7 +4304,9 @@ def _gemini_to_openai_chunk( # Accumulate signature for Claude caching if has_sig and is_thought and accumulator is not None: - if not self._is_claude(model) or _is_valid_thinking_signature(signature): + if not self._is_claude(model) or _is_valid_thinking_signature( + signature + ): accumulator["thought_signature"] = signature # Skip standalone signature parts @@ -4351,7 +4423,9 @@ def _gemini_to_openai_non_streaming( ) if has_sig and is_thought: - if not self._is_claude(model) or _is_valid_thinking_signature(signature): + if not self._is_claude(model) or _is_valid_thinking_signature( + signature + ): thought_sig = signature text_value = None @@ -4604,9 +4678,7 @@ def _cache_thinking( ) -> None: """Cache Claude thinking content.""" if not _is_valid_thinking_signature(signature): - lib_logger.debug( - "[Thinking Cache] Skipping cache due to invalid signature" - ) + lib_logger.debug("[Thinking Cache] Skipping cache due to invalid signature") return cache_key = self._generate_thinking_cache_key(text, tool_calls) @@ -4781,12 +4853,18 @@ async def acompletion( ) # Add interleaved thinking hint for Claude thinking models with tools + # Prepend at start AND append at end for maximum emphasis if ( tools and self._is_claude(model) and thinking_enabled and self._enable_claude_interleaved_hint ): + # Prepend at start of system instructions + self._inject_tool_hardening_instruction( + gemini_payload, self._claude_interleaved_hint + ) + # Also append at end of system instructions self._append_system_instruction( gemini_payload, self._claude_interleaved_hint ) @@ -4849,8 +4927,13 @@ async def acompletion( # Transform to Antigravity format with real project ID payload = self._transform_to_antigravity_format( - gemini_payload, model, project_id, max_tokens, reasoning_effort, tool_choice, - original_messages=messages + gemini_payload, + model, + project_id, + max_tokens, + reasoning_effort, + tool_choice, + original_messages=messages, ) file_logger.log_request(payload) From d4ad8af144fd0a3f5b210604acb87fd5fb839b8a Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 8 Jan 2026 10:06:05 +0100 Subject: [PATCH 28/36] feat(anthropic): respect explicit thinking_budget from Anthropic routes - Add explicit_budget parameter to _get_thinking_config - Cap Claude thinking budget at 31999 when explicit budget provided - Pass thinking_budget kwarg from Anthropic translator to provider --- .../providers/antigravity_provider.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 240950f96..edaf02668 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -1944,7 +1944,10 @@ def _close_tool_loop_for_thinking( # ========================================================================= def _get_thinking_config( - self, reasoning_effort: Optional[str], model: str + self, + reasoning_effort: Optional[str], + model: str, + explicit_budget: Optional[int] = None, ) -> Optional[Dict[str, Any]]: """ Map reasoning_effort to thinking configuration. @@ -1952,6 +1955,9 @@ def _get_thinking_config( - Gemini 2.5 & Claude: thinkingBudget (integer tokens) - Gemini 3 Pro: thinkingLevel (string: "low"/"high") - Gemini 3 Flash: thinkingLevel (string: "minimal"/"low"/"medium"/"high") + + If explicit_budget is provided (from Anthropic route), it takes precedence + over reasoning_effort mapping. For Claude, explicit budget is capped at 31999. """ internal = self._alias_to_internal(model) is_gemini_25 = "gemini-2.5" in model @@ -1959,6 +1965,18 @@ def _get_thinking_config( is_gemini_3_flash = "gemini-3-flash" in model or "gemini-3-flash" in internal is_claude = self._is_claude(model) + if not (is_gemini_25 or is_gemini_3 or is_claude): + return None + + # Handle explicit budget from Anthropic route (takes precedence) + if explicit_budget is not None and (is_gemini_25 or is_claude): + if explicit_budget <= 0: + return {"thinkingBudget": 0, "include_thoughts": False} + # Cap Claude budget at max allowed + if is_claude: + explicit_budget = min(explicit_budget, CLAUDE_FORCED_THINKING_BUDGET) + return {"thinkingBudget": explicit_budget, "include_thoughts": True} + if not (is_gemini_25 or is_gemini_3 or is_claude): return None @@ -3756,7 +3774,10 @@ async def acompletion( # Gemini 3 performs better with temperature=1 for tool use gen_config["temperature"] = 1.0 - thinking_config = self._get_thinking_config(reasoning_effort, model) + explicit_thinking_budget = kwargs.get("thinking_budget") + thinking_config = self._get_thinking_config( + reasoning_effort, model, explicit_thinking_budget + ) if thinking_config: gen_config.setdefault("thinkingConfig", {}).update(thinking_config) From 9d568fe9d94aab5db4c6087115ba1f27d12a0590 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 8 Jan 2026 10:13:50 +0100 Subject: [PATCH 29/36] feat(anthropic): always use max thinking budget (31999) for Claude Ignore client's budget_tokens value and always use 31999 for Claude via Anthropic routes to ensure full thinking capacity. --- .../anthropic_compat/translator.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index dfbea758e..7b6122990 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -166,14 +166,20 @@ def anthropic_to_openai_messages( ) elif block_type == "thinking": signature = block.get("signature", "") - if signature and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH: + if ( + signature + and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH + ): thinking_text = block.get("thinking", "") if thinking_text: reasoning_content += thinking_text thinking_signature = signature elif block_type == "redacted_thinking": signature = block.get("signature", "") - if signature and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH: + if ( + signature + and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH + ): thinking_signature = signature elif block_type == "tool_use": # Anthropic tool_use -> OpenAI tool_calls @@ -227,7 +233,9 @@ def anthropic_to_openai_messages( tool_content_parts.append( { "type": "image_url", - "image_url": {"url": source.get("url", "")}, + "image_url": { + "url": source.get("url", "") + }, } ) @@ -268,7 +276,9 @@ def anthropic_to_openai_messages( { "role": "tool", "tool_call_id": block.get("tool_use_id", ""), - "content": str(tool_content) if tool_content else "", + "content": str(tool_content) + if tool_content + else "", } ) continue # Don't add to current message @@ -538,24 +548,16 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, # and doesn't affect the model's behavior. # Handle Anthropic thinking config -> reasoning_effort translation - # Pass through the exact budget_tokens value when specified, allowing the - # provider to use the client's requested thinking budget directly. + # Always use max thinking budget (31999) for Claude via Anthropic routes if request.thinking: if request.thinking.type == "enabled": - budget = request.thinking.budget_tokens - if budget: - # Pass the exact budget through for the provider to use - openai_request["reasoning_effort"] = "high" - openai_request["thinking_budget"] = budget - else: - # No specific budget requested, use high effort - openai_request["reasoning_effort"] = "high" + openai_request["reasoning_effort"] = "high" + openai_request["thinking_budget"] = 31999 elif request.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" elif _is_opus_model(request.model): - # Enable thinking for Opus models when no thinking config is provided openai_request["reasoning_effort"] = "high" - + openai_request["thinking_budget"] = 31999 return openai_request @@ -581,8 +583,8 @@ def _is_opus_model(model_name: str) -> bool: # - "antigravity/claude-opus-4-5" # Avoid matching things like "magnum-opus" or other non-Claude models opus_patterns = [ - r'claude[-_]?opus', # "claude-opus", "claude_opus", "claudeopus" - r'opus[-_]?\d', # "opus-4", "opus_4", "opus4" (with version number) - r'\d[-_]?opus(?:[-_]|$)', # "4-opus", "4_opus" at word boundary + r"claude[-_]?opus", # "claude-opus", "claude_opus", "claudeopus" + r"opus[-_]?\d", # "opus-4", "opus_4", "opus4" (with version number) + r"\d[-_]?opus(?:[-_]|$)", # "4-opus", "4_opus" at word boundary ] return any(re.search(pattern, model_lower) for pattern in opus_patterns) From 67ffea53f5596d2595317f4635a2126b972b8650 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 8 Jan 2026 14:19:31 +0100 Subject: [PATCH 30/36] fix(anthropic): inject [Continue] for fresh thinking turn when history lacks thinking blocks --- .../anthropic_compat/translator.py | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 7b6122990..649b0c8e1 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -497,6 +497,26 @@ def openai_to_anthropic_response(openai_response: dict, original_model: str) -> } +def _history_supports_thinking(anthropic_messages: List[dict]) -> bool: + for msg in anthropic_messages: + if msg.get("role") != "assistant": + continue + content = msg.get("content", "") + if not isinstance(content, list): + return False + first_block = next((b for b in content if isinstance(b, dict)), None) + if not first_block: + return False + if first_block.get("type") not in ("thinking", "redacted_thinking"): + return False + return True + + +def _inject_continue_for_fresh_thinking_turn(openai_messages: List[dict]) -> List[dict]: + openai_messages.append({"role": "user", "content": "[Continue]"}) + return openai_messages + + def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, Any]: """ Translate a complete Anthropic Messages API request to OpenAI format. @@ -512,9 +532,11 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, """ anthropic_request = request.model_dump(exclude_none=True) + messages = anthropic_request.get("messages", []) openai_messages = anthropic_to_openai_messages( - anthropic_request.get("messages", []), anthropic_request.get("system") + messages, anthropic_request.get("system") ) + thinking_compatible = _history_supports_thinking(messages) openai_tools = anthropic_to_openai_tools(anthropic_request.get("tools")) openai_tool_choice = anthropic_to_openai_tool_choice( @@ -551,11 +573,19 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, # Always use max thinking budget (31999) for Claude via Anthropic routes if request.thinking: if request.thinking.type == "enabled": + if not thinking_compatible: + openai_messages = _inject_continue_for_fresh_thinking_turn( + openai_messages + ) + openai_request["messages"] = openai_messages openai_request["reasoning_effort"] = "high" openai_request["thinking_budget"] = 31999 elif request.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" elif _is_opus_model(request.model): + if not thinking_compatible: + openai_messages = _inject_continue_for_fresh_thinking_turn(openai_messages) + openai_request["messages"] = openai_messages openai_request["reasoning_effort"] = "high" openai_request["thinking_budget"] = 31999 return openai_request From b7b5d07b7e14a1476fffe60ecb3e14e3d0df62ff Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Thu, 8 Jan 2026 14:36:24 +0100 Subject: [PATCH 31/36] fix(token-count): include Antigravity preprompt tokens in count Token counting endpoints (/v1/token-count and /v1/messages/count_tokens) were returning inaccurate counts because they didn't include the Antigravity preprompts that get injected during actual API calls. - Add get_antigravity_preprompt_text() helper to expose preprompt text - Update RotatingClient.token_count() to add preprompt tokens for Antigravity provider models Signed-off-by: Moeeze Hassan --- src/rotator_library/client.py | 31 ++++++++++++++-- .../providers/antigravity_provider.py | 36 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/rotator_library/client.py b/src/rotator_library/client.py index 5fc35c359..caa0d02cc 100644 --- a/src/rotator_library/client.py +++ b/src/rotator_library/client.py @@ -2464,7 +2464,12 @@ def aembedding( ) def token_count(self, **kwargs) -> int: - """Calculates the number of tokens for a given text or list of messages.""" + """Calculates the number of tokens for a given text or list of messages. + + For Antigravity provider models, this also includes the preprompt tokens + that get injected during actual API calls (agent instruction + identity override). + This ensures token counts match actual usage. + """ kwargs = self._convert_model_params(**kwargs) model = kwargs.get("model") text = kwargs.get("text") @@ -2472,13 +2477,33 @@ def token_count(self, **kwargs) -> int: if not model: raise ValueError("'model' is a required parameter.") + + # Calculate base token count if messages: - return token_counter(model=model, messages=messages) + base_count = token_counter(model=model, messages=messages) elif text: - return token_counter(model=model, text=text) + base_count = token_counter(model=model, text=text) else: raise ValueError("Either 'text' or 'messages' must be provided.") + # Add preprompt tokens for Antigravity provider + # The Antigravity provider injects system instructions during actual API calls, + # so we need to account for those tokens in the count + provider = model.split("/")[0] if "/" in model else "" + if provider == "antigravity": + try: + from .providers.antigravity_provider import get_antigravity_preprompt_text + + preprompt_text = get_antigravity_preprompt_text() + if preprompt_text: + preprompt_tokens = token_counter(model=model, text=preprompt_text) + base_count += preprompt_tokens + except ImportError: + # Provider not available, skip preprompt token counting + pass + + return base_count + async def get_available_models(self, provider: str) -> List[str]: """Returns a list of available models for a specific provider, with caching.""" lib_logger.info(f"Getting available models for provider: {provider}") diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index edaf02668..f260db5cb 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -449,6 +449,42 @@ def _is_valid_thinking_signature(signature): # ============================================================================= +def get_antigravity_preprompt_text() -> str: + """ + Get the combined Antigravity preprompt text that gets injected into requests. + + This function returns the exact text that gets prepended to system instructions + during actual API calls. It respects the current configuration settings: + - PREPEND_INSTRUCTION: Whether to include any preprompt at all + - USE_SHORT_ANTIGRAVITY_PROMPTS: Whether to use short or full versions + - INJECT_IDENTITY_OVERRIDE: Whether to include the identity override + + This is useful for accurate token counting - the token count endpoints should + include these preprompts to match what actually gets sent to the API. + + Returns: + The combined preprompt text, or empty string if prepending is disabled. + """ + if not PREPEND_INSTRUCTION: + return "" + + # Choose prompt versions based on USE_SHORT_ANTIGRAVITY_PROMPTS setting + if USE_SHORT_ANTIGRAVITY_PROMPTS: + agent_instruction = ANTIGRAVITY_AGENT_SYSTEM_INSTRUCTION_SHORT + override_instruction = ANTIGRAVITY_IDENTITY_OVERRIDE_INSTRUCTION_SHORT + else: + agent_instruction = ANTIGRAVITY_AGENT_SYSTEM_INSTRUCTION + override_instruction = ANTIGRAVITY_IDENTITY_OVERRIDE_INSTRUCTION + + # Build the combined preprompt + parts = [agent_instruction] + + if INJECT_IDENTITY_OVERRIDE: + parts.append(override_instruction) + + return "\n".join(parts) + + def _sanitize_headers(headers: Dict[str, str]) -> Dict[str, str]: """ Strip identifiable client headers for privacy/security. From 49d2e474fc0442a51dd6557523fa8aa36caeb918 Mon Sep 17 00:00:00 2001 From: Moeeze Hassan Date: Sat, 10 Jan 2026 01:04:32 +0100 Subject: [PATCH 32/36] fix(antigravity): remove stale interleaved thinking references Remove remaining references to removed interleaved thinking attributes that were brought in during merge. --- .../providers/antigravity_provider.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 03e170012..09c3b97c7 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -1436,9 +1436,7 @@ def _log_config(self) -> None: f"gemini3_fix={self._enable_gemini3_tool_fix}, gemini3_strict_schema={self._gemini3_enforce_strict_schema}, " f"claude_fix={self._enable_claude_tool_fix}, thinking_sanitization={self._enable_thinking_sanitization}, " f"parallel_tool_claude={self._enable_parallel_tool_instruction_claude}, " - f"parallel_tool_gemini3={self._enable_parallel_tool_instruction_gemini3}, " - f"claude_interleaved_hint={self._enable_claude_interleaved_hint}, " - f"claude_tool_result_reminder={self._enable_claude_tool_result_reminder}" + f"parallel_tool_gemini3={self._enable_parallel_tool_instruction_gemini3}" ) def _sanitize_tool_name(self, name: str) -> str: @@ -2763,25 +2761,11 @@ def _transform_tool_message( func_name = GEMINI3_TOOL_RENAMES.get(func_name, func_name) func_name = f"{self._gemini3_tool_prefix}{func_name}" - should_add_reminder = ( - self._is_claude(model) - and self._enable_claude_tool_result_reminder - and self._claude_tool_result_reminder - ) - try: parsed_content = json.loads(content) except (json.JSONDecodeError, TypeError): parsed_content = content - if should_add_reminder: - if isinstance(parsed_content, str): - parsed_content = ( - f"{parsed_content}\n\n{self._claude_tool_result_reminder}" - ) - elif isinstance(parsed_content, dict): - parsed_content["_system_reminder"] = self._claude_tool_result_reminder - return [ { "functionResponse": { From 8e10a66ba96e8dd41140ea3cc8b0a9b9b9b8940e Mon Sep 17 00:00:00 2001 From: Mirrowel <28632877+Mirrowel@users.noreply.github.com> Date: Thu, 15 Jan 2026 18:17:10 +0100 Subject: [PATCH 33/36] =?UTF-8?q?refactor(rotator=5Flibrary):=20?= =?UTF-8?q?=F0=9F=94=A8=20standardize=20thinking=20budget=20mapping=20and?= =?UTF-8?q?=20remove=20legacy=20hacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces ad-hoc thinking logic with a structured mapping from Anthropic `budget_tokens` to `reasoning_effort` levels. This change aligns the translation layer with standard provider capabilities and cleans up deprecated workarounds. - Implement `_budget_to_reasoning_effort` to convert token counts to reasoning levels (e.g., "low", "medium", "high", "granular"). - Remove legacy logic that forced max thinking budget for Claude Opus models. - Remove workaround for injecting "[Continue]" messages into conversation history. - Delete unused helper functions in `AntigravityProvider` (signature validation, content reordering, and explicit budget overrides). --- .../anthropic_compat/translator.py | 132 +++++++++--------- .../providers/antigravity_provider.py | 104 +------------- 2 files changed, 70 insertions(+), 166 deletions(-) diff --git a/src/rotator_library/anthropic_compat/translator.py b/src/rotator_library/anthropic_compat/translator.py index 649b0c8e1..875d19b6a 100644 --- a/src/rotator_library/anthropic_compat/translator.py +++ b/src/rotator_library/anthropic_compat/translator.py @@ -14,6 +14,67 @@ MIN_THINKING_SIGNATURE_LENGTH = 100 +# ============================================================================= +# THINKING BUDGET TO REASONING EFFORT MAPPING +# ============================================================================= + +# Budget thresholds for reasoning effort levels (based on token counts) +# These map Anthropic's budget_tokens to OpenAI-style reasoning_effort levels +THINKING_BUDGET_THRESHOLDS = { + "minimal": 4096, + "low": 8192, + "low_medium": 12288, + "medium": 16384, + "medium_high": 24576, + "high": 32768, +} + +# Providers that support granular reasoning effort levels (low_medium, medium_high, etc.) +# Other providers will receive simplified levels (low, medium, high) +GRANULAR_REASONING_PROVIDERS = {"antigravity"} + + +def _budget_to_reasoning_effort(budget_tokens: int, model: str) -> str: + """ + Map Anthropic thinking budget_tokens to a reasoning_effort level. + + Args: + budget_tokens: The thinking budget in tokens from the Anthropic request + model: The model name (used to determine if provider supports granular levels) + + Returns: + A reasoning_effort level string (e.g., "low", "medium", "high") + """ + # Determine granular level based on budget + if budget_tokens <= THINKING_BUDGET_THRESHOLDS["minimal"]: + granular_level = "minimal" + elif budget_tokens <= THINKING_BUDGET_THRESHOLDS["low"]: + granular_level = "low" + elif budget_tokens <= THINKING_BUDGET_THRESHOLDS["low_medium"]: + granular_level = "low_medium" + elif budget_tokens <= THINKING_BUDGET_THRESHOLDS["medium"]: + granular_level = "medium" + elif budget_tokens <= THINKING_BUDGET_THRESHOLDS["medium_high"]: + granular_level = "medium_high" + else: + granular_level = "high" + + # Check if provider supports granular levels + provider = model.split("/")[0].lower() if "/" in model else "" + if provider in GRANULAR_REASONING_PROVIDERS: + return granular_level + + # Simplify to basic levels for non-granular providers + simplify_map = { + "minimal": "low", + "low": "low", + "low_medium": "medium", + "medium": "medium", + "medium_high": "high", + "high": "high", + } + return simplify_map.get(granular_level, "medium") + def _reorder_assistant_content(content: List[dict]) -> List[dict]: """ @@ -497,26 +558,6 @@ def openai_to_anthropic_response(openai_response: dict, original_model: str) -> } -def _history_supports_thinking(anthropic_messages: List[dict]) -> bool: - for msg in anthropic_messages: - if msg.get("role") != "assistant": - continue - content = msg.get("content", "") - if not isinstance(content, list): - return False - first_block = next((b for b in content if isinstance(b, dict)), None) - if not first_block: - return False - if first_block.get("type") not in ("thinking", "redacted_thinking"): - return False - return True - - -def _inject_continue_for_fresh_thinking_turn(openai_messages: List[dict]) -> List[dict]: - openai_messages.append({"role": "user", "content": "[Continue]"}) - return openai_messages - - def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, Any]: """ Translate a complete Anthropic Messages API request to OpenAI format. @@ -536,7 +577,6 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, openai_messages = anthropic_to_openai_messages( messages, anthropic_request.get("system") ) - thinking_compatible = _history_supports_thinking(messages) openai_tools = anthropic_to_openai_tools(anthropic_request.get("tools")) openai_tool_choice = anthropic_to_openai_tool_choice( @@ -570,51 +610,17 @@ def translate_anthropic_request(request: AnthropicMessagesRequest) -> Dict[str, # and doesn't affect the model's behavior. # Handle Anthropic thinking config -> reasoning_effort translation - # Always use max thinking budget (31999) for Claude via Anthropic routes + # Only set reasoning_effort if thinking is explicitly configured if request.thinking: if request.thinking.type == "enabled": - if not thinking_compatible: - openai_messages = _inject_continue_for_fresh_thinking_turn( - openai_messages + # Only set reasoning_effort if budget_tokens was specified + if request.thinking.budget_tokens is not None: + openai_request["reasoning_effort"] = _budget_to_reasoning_effort( + request.thinking.budget_tokens, request.model ) - openai_request["messages"] = openai_messages - openai_request["reasoning_effort"] = "high" - openai_request["thinking_budget"] = 31999 + # If thinking enabled but no budget specified, don't set anything + # Let the provider decide the default elif request.thinking.type == "disabled": openai_request["reasoning_effort"] = "disable" - elif _is_opus_model(request.model): - if not thinking_compatible: - openai_messages = _inject_continue_for_fresh_thinking_turn(openai_messages) - openai_request["messages"] = openai_messages - openai_request["reasoning_effort"] = "high" - openai_request["thinking_budget"] = 31999 - return openai_request - - -def _is_opus_model(model_name: str) -> bool: - """ - Check if a model name refers to a Claude Opus model. - Uses specific pattern matching to avoid false positives with model names - that might contain "opus" as part of another word. - - Args: - model_name: The model name to check - - Returns: - True if the model is a Claude Opus model, False otherwise - """ - import re - - model_lower = model_name.lower() - # Match Claude Opus models specifically: - # - "claude-opus-4-5", "claude-4-opus", "claude_opus" - # - "opus-4", "opus-4.5", "opus4" (standalone with version) - # - "antigravity/claude-opus-4-5" - # Avoid matching things like "magnum-opus" or other non-Claude models - opus_patterns = [ - r"claude[-_]?opus", # "claude-opus", "claude_opus", "claudeopus" - r"opus[-_]?\d", # "opus-4", "opus_4", "opus4" (with version number) - r"\d[-_]?opus(?:[-_]|$)", # "4-opus", "4_opus" at word boundary - ] - return any(re.search(pattern, model_lower) for pattern in opus_patterns) + return openai_request diff --git a/src/rotator_library/providers/antigravity_provider.py b/src/rotator_library/providers/antigravity_provider.py index 4e4cabf66..102d7a956 100644 --- a/src/rotator_library/providers/antigravity_provider.py +++ b/src/rotator_library/providers/antigravity_provider.py @@ -445,39 +445,6 @@ def _get_claude_thinking_cache_file(): # Exact prompt from CLIProxyAPI commit 1b2f9076715b62610f9f37d417e850832b3c7ed1 ANTIGRAVITY_AGENT_SYSTEM_INSTRUCTION_SHORT = """You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding.You are pair programming with a USER to solve their coding task. The task may require creating a new codebase, modifying or debugging an existing codebase, or simply answering a question.**Absolute paths only****Proactiveness**""" -# ============================================================================= -# CLAUDE INTERLEAVED THINKING CONFIGURATION -# ============================================================================= - -# Claude thinking signature validation -# Minimum length for a thinking signature to be considered valid -MIN_THINKING_SIGNATURE_LENGTH = 100 - -# Forced thinking budget for Claude models -# This is injected to ensure Claude models always produce thinking output -CLAUDE_FORCED_THINKING_BUDGET = 31999 - - -def _is_valid_thinking_signature(signature): - """Check if a thinking signature is valid (meets minimum length requirement).""" - return ( - isinstance(signature, str) and len(signature) >= MIN_THINKING_SIGNATURE_LENGTH - ) - - -# Claude interleaved thinking hint - MUST come AFTER Antigravity system prompts -# This instructs Claude to emit thinking blocks before and after tool calls -DEFAULT_CLAUDE_INTERLEAVED_THINKING_HINT = """CRITICAL: Interleaved thinking is required and IS UNCOMPROMISINGLY A MUST DO. Emit a thinking block: -- Before every tool call (to reason about what you're doing) -- After every tool result (to analyze the result before proceeding) -Never skip thinking, even for simple or sequential tool calls.""" - -# Short reminder appended to tool results to reinforce interleaved thinking -DEFAULT_CLAUDE_TOOL_RESULT_THINKING_REMINDER = """ -CRITICAL: Interleaved thinking is required. You MUST emit a thinking block NOW to analyze this tool result before proceeding with any response or tool call. -""" - - # ============================================================================= # HELPER FUNCTIONS # ============================================================================= @@ -533,56 +500,6 @@ def _sanitize_headers(headers: Dict[str, str]) -> Dict[str, str]: } -def _reorder_assistant_content(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Reorder assistant message content blocks to ensure correct order: - 1. Thinking blocks come first (required when thinking is enabled) - 2. Text blocks come in the middle (filtering out empty ones) - 3. Tool_use blocks come at the end (required before tool_result) - - This matches Anthropic's expected ordering and prevents API errors. - """ - if not isinstance(content, list): - return content - - if len(content) <= 1: - return content - - thinking_blocks = [] - text_blocks = [] - tool_use_blocks = [] - other_blocks = [] - - for block in content: - if not isinstance(block, dict): - other_blocks.append(block) - continue - - block_type = block.get("type", "") - - if block_type in ("thinking", "redacted_thinking"): - sanitized = { - "type": block_type, - "thinking": block.get("thinking", ""), - } - if block.get("signature"): - sanitized["signature"] = block["signature"] - thinking_blocks.append(sanitized) - - elif block_type == "tool_use": - tool_use_blocks.append(block) - - elif block_type == "text": - text = block.get("text", "") - if text and text.strip(): - text_blocks.append(block) - - else: - other_blocks.append(block) - - return thinking_blocks + other_blocks + text_blocks + tool_use_blocks - - def _generate_request_id() -> str: """Generate Antigravity request ID: agent-{uuid}""" return f"agent-{uuid.uuid4()}" @@ -2383,7 +2300,6 @@ def _get_thinking_config( self, reasoning_effort: Optional[str], model: str, - explicit_budget: Optional[int] = None, ) -> Optional[Dict[str, Any]]: """ Map reasoning_effort to thinking configuration. @@ -2391,9 +2307,6 @@ def _get_thinking_config( - Gemini 2.5 & Claude: thinkingBudget (integer tokens) - Gemini 3 Pro: thinkingLevel (string: "low"/"high") - Gemini 3 Flash: thinkingLevel (string: "minimal"/"low"/"medium"/"high") - - If explicit_budget is provided (from Anthropic route), it takes precedence - over reasoning_effort mapping. For Claude, explicit budget is capped at 31999. """ internal = self._alias_to_internal(model) is_gemini_25 = "gemini-2.5" in model @@ -2401,18 +2314,6 @@ def _get_thinking_config( is_gemini_3_flash = "gemini-3-flash" in model or "gemini-3-flash" in internal is_claude = self._is_claude(model) - if not (is_gemini_25 or is_gemini_3 or is_claude): - return None - - # Handle explicit budget from Anthropic route (takes precedence) - if explicit_budget is not None and (is_gemini_25 or is_claude): - if explicit_budget <= 0: - return {"thinkingBudget": 0, "include_thoughts": False} - # Cap Claude budget at max allowed - if is_claude: - explicit_budget = min(explicit_budget, CLAUDE_FORCED_THINKING_BUDGET) - return {"thinkingBudget": explicit_budget, "include_thoughts": True} - if not (is_gemini_25 or is_gemini_3 or is_claude): return None @@ -4131,10 +4032,7 @@ async def acompletion( # Gemini 3 performs better with temperature=1 for tool use gen_config["temperature"] = 1.0 - explicit_thinking_budget = kwargs.get("thinking_budget") - thinking_config = self._get_thinking_config( - reasoning_effort, model, explicit_thinking_budget - ) + thinking_config = self._get_thinking_config(reasoning_effort, model) if thinking_config: gen_config.setdefault("thinkingConfig", {}).update(thinking_config) From d9f2ddb7c9cfe4c2ec5140b7767532e4dcb67013 Mon Sep 17 00:00:00 2001 From: Mirrowel <28632877+Mirrowel@users.noreply.github.com> Date: Thu, 15 Jan 2026 19:17:42 +0100 Subject: [PATCH 34/36] =?UTF-8?q?feat(logging):=20=E2=9C=A8=20implement=20?= =?UTF-8?q?nested=20transaction=20logging=20for=20anthropic=20compatibilit?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change introduces a hierarchical logging structure to better trace requests passing through the translation layer. - Update `TransactionLogger` to support nested directories (`parent_dir`) and custom filenames, allowing internal OpenAI transactions to be logged as children of the original Anthropic request. - Implement full response reconstruction in `anthropic_streaming_wrapper` to accumulate and log the final state of streaming interactions (including thinking blocks and tool calls). - Modify `RotatingClient` to pass logging context down to the translation layer. - Switch `proxy_app` to use `RawIOLogger` when enabled for better debugging of the proxy boundary. --- src/proxy_app/main.py | 11 ++- .../anthropic_compat/streaming.py | 97 +++++++++++++++++-- src/rotator_library/client.py | 65 ++++++++++++- src/rotator_library/transaction_logger.py | 41 ++++++-- 4 files changed, 192 insertions(+), 22 deletions(-) diff --git a/src/proxy_app/main.py b/src/proxy_app/main.py index 406231d29..4d8dba99a 100644 --- a/src/proxy_app/main.py +++ b/src/proxy_app/main.py @@ -1025,8 +1025,15 @@ async def anthropic_messages( This endpoint is compatible with Claude Code and other Anthropic API clients. """ - # Initialize logger if enabled - logger = DetailedLogger() if ENABLE_REQUEST_LOGGING else None + # Initialize raw I/O logger if enabled (for debugging proxy boundary) + logger = RawIOLogger() if ENABLE_RAW_LOGGING else None + + # Log raw Anthropic request if raw logging is enabled + if logger: + logger.log_request( + headers=dict(request.headers), + body=body.model_dump(exclude_none=True), + ) try: # Log the request to console diff --git a/src/rotator_library/anthropic_compat/streaming.py b/src/rotator_library/anthropic_compat/streaming.py index e3ab84abe..1ece6a6c4 100644 --- a/src/rotator_library/anthropic_compat/streaming.py +++ b/src/rotator_library/anthropic_compat/streaming.py @@ -8,7 +8,10 @@ import json import logging import uuid -from typing import AsyncGenerator, Callable, Optional, Awaitable +from typing import AsyncGenerator, Callable, Optional, Awaitable, Any, TYPE_CHECKING + +if TYPE_CHECKING: + from ..transaction_logger import TransactionLogger logger = logging.getLogger("rotator_library.anthropic_compat") @@ -18,6 +21,7 @@ async def anthropic_streaming_wrapper( original_model: str, request_id: Optional[str] = None, is_disconnected: Optional[Callable[[], Awaitable[bool]]] = None, + transaction_logger: Optional["TransactionLogger"] = None, ) -> AsyncGenerator[str, None]: """ Convert OpenAI streaming format to Anthropic streaming format. @@ -39,6 +43,7 @@ async def anthropic_streaming_wrapper( original_model: The model name to include in responses request_id: Optional request ID (auto-generated if not provided) is_disconnected: Optional async callback that returns True if client disconnected + transaction_logger: Optional TransactionLogger for logging the final Anthropic response Yields: SSE format strings in Anthropic's streaming format @@ -55,6 +60,9 @@ async def anthropic_streaming_wrapper( input_tokens = 0 output_tokens = 0 cached_tokens = 0 # Track cached tokens for proper Anthropic format + accumulated_text = "" # Track accumulated text for logging + accumulated_thinking = "" # Track accumulated thinking for logging + stop_reason_final = "end_turn" # Track final stop reason for logging try: async for chunk_str in openai_stream: @@ -71,7 +79,10 @@ async def anthropic_streaming_wrapper( # Claude Code and other clients require message_start before message_stop if not message_started: # Build usage with cached tokens properly handled - usage_dict = {"input_tokens": input_tokens - cached_tokens, "output_tokens": 0} + usage_dict = { + "input_tokens": input_tokens - cached_tokens, + "output_tokens": 0, + } if cached_tokens > 0: usage_dict["cache_read_input_tokens"] = cached_tokens usage_dict["cache_creation_input_tokens"] = 0 @@ -111,6 +122,7 @@ async def anthropic_streaming_wrapper( # Determine stop_reason based on whether we had tool calls stop_reason = "tool_use" if tool_calls_by_index else "end_turn" + stop_reason_final = stop_reason # Build final usage dict with cached tokens final_usage = {"output_tokens": output_tokens} @@ -123,6 +135,66 @@ async def anthropic_streaming_wrapper( # Send message_stop yield 'event: message_stop\ndata: {"type": "message_stop"}\n\n' + + # Log final Anthropic response if logger provided + if transaction_logger: + # Build content blocks for logging + content_blocks = [] + if accumulated_thinking: + content_blocks.append( + { + "type": "thinking", + "thinking": accumulated_thinking, + } + ) + if accumulated_text: + content_blocks.append( + { + "type": "text", + "text": accumulated_text, + } + ) + # Add tool use blocks + for tc_index in sorted(tool_calls_by_index.keys()): + tc = tool_calls_by_index[tc_index] + # Parse arguments JSON string to dict + try: + input_data = json.loads(tc.get("arguments", "{}")) + except json.JSONDecodeError: + input_data = {} + content_blocks.append( + { + "type": "tool_use", + "id": tc.get("id", ""), + "name": tc.get("name", ""), + "input": input_data, + } + ) + + # Build usage for logging + log_usage = { + "input_tokens": input_tokens - cached_tokens, + "output_tokens": output_tokens, + } + if cached_tokens > 0: + log_usage["cache_read_input_tokens"] = cached_tokens + log_usage["cache_creation_input_tokens"] = 0 + + anthropic_response = { + "id": request_id, + "type": "message", + "role": "assistant", + "content": content_blocks, + "model": original_model, + "stop_reason": stop_reason_final, + "stop_sequence": None, + "usage": log_usage, + } + transaction_logger.log_response( + anthropic_response, + filename="anthropic_response.json", + ) + break try: @@ -139,12 +211,17 @@ async def anthropic_streaming_wrapper( output_tokens = usage.get("completion_tokens", output_tokens) # Extract cached tokens from prompt_tokens_details if usage.get("prompt_tokens_details"): - cached_tokens = usage["prompt_tokens_details"].get("cached_tokens", cached_tokens) + cached_tokens = usage["prompt_tokens_details"].get( + "cached_tokens", cached_tokens + ) # Send message_start on first chunk if not message_started: # Build usage with cached tokens properly handled for Anthropic format - usage_dict = {"input_tokens": input_tokens - cached_tokens, "output_tokens": 0} + usage_dict = { + "input_tokens": input_tokens - cached_tokens, + "output_tokens": 0, + } if cached_tokens > 0: usage_dict["cache_read_input_tokens"] = cached_tokens usage_dict["cache_creation_input_tokens"] = 0 @@ -165,7 +242,7 @@ async def anthropic_streaming_wrapper( yield f"event: message_start\ndata: {json.dumps(message_start)}\n\n" message_started = True - choices = chunk.get("choices", []) + choices = chunk.get("choices") or [] if not choices: continue @@ -191,6 +268,8 @@ async def anthropic_streaming_wrapper( "delta": {"type": "thinking_delta", "thinking": reasoning_content}, } yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + # Accumulate thinking for logging + accumulated_thinking += reasoning_content # Handle text content content = delta.get("content") @@ -218,8 +297,11 @@ async def anthropic_streaming_wrapper( "delta": {"type": "text_delta", "text": content}, } yield f"event: content_block_delta\ndata: {json.dumps(block_delta)}\n\n" + # Accumulate text for logging + accumulated_text += content # Handle tool calls + # Use `or []` to handle providers that send "tool_calls": null tool_calls = delta.get("tool_calls", []) for tc in tool_calls: tc_index = tc.get("index", 0) @@ -289,7 +371,10 @@ async def anthropic_streaming_wrapper( # Claude Code and other clients may ignore events that come before message_start if not message_started: # Build usage with cached tokens properly handled - usage_dict = {"input_tokens": input_tokens - cached_tokens, "output_tokens": 0} + usage_dict = { + "input_tokens": input_tokens - cached_tokens, + "output_tokens": 0, + } if cached_tokens > 0: usage_dict["cache_read_input_tokens"] = cached_tokens usage_dict["cache_creation_input_tokens"] = 0 diff --git a/src/rotator_library/client.py b/src/rotator_library/client.py index 1a518a641..a0ec4dfa6 100644 --- a/src/rotator_library/client.py +++ b/src/rotator_library/client.py @@ -1244,13 +1244,22 @@ async def _execute_with_retry( f"No API keys or OAuth credentials configured for provider: {provider}" ) + # Extract internal logging parameters (not passed to API) + parent_log_dir = kwargs.pop("_parent_log_dir", None) + # Establish a global deadline for the entire request lifecycle. deadline = time.time() + self.global_timeout # Create transaction logger if request logging is enabled transaction_logger = None if self.enable_request_logging: - transaction_logger = TransactionLogger(provider, model, enabled=True) + transaction_logger = TransactionLogger( + provider, + model, + enabled=True, + api_format="oai", + parent_dir=parent_log_dir, + ) transaction_logger.log_request(kwargs) # Create a mutable copy of the keys and shuffle it to ensure @@ -2000,6 +2009,9 @@ async def _streaming_acompletion_with_retry( model = kwargs.get("model") provider = model.split("/")[0] + # Extract internal logging parameters (not passed to API) + parent_log_dir = kwargs.pop("_parent_log_dir", None) + # Create a mutable copy of the keys and shuffle it. credentials_for_provider = list(self.all_credentials[provider]) random.shuffle(credentials_for_provider) @@ -2022,7 +2034,13 @@ async def _streaming_acompletion_with_retry( # Create transaction logger if request logging is enabled transaction_logger = None if self.enable_request_logging: - transaction_logger = TransactionLogger(provider, model, enabled=True) + transaction_logger = TransactionLogger( + provider, + model, + enabled=True, + api_format="oai", + parent_dir=parent_log_dir, + ) transaction_logger.log_request(kwargs) tried_creds = set() @@ -2921,7 +2939,9 @@ def token_count(self, **kwargs) -> int: provider = model.split("/")[0] if "/" in model else "" if provider == "antigravity": try: - from .providers.antigravity_provider import get_antigravity_preprompt_text + from .providers.antigravity_provider import ( + get_antigravity_preprompt_text, + ) preprompt_text = get_antigravity_preprompt_text() if preprompt_text: @@ -3477,9 +3497,31 @@ async def anthropic_messages( request_id = f"msg_{uuid.uuid4().hex[:24]}" original_model = request.model + # Extract provider from model for logging + provider = original_model.split("/")[0] if "/" in original_model else "unknown" + + # Create Anthropic transaction logger if request logging is enabled + anthropic_logger = None + if self.enable_request_logging: + anthropic_logger = TransactionLogger( + provider, + original_model, + enabled=True, + api_format="ant", + ) + # Log original Anthropic request + anthropic_logger.log_request( + request.model_dump(exclude_none=True), + filename="anthropic_request.json", + ) + # Translate Anthropic request to OpenAI format openai_request = translate_anthropic_request(request) + # Pass parent log directory to acompletion for nested logging + if anthropic_logger and anthropic_logger.log_dir: + openai_request["_parent_log_dir"] = anthropic_logger.log_dir + if request.stream: # Streaming response response_generator = self.acompletion( @@ -3494,11 +3536,13 @@ async def anthropic_messages( is_disconnected = raw_request.is_disconnected # Return the streaming wrapper + # Note: For streaming, the anthropic response logging happens in the wrapper return anthropic_streaming_wrapper( openai_stream=response_generator, original_model=original_model, request_id=request_id, is_disconnected=is_disconnected, + transaction_logger=anthropic_logger, ) else: # Non-streaming response @@ -3510,13 +3554,24 @@ async def anthropic_messages( # Convert OpenAI response to Anthropic format openai_response = ( - response.model_dump() if hasattr(response, "model_dump") else dict(response) + response.model_dump() + if hasattr(response, "model_dump") + else dict(response) + ) + anthropic_response = openai_to_anthropic_response( + openai_response, original_model ) - anthropic_response = openai_to_anthropic_response(openai_response, original_model) # Override the ID with our request ID anthropic_response["id"] = request_id + # Log Anthropic response + if anthropic_logger: + anthropic_logger.log_response( + anthropic_response, + filename="anthropic_response.json", + ) + return anthropic_response async def anthropic_count_tokens( diff --git a/src/rotator_library/transaction_logger.py b/src/rotator_library/transaction_logger.py index 58b3155f5..6b95234a2 100644 --- a/src/rotator_library/transaction_logger.py +++ b/src/rotator_library/transaction_logger.py @@ -98,11 +98,19 @@ class TransactionLogger: "provider", "model", "streaming", + "api_format", "_dir_available", "_context", ) - def __init__(self, provider: str, model: str, enabled: bool = True): + def __init__( + self, + provider: str, + model: str, + enabled: bool = True, + api_format: str = "oai", + parent_dir: Optional[Path] = None, + ): """ Initialize transaction logger. @@ -110,11 +118,14 @@ def __init__(self, provider: str, model: str, enabled: bool = True): provider: Provider name (e.g., 'antigravity', 'openai') model: Model name (will be sanitized for filesystem) enabled: Whether logging is enabled + api_format: API format prefix ('oai' for OpenAI, 'ant' for Anthropic) + parent_dir: Optional parent directory for nested logging """ self.enabled = enabled self.start_time = time.time() self.request_id = str(uuid.uuid4())[:8] # 8-char short ID self.provider = provider + self.api_format = api_format # Strip provider prefix from model if present # e.g., "antigravity/claude-opus-4.5" → "claude-opus-4.5" @@ -131,12 +142,19 @@ def __init__(self, provider: str, model: str, enabled: bool = True): if not enabled: return - # Create directory: MMDD_HHMMSS_{provider}_{model}_{request_id} + # Create directory based on whether we have a parent directory timestamp = datetime.now().strftime("%m%d_%H%M%S") safe_provider = _sanitize_name(provider) - dir_name = f"{timestamp}_{safe_provider}_{self.model}_{self.request_id}" - self.log_dir = _get_transactions_dir() / dir_name + if parent_dir: + # Nested logging: create subdirectory inside parent + # e.g., parent_dir/openai/ for OpenAI translation layer + subdir_name = "openai" if api_format == "oai" else api_format + self.log_dir = parent_dir / subdir_name + else: + # Root-level logging: MMDD_HHMMSS_{api_format}_{provider}_{model}_{request_id} + dir_name = f"{timestamp}_{api_format}_{safe_provider}_{self.model}_{self.request_id}" + self.log_dir = _get_transactions_dir() / dir_name try: self.log_dir.mkdir(parents=True, exist_ok=True) @@ -162,12 +180,15 @@ def get_context(self) -> TransactionContext: ) return self._context - def log_request(self, request_data: Dict[str, Any]) -> None: + def log_request( + self, request_data: Dict[str, Any], filename: str = "request.json" + ) -> None: """ - Log the OpenAI-compatible request received by client.py. + Log the request received by client.py. Args: request_data: The request data dict (messages, model, etc.) + filename: Custom filename for the log file (default: request.json) """ if not self.enabled or not self._dir_available: return @@ -179,7 +200,7 @@ def log_request(self, request_data: Dict[str, Any]) -> None: "timestamp_utc": datetime.utcnow().isoformat(), "data": request_data, } - self._write_json("request.json", data) + self._write_json(filename, data) def log_stream_chunk(self, chunk: Dict[str, Any]) -> None: """ @@ -203,14 +224,16 @@ def log_response( response_data: Dict[str, Any], status_code: int = 200, headers: Optional[Dict[str, Any]] = None, + filename: str = "response.json", ) -> None: """ - Log the OpenAI-compatible response returned by client.py. + Log the response returned by client.py. Args: response_data: The response data dict status_code: HTTP status code (default 200) headers: Optional response headers + filename: Custom filename for the log file (default: response.json) """ if not self.enabled or not self._dir_available: return @@ -226,7 +249,7 @@ def log_response( "headers": dict(headers) if headers else None, "data": response_data, } - self._write_json("response.json", data) + self._write_json(filename, data) # Also write metadata self._log_metadata(response_data, status_code, duration_ms) From 6d9f9cc60f4404e7e39b2ab13412fea24825c1d5 Mon Sep 17 00:00:00 2001 From: Mirrowel <28632877+Mirrowel@users.noreply.github.com> Date: Thu, 15 Jan 2026 19:18:10 +0100 Subject: [PATCH 35/36] =?UTF-8?q?fix(anthropic-compat):=20=F0=9F=90=9B=20h?= =?UTF-8?q?andle=20null=20tool=5Fcalls=20in=20streaming=20delta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation using `delta.get("tool_calls", [])` would return `None` if the provider explicitly sent `"tool_calls": null`, bypassing the default value. This change ensures `tool_calls` always resolves to a list using the `or []` pattern, preventing potential errors during iteration. --- src/rotator_library/anthropic_compat/streaming.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rotator_library/anthropic_compat/streaming.py b/src/rotator_library/anthropic_compat/streaming.py index 1ece6a6c4..870dae170 100644 --- a/src/rotator_library/anthropic_compat/streaming.py +++ b/src/rotator_library/anthropic_compat/streaming.py @@ -302,7 +302,7 @@ async def anthropic_streaming_wrapper( # Handle tool calls # Use `or []` to handle providers that send "tool_calls": null - tool_calls = delta.get("tool_calls", []) + tool_calls = delta.get("tool_calls") or [] for tc in tool_calls: tc_index = tc.get("index", 0) From 1798e75e8c3e9734ddffd65c238601fb455a8e3c Mon Sep 17 00:00:00 2001 From: Mirrowel <28632877+Mirrowel@users.noreply.github.com> Date: Thu, 15 Jan 2026 21:21:55 +0100 Subject: [PATCH 36/36] =?UTF-8?q?docs:=20=F0=9F=93=9A=20document=20anthrop?= =?UTF-8?q?ic=20api=20compatibility=20layer=20and=20client=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates project documentation to reflect the new Anthropic API compatibility features: - **README.md**: Add setup guides for Claude Code and Anthropic Python SDK, plus API endpoint details. - **DOCUMENTATION.md**: Add deep dive into the `anthropic_compat` architecture, including translation logic and streaming behavior. - **Library Docs**: Document `anthropic_messages` and `anthropic_count_tokens` methods in `rotator_library`. --- DOCUMENTATION.md | 103 ++++++++++++++++++++++++++++++++++ README.md | 54 ++++++++++++++++-- src/rotator_library/README.md | 56 ++++++++++++++++++ 3 files changed, 209 insertions(+), 4 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 5dafe6b87..f7ebbde50 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -10,6 +10,7 @@ The project is a monorepo containing two primary components: * **Batch Manager**: Optimizes high-volume embedding requests. * **Detailed Logger**: Provides per-request file logging for debugging. * **OpenAI-Compatible Endpoints**: `/v1/chat/completions`, `/v1/embeddings`, etc. + * **Anthropic-Compatible Endpoints**: `/v1/messages`, `/v1/messages/count_tokens` for Claude Code and other Anthropic API clients. * **Model Filter GUI**: Visual interface for configuring model ignore/whitelist rules per provider (see Section 6). 2. **The Resilience Library (`rotator_library`)**: This is the core engine that provides high availability. It is consumed by the proxy app to manage a pool of API keys, handle errors gracefully, and ensure requests are completed successfully even when individual keys or provider endpoints face issues. @@ -816,6 +817,108 @@ When a custom cap triggers a cooldown longer than the exhaustion threshold, it a **Defaults:** See `src/rotator_library/config/defaults.py` for all configurable defaults. +### 2.21. Anthropic API Compatibility (`anthropic_compat/`) + +A translation layer that enables Anthropic API clients (like Claude Code) to use any OpenAI-compatible provider through the proxy. + +#### Architecture + +The module consists of three components: + +| File | Purpose | +|------|---------| +| `models.py` | Pydantic models for Anthropic request/response formats (`AnthropicMessagesRequest`, `AnthropicMessage`, `AnthropicTool`, etc.) | +| `translator.py` | Bidirectional format translation functions | +| `streaming.py` | SSE format conversion for streaming responses | + +#### Request Translation (`translate_anthropic_request`) + +Converts Anthropic Messages API requests to OpenAI Chat Completions format: + +**Message Conversion:** +- Anthropic `system` field → OpenAI system message +- `content` blocks (text, image, tool_use, tool_result) → OpenAI format +- Image blocks with base64 data → OpenAI `image_url` with data URI +- Document blocks (PDF, etc.) → OpenAI `image_url` format + +**Tool Conversion:** +- Anthropic `tools` with `input_schema` → OpenAI `tools` with `parameters` +- `tool_choice.type: "any"` → `"required"` +- `tool_choice.type: "tool"` → `{"type": "function", "function": {"name": ...}}` + +**Thinking Configuration:** +- `thinking.type: "enabled"` → `reasoning_effort: "high"` + `thinking_budget` +- `thinking.type: "disabled"` → `reasoning_effort: "disable"` +- Opus models default to thinking enabled + +**Special Handling:** +- Reorders assistant content blocks: thinking → text → tool_use +- Injects `[Continue]` prompt for fresh thinking turns +- Preserves thinking signatures for multi-turn conversations + +#### Response Translation (`openai_to_anthropic_response`) + +Converts OpenAI Chat Completions responses to Anthropic Messages format: + +**Content Blocks:** +- `reasoning_content` → thinking block with signature +- `content` → text block +- `tool_calls` → tool_use blocks with parsed JSON input + +**Field Mapping:** +- `finish_reason: "stop"` → `stop_reason: "end_turn"` +- `finish_reason: "length"` → `stop_reason: "max_tokens"` +- `finish_reason: "tool_calls"` → `stop_reason: "tool_use"` + +**Usage Translation:** +- `prompt_tokens` minus `cached_tokens` → `input_tokens` +- `completion_tokens` → `output_tokens` +- `prompt_tokens_details.cached_tokens` → `cache_read_input_tokens` + +#### Streaming Wrapper (`anthropic_streaming_wrapper`) + +Converts OpenAI SSE streaming format to Anthropic's event-based format: + +**Event Types Generated:** +``` +message_start → Initial message metadata +content_block_start → Start of text/thinking/tool_use block +content_block_delta → Incremental content (text_delta, thinking_delta, input_json_delta) +content_block_stop → End of content block +message_delta → Final metadata (stop_reason, usage) +message_stop → End of message +``` + +**Features:** +- Accumulates tool call arguments across chunks +- Handles thinking/reasoning content from `delta.reasoning_content` +- Proper block indexing for multiple content blocks +- Cache token handling in usage statistics +- Error recovery with proper message structure + +#### Client Integration + +The `RotatingClient` provides two methods for Anthropic compatibility: + +```python +async def anthropic_messages(self, request, raw_request=None, pre_request_callback=None): + """Handle Anthropic Messages API requests.""" + # 1. Translate Anthropic request to OpenAI format + # 2. Call acompletion() with translated request + # 3. Convert response back to Anthropic format + # 4. For streaming: wrap with anthropic_streaming_wrapper + +async def anthropic_count_tokens(self, request): + """Count tokens for Anthropic-format request.""" + # Translates messages and tools, then uses token_count() +``` + +#### Authentication + +The proxy accepts both Anthropic and OpenAI authentication styles: +- `x-api-key` header (Anthropic style) +- `Authorization: Bearer` header (OpenAI style) + ### 3.5. Antigravity (`antigravity_provider.py`) The most sophisticated provider implementation, supporting Google's internal Antigravity API for Gemini 3 and Claude models (including **Claude Opus 4.5**, Anthropic's most powerful model). diff --git a/README.md b/README.md index cd650e5ec..a7c3c4383 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,20 @@ **One proxy. Any LLM provider. Zero code changes.** -A self-hosted proxy that provides a single, OpenAI-compatible API endpoint for all your LLM providers. Works with any application that supports custom OpenAI base URLs—no code changes required in your existing tools. +A self-hosted proxy that provides OpenAI and Anthropic compatible API endpoints for all your LLM providers. Works with any application that supports custom OpenAI or Anthropic base URLs—including Claude Code, Opencode, and more—no code changes required in your existing tools. This project consists of two components: -1. **The API Proxy** — A FastAPI application providing a universal `/v1/chat/completions` endpoint +1. **The API Proxy** — A FastAPI application providing universal `/v1/chat/completions` (OpenAI) and `/v1/messages` (Anthropic) endpoints 2. **The Resilience Library** — A reusable Python library for intelligent API key management, rotation, and failover --- ## Why Use This? -- **Universal Compatibility** — Works with any app supporting OpenAI-compatible APIs: Opencode, Continue, Roo/Kilo Code, JanitorAI, SillyTavern, custom applications, and more +- **Universal Compatibility** — Works with any app supporting OpenAI or Anthropic APIs: Claude Code, Opencode, Continue, Roo/Kilo Code, Cursor, JanitorAI, SillyTavern, custom applications, and more - **One Endpoint, Many Providers** — Configure Gemini, OpenAI, Anthropic, and [any LiteLLM-supported provider](https://docs.litellm.ai/docs/providers) once. Access them all through a single API key +- **Anthropic API Compatible** — Use Claude Code or any Anthropic SDK client with non-Anthropic providers like Gemini, OpenAI, or custom models - **Built-in Resilience** — Automatic key rotation, failover on errors, rate limit handling, and intelligent cooldowns - **Exclusive Provider Support** — Includes custom providers not available elsewhere: **Antigravity** (Gemini 3 + Claude Sonnet/Opus 4.5), **Gemini CLI**, **Qwen Code**, and **iFlow** @@ -177,12 +178,57 @@ In your configuration file (e.g., `config.json`): +
+Claude Code + +Claude Code natively supports custom Anthropic API endpoints. The recommended setup is to edit your Claude Code `settings.json`: + +```json +{ + "env": { + "ANTHROPIC_AUTH_TOKEN": "your-proxy-api-key", + "ANTHROPIC_BASE_URL": "http://127.0.0.1:8000", + "ANTHROPIC_DEFAULT_OPUS_MODEL": "gemini/gemini-3-pro", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "gemini/gemini-3-flash", + "ANTHROPIC_DEFAULT_HAIKU_MODEL": "openai/gpt-5-mini" + } +} +``` + +Now you can use Claude Code with Gemini, OpenAI, or any other configured provider. + +
+ +
+Anthropic Python SDK + +```python +from anthropic import Anthropic + +client = Anthropic( + base_url="http://127.0.0.1:8000", + api_key="your-proxy-api-key" +) + +# Use any provider through Anthropic's API format +response = client.messages.create( + model="gemini/gemini-3-flash", # provider/model format + max_tokens=1024, + messages=[{"role": "user", "content": "Hello!"}] +) +print(response.content[0].text) +``` + +
+ ### API Endpoints | Endpoint | Description | |----------|-------------| | `GET /` | Status check — confirms proxy is running | -| `POST /v1/chat/completions` | Chat completions (main endpoint) | +| `POST /v1/chat/completions` | Chat completions (OpenAI format) | +| `POST /v1/messages` | Chat completions (Anthropic format) — Claude Code compatible | +| `POST /v1/messages/count_tokens` | Count tokens for Anthropic-format requests | | `POST /v1/embeddings` | Text embeddings | | `GET /v1/models` | List all available models with pricing & capabilities | | `GET /v1/models/{model_id}` | Get details for a specific model | diff --git a/src/rotator_library/README.md b/src/rotator_library/README.md index c7b3c8668..22d2bf6e9 100644 --- a/src/rotator_library/README.md +++ b/src/rotator_library/README.md @@ -5,6 +5,7 @@ A robust, asynchronous, and thread-safe Python library for managing a pool of AP ## Key Features - **Asynchronous by Design**: Built with `asyncio` and `httpx` for high-performance, non-blocking I/O. +- **Anthropic API Compatibility**: Built-in translation layer (`anthropic_compat`) enables Anthropic API clients (like Claude Code) to use any supported provider. - **Advanced Concurrency Control**: A single API key can be used for multiple concurrent requests. By default, it supports concurrent requests to *different* models. With configuration (`MAX_CONCURRENT_REQUESTS_PER_KEY_`), it can also support multiple concurrent requests to the *same* model using the same key. - **Smart Key Management**: Selects the optimal key for each request using a tiered, model-aware locking strategy to distribute load evenly and maximize availability. - **Configurable Rotation Strategy**: Choose between deterministic least-used selection (perfect balance) or default weighted random selection (unpredictable, harder to fingerprint). @@ -173,6 +174,61 @@ Fetches a list of available models for a specific provider, applying any configu Fetches a dictionary of all available models, grouped by provider, or as a single flat list if `grouped=False`. +#### `async def anthropic_messages(self, request, raw_request=None, pre_request_callback=None) -> Any:` + +Handle Anthropic Messages API requests. Accepts requests in Anthropic's format, translates them to OpenAI format internally, processes them through `acompletion`, and returns responses in Anthropic's format. + +- **Parameters**: + - `request`: An `AnthropicMessagesRequest` object (from `anthropic_compat.models`) + - `raw_request`: Optional raw request object for client disconnect checks + - `pre_request_callback`: Optional async callback before each API request +- **Returns**: + - For non-streaming: dict in Anthropic Messages format + - For streaming: AsyncGenerator yielding Anthropic SSE format strings + +#### `async def anthropic_count_tokens(self, request) -> dict:` + +Handle Anthropic count_tokens API requests. Counts the number of tokens that would be used by a Messages API request. + +- **Parameters**: `request` - An `AnthropicCountTokensRequest` object +- **Returns**: Dict with `input_tokens` count in Anthropic format + +## Anthropic API Compatibility + +The library includes a translation layer (`anthropic_compat`) that enables Anthropic API clients to use any OpenAI-compatible provider. + +### Usage + +```python +from rotator_library.anthropic_compat import ( + AnthropicMessagesRequest, + AnthropicCountTokensRequest, + translate_anthropic_request, + openai_to_anthropic_response, + anthropic_streaming_wrapper, +) + +# Create an Anthropic-format request +request = AnthropicMessagesRequest( + model="gemini/gemini-2.5-flash", + max_tokens=1024, + messages=[{"role": "user", "content": "Hello!"}] +) + +# Use with RotatingClient +async with RotatingClient(api_keys=api_keys) as client: + response = await client.anthropic_messages(request) + print(response["content"][0]["text"]) +``` + +### Features + +- **Full Message Translation**: Converts between Anthropic and OpenAI message formats including text, images, tool_use, and tool_result blocks +- **Extended Thinking Support**: Translates Anthropic's `thinking` configuration to `reasoning_effort` for providers that support it +- **Streaming SSE Conversion**: Converts OpenAI streaming chunks to Anthropic's SSE event format (`message_start`, `content_block_delta`, etc.) +- **Cache Token Handling**: Properly translates `prompt_tokens_details.cached_tokens` to Anthropic's `cache_read_input_tokens` +- **Tool Call Support**: Full support for tool definitions and tool use/result blocks + ## Credential Tool The library includes a utility to manage credentials easily: