Release 2026.4.1 (#643)

AGENTS.mk

@@ -6,113 +6,80 @@
 
 MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 MAKEFILE_DIR := $(realpath $(dir $(MAKEFILE_PATH)))
-EXTRACT_INFO_CMAKE := ${MAKEFILE_DIR}/extract_info.cmake
-IDEAS_MAKEFILE := $(MAKEFILE_DIR)/IDEAS.mk
-
-ANTHROPIC_AUTH_TOKEN ?= $(OPENROUTER_API_KEY)
-ANTHROPIC_BASE_URL ?= https://openrouter.ai/api
-ANTHROPIC_API_KEY ?= ""
 
 AGENT_PROVIDER ?= openrouter
 AGENT_MODEL ?= anthropic/claude-sonnet-4.6
-BASE_URL ?= "https://openrouter.ai/api/v1"
-TRANSLATION_DIR ?= translation.$(shell git --git-dir=${MAKEFILE_DIR}/.git rev-parse HEAD)
-
-export EXTRACT_INFO_CMAKE
-
-TARGETS ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -type f -executable -exec basename {} \; | cut -d. -f1 | sed -e "s/^lib//gi")
-ifeq (${TARGETS},)
-ifeq ($(filter cmake clean,$(MAKECMDGOALS)),)
-$(error No TARGETS found! You need to run cmake!)
-endif
-endif
+AGENT_BASE_URL ?= "https://openrouter.ai/api/v1"
 
 # Docker configuration
-DOCKER_DIR := ${MAKEFILE_DIR}/docker
+DOCKER_HOSTDIR := ${MAKEFILE_DIR}/docker
 DOCKER_WORKDIR := /home/user/IDEAS
-# Relative path to the current working directory
-DOCKER_REL_CWD := $(patsubst $(MAKEFILE_DIR)/%,%,$(CURDIR))
 DOCKER_RUN ?= docker run --rm \
     --init \
     -it \
     -v $(MAKEFILE_DIR):$(DOCKER_WORKDIR) \
-    -v $(DOCKER_DIR)/.venv:$(DOCKER_WORKDIR)/.venv \
+    -w $(DOCKER_WORKDIR)/$(patsubst $(MAKEFILE_DIR)/%,%,$(CURDIR)) \
     -e OPENROUTER_API_KEY \
     -e TRANSLATION_DIR \
     -e AGENT_PROVIDER \
     -e AGENT_MODEL \
-    -e BASE_URL \
+    -e AGENT_BASE_URL \
     -e RUSTFLAGS \
-    -e VERBOSE \
     ideas-$(shell id -u)
 
 ifdef DOCKER_RUN
-    # Touch directory for correct permissions when mounted
-	VENV_SETUP = mkdir -p $(DOCKER_DIR)/.venv
-    # Run inside Docker container with exit-on-error
-	RUN_CMD = $(DOCKER_RUN) /bin/sh -c 'set -e; cd $(DOCKER_WORKDIR)/$(DOCKER_REL_CWD); $(1)'
+	RUN_PREFIX = \
+        mkdir -p $(DOCKER_HOSTDIR)/.venv && \
+        $(DOCKER_RUN) \
+        /bin/sh -c 'set -e;
+	RUN_SUFFIX = '
 else
-	VENV_SETUP = @true
-	RUN_CMD = $(1)
+	RUN_PREFIX =
+	RUN_SUFFIX =
 endif
 
-# cmake
-.PHONY: cmake
-cmake: build-ninja/cmake.log
-
-build-ninja/cmake.log: test_case/CMakeLists.txt ${EXTRACT_INFO_CMAKE}
-	uv run python -m ideas.cmake source_dir=test_case build_dir=build-ninja
-	@touch $@
-
-build-ninja/CMakeCache.txt: build-ninja/cmake.log
-build-ninja/compile_commands.json: build-ninja/cmake.log
-build-ninja/build.log: build-ninja/cmake.log
 
 # test generation from project
 .PHONY: testgen
 testgen: test_crate/tests/test_assert.rs ;
 
 .PRECIOUS: test_crate/tests/test_assert.rs
 test_crate/tests/test_assert.rs:
-	$(VENV_SETUP)
-	$(call RUN_CMD,\
+	$(RUN_PREFIX) \
         uv run python -m ideas.agents.testgen model=$(if $(AGENT_PROVIDER),${AGENT_PROVIDER}/,)${AGENT_MODEL} \
             c_code=test_case \
             project_name=$(notdir $(CURDIR)) \
             test_vectors_out=test_vectors/agent \
             test_crate_out=test_crate \
             hydra.output_subdir=.testgen \
             hydra.job.name=testgen \
-            hydra.run.dir=test_vectors; \
-    )
+            hydra.run.dir=test_vectors \
+    $(RUN_SUFFIX)
+    # Agent is not guaranteed to write file
+	[ -f test_crate/tests/test_assert.rs ] || { echo "ERROR: Agent failed to generate test_crate/tests/test_assert.rs"; exit 1; }
 
 
 # library targets: generate tests from the consolidated lib.c
 .PRECIOUS: test_crates/%/tests/test_assert.rs
 test_crates/%/tests/test_assert.rs: ${TRANSLATION_DIR}/%/src/lib.c | build-ninja/lib%.so.type
-    # Copy lib.c into test_targets/<target>/src/ so
-    # build.rs can use ../../test_targets/<target>/src/lib.c
+    # Copy lib.c into test_crates/<target>/src/ so
+    # build.rs can use ../../test_crates/<target>/src/lib.c
     # both in Docker /tmp and on disk
-	mkdir -p test_targets/$*/src
-	cp ${TRANSLATION_DIR}/$*/src/lib.c test_targets/$*/src/lib.c
-	$(VENV_SETUP)
-	$(call RUN_CMD,\
+	mkdir -p test_crates/$*/src
+	cp ${TRANSLATION_DIR}/$*/src/lib.c test_crates/$*/src/lib.c
+	$(RUN_PREFIX) \
         uv run python -m ideas.agents.testgen model=$(if $(AGENT_PROVIDER),${AGENT_PROVIDER}/,)${AGENT_MODEL} \
-            c_code=test_targets/$*/src/lib.c \
+            c_code=test_crates/$*/src/lib.c \
             project_name=$* \
             test_vectors_out=test_vectors/$*/agent \
             test_crate_out=test_crates/$* \
             hydra.output_subdir=.testgen \
             hydra.job.name=testgen \
-            hydra.run.dir=test_vectors/$*; \
-    )
+            hydra.run.dir=test_vectors/$* \
+    $(RUN_SUFFIX)
+    # Agent is not guaranteed to write file
+	[ -f test_crates/$*/tests/test_assert.rs ] || { echo "ERROR: Agent failed to generate test_crates/$*/tests/test_assert.rs"; exit 1; }
 
 # executable targets: do nothing
 test_crates/%/tests/test_assert.rs: ${TRANSLATION_DIR}/%/src/main.c | build-ninja/%.type
-	mkdir -p test_crates/$*/tests
-	touch test_crates/$*/tests/test_assert.rs
-
-# fallback
-test_crates/%/tests/test_assert.rs:
-	mkdir -p test_crates/$*/tests
-	touch test_crates/$*/tests/test_assert.rs
+	$(error Agent cannot generate tests for binary targets yet!)

IDEAS.mk

@@ -55,8 +55,10 @@ build-ninja/build.log: build-ninja/cmake.log
 # init
 .PHONY: init
 init: $(patsubst %,${TRANSLATION_DIR}/%/init,${TARGETS}) ;
-${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/src/lib.c | build-ninja/lib%.so.type ;
-${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/src/main.c | build-ninja/%.type ;
+${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/src/lib.c | build-ninja/lib%.so.type
+	touch ${TRANSLATION_DIR}/$*/src/lib.c
+${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/src/main.c | build-ninja/%.type
+	touch ${TRANSLATION_DIR}/$*/src/main.c
 
 # initialize workspace
 .PRECIOUS: ${TRANSLATION_DIR}/Cargo.toml
@@ -103,7 +105,7 @@ ${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/lib.rs | build-ninja/li
 ${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/main.rs | build-ninja/%.type ;
 
 .PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.rs
-${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c ${TRANSLATION_DIR}/%/tests/test_cases.rs | ${TRANSLATION_DIR}/%/Cargo.toml
+${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c | ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/tests/test_assert.rs
 	-uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \
                                  filename=${TRANSLATION_DIR}/$*/src/lib.c \
                                  cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \
@@ -210,35 +212,24 @@ test_vectors/%/%.json:
 	$(error $@ not found)
 
 
-# testgen
-.PHONY: testgen_argless
-testgen_argless: $(patsubst %,test_vectors/%/testgen_argless,${TARGETS})
-test_vectors/%/testgen_argless: | build-ninja/lib%.so.type ;
-test_vectors/%/testgen_argless: test_vectors/%/argless.json | build-ninja/%.type ;
-
-.PRECIOUS: test_vectors/%/argless.json
-test_vectors/%/argless.json: | build-ninja/%.type
-	-uv run python -m ideas.testgen artifact=build-ninja/$* \
-                                  test_vector=$@ \
-                                  hydra.output_subdir=.testgen \
-                                  hydra.job.name=testgen \
-                                  hydra.run.dir=test_vectors/$*
-
 # testgen for each C target
 .PHONY: testgen_target
-testgen_target: $(patsubst %,${TRANSLATION_DIR}/%/tests/test_assert.rs,${TARGETS}) ;
+testgen_target: $(patsubst %,test_crates/%/tests/test_assert.rs,${TARGETS}) ;
 
-${TRANSLATION_DIR}/%/tests/test_assert.rs: test_crates/%/tests/test_assert.rs build-ninja/lib%.so.type
-	mkdir -p $(dir $@)
-	cp test_crates/$*/tests/test_assert.rs $@
+.PRECIOUS: test_crates/%/tests/test_assert.rs
+test_crates/%/tests/test_assert.rs: | ${TRANSLATION_DIR}/%/src/lib.c build-ninja/lib%.so.type
+	-@$(MAKE) -j1 -f $(AGENTS_MAKEFILE) test_crates/$*/tests/test_assert.rs
 
 .PRECIOUS: test_crates/%/tests/test_assert.rs
-test_crates/%/tests/test_assert.rs:
+test_crates/%/tests/test_assert.rs: | ${TRANSLATION_DIR}/%/src/main.c | build-ninja/%.type
 	-@$(MAKE) -j1 -f $(AGENTS_MAKEFILE) test_crates/$*/tests/test_assert.rs
 
-${TRANSLATION_DIR}/%/tests/test_assert.rs: build-ninja/%.type
+${TRANSLATION_DIR}/%/tests/test_assert.rs: test_crates/%/tests/test_assert.rs | build-ninja/lib%.so.type
 	mkdir -p $(dir $@)
-	touch $@
+	cp test_crates/$*/tests/test_assert.rs $@
+
+${TRANSLATION_DIR}/%/tests/test_assert.rs: | build-ninja/%.type
+	$(error Agent cannot generate tests for binary targets yet!)
 
 
 # clean

Makefile

@@ -99,6 +99,12 @@ kill:## Kill all vLLM servers
 .PHONY: FORCE
 FORCE:
 
+.PHONY: examples
+examples:## Print out examples
+examples: $(addsuffix /print,${EXAMPLES}) ;
+examples/%/print: FORCE
+	@if [ -d "$(@D)" ]; then echo "$(@D)"; fi
+
 .PHONY: examples/init
 examples/init:## Initialize all examples
 examples/init: $(addsuffix /init,${EXAMPLES}) ;
@@ -122,15 +128,6 @@ examples/%/cmake: FORCE
 	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake
 
 
-.PHONE: examples/testgen_argless
-examples/testgen_argless:## Generate argless tests for executable targets in all C examples
-examples/testgen_argless: $(addsuffix /testgen_argless,${EXAMPLES})
-examples/%/testgen_argless:## Generate argless tests for executable targets in a specific C example
-examples/%/testgen_argless: FORCE
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) testgen_argless
-
-
 .PHONY: examples/testgen_agent
 examples/testgen_agent:## Generate test vectors for all C examples with an agent
 examples/testgen_agent: $(addsuffix /testgen_agent,${EXAMPLES})
@@ -145,22 +142,9 @@ examples/testgen_agent_target: $(addsuffix /testgen_agent_target,${EXAMPLES})
 examples/%/testgen_agent_target:## Generate test vectors for all targets in a specific C example with an agent
 examples/%/testgen_agent_target: FORCE
 	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) init
 	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) testgen_target
 
 
-.PHONY: examples/testgen_and_translate
-examples/testgen_and_translate:## Generate tests and translate all examples
-examples/testgen_and_translate: $(addsuffix /testgen_and_translate,${EXAMPLES})
-examples/%/testgen_and_translate:## Generate tests and translate specific example
-examples/%/testgen_and_translate: FORCE
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) testgen_argless
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) init
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) testgen_target
-	-@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) translate
-
-
 .PHONY: examples/translate
 examples/translate:## Translate all examples
 examples/translate: $(addsuffix /translate,${EXAMPLES})

src/ideas/agents/testgen.py

@@ -125,8 +125,9 @@ class TestgenInstructions:
         Write a `{rs_crate_path}/build.rs` that:
         1. Uses `cc::Build::new()` with `.compiler("clang")` to compile **all** C source files discovered in Step 1.
         2. Adds the correct include directories so the C headers are found.
-        3. Links any extra system libraries the C project requires (e.g. `println!("cargo::rustc-link-lib=m");`).
-        4. Passes `-w` (suppress warnings) and `-std=c99`.
+        3. Uses `.warnings(false)` to suppress warnings.
+        4. Uses `.std("c99")` to specify the C standard.
+        5. Links any extra system libraries the C project requires (e.g. `println!("cargo::rustc-link-lib=m");`).
         """
     )
 
@@ -351,7 +352,7 @@ class TestgenInstructions:
            JSON file.
            - For floating-point fields use an epsilon comparison:
              ```rust
-             assert!((actual - expected).abs() < 1e-4,
+             assert!((actual - expected).abs() / expected.abs() < 1e-3,
                      "field `<name>`: expected {{expected}}, got {{actual}}");
              ```
            - For integer / bool fields use `assert_eq!`.
@@ -442,9 +443,7 @@ def _main(cfg: TestgenConfig) -> None:
     logger_trajectory.addHandler(fh)
     # Simultaneous print and log to file
     printer = LoggingConsolePrinter(logger=logger_trajectory)
-
-    name = "C library test vector generator"
-    agent = RelentlessAgent(name=name)
+    agent = RelentlessAgent(name="C library test vector generator")
 
     project_name = cfg.project_name
     work_dir = Path(tempfile.mkdtemp()) / project_name