feat(eval): compare a run against a saved baseline and fail on regression

cmd/root/eval.go

@@ -1,6 +1,7 @@
 package root
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -25,6 +26,13 @@ type evalFlags struct {
 
 	runConfig config.RuntimeConfig
 	outputDir string
+
+	// baseline is a previously saved run (an -eval.json written by a prior
+	// invocation) to compare this run against; empty disables the check.
+	baseline string
+	// regressionTolerance is how far an aggregate quality rate may fall before
+	// the comparison fails. See evaluation.Compare for the exact semantics.
+	regressionTolerance float64
 }
 
 func newEvalCmd() *cobra.Command {
@@ -48,11 +56,18 @@ func newEvalCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&flags.KeepContainers, "keep-containers", false, "Keep containers after evaluation (don't use --rm)")
 	cmd.Flags().StringSliceVarP(&flags.EnvVars, "env", "e", nil, "Environment variables to pass to container (KEY or KEY=VALUE)")
 	cmd.Flags().IntVar(&flags.Repeat, "repeat", 1, "Number of times to repeat each evaluation (useful for computing baselines)")
+	cmd.Flags().StringVar(&flags.baseline, "baseline", "", "Compare against a previously saved run JSON (<output>/<run>.json) and exit non-zero on regression")
+	cmd.Flags().Float64Var(&flags.regressionTolerance, "regression-tolerance", 0, "How far an aggregate quality rate may fall before --baseline reports a regression (0-1)")
 
 	return cmd
 }
 
 func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandErr error) {
+	if f.regressionTolerance > evaluation.MaxTolerance {
+		return fmt.Errorf("--regression-tolerance must be between 0 and %v; %v would disable the aggregate gate",
+			evaluation.MaxTolerance, f.regressionTolerance)
+	}
+
 	telemetry.TrackCommand(cmd.Context(), "eval", args)
 	defer func() { // do not inline this defer so that commandErr is not resolved early
 		telemetry.TrackCommandError(cmd.Context(), "eval", args, commandErr)
@@ -149,5 +164,41 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) (commandEr
 
 	fmt.Fprintf(teeOut, "Log: %s\n", logPath)
 
-	return evalErr
+	// Only compare a run that completed. A partial run's missing evaluations
+	// register as "absent" and do not gate, so comparing would print
+	// "✅ No regression against baseline" for a broken run and then exit
+	// non-zero — contradictory, and the reassuring half is the one people read.
+	if evalErr != nil {
+		if f.baseline != "" {
+			fmt.Fprintln(teeOut, "\nSkipping baseline comparison: the run did not complete.")
+		}
+		return evalErr
+	}
+
+	return f.checkBaseline(teeOut, run)
+}
+
+// checkBaseline compares run against the configured baseline and returns a
+// non-nil error when it regressed, so CI fails on the exit code. A no-op when
+// --baseline was not supplied.
+func (f *evalFlags) checkBaseline(out io.Writer, run *evaluation.EvalRun) error {
+	if f.baseline == "" {
+		return nil
+	}
+
+	baseline, err := evaluation.LoadBaseline(f.baseline)
+	if err != nil {
+		return err
+	}
+
+	comparison, err := evaluation.Compare(baseline, run, f.regressionTolerance)
+	if err != nil {
+		return err
+	}
+	evaluation.PrintComparison(out, comparison)
+
+	if comparison.Regressed {
+		return errors.New("evaluation regressed against baseline")
+	}
+	return nil
 }

cmd/root/eval_baseline_test.go

@@ -0,0 +1,127 @@
+package root
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/docker/docker-agent/pkg/evaluation"
+	"github.com/docker/docker-agent/pkg/session"
+)
+
+// sizeRun builds a run in the shape the eval command produces, including the
+// session that carries the saved pass/fail flag.
+func sizeRun(pass bool) *evaluation.EvalRun {
+	r := evaluation.Result{
+		InputPath:    "a.json",
+		Title:        "a",
+		SizeExpected: "medium",
+		Size:         "medium",
+		Session:      &session.Session{Title: "a"},
+	}
+	if !pass {
+		r.Size = "short"
+	}
+	return &evaluation.EvalRun{Name: "run", Results: []evaluation.Result{r}}
+}
+
+// saveBaseline writes a run exactly as `docker agent eval` does.
+func saveBaseline(t *testing.T, run *evaluation.EvalRun) string {
+	t.Helper()
+	path, err := evaluation.SaveRunSessionsJSON(run, t.TempDir())
+	require.NoError(t, err)
+	return path
+}
+
+func TestCheckBaseline_NoBaselineIsANoOp(t *testing.T) {
+	t.Parallel()
+
+	f := &evalFlags{}
+	var buf bytes.Buffer
+	require.NoError(t, f.checkBaseline(&buf, sizeRun(false)))
+	assert.Empty(t, buf.String(), "without --baseline nothing is compared or printed")
+}
+
+func TestCheckBaseline_RegressionReturnsAnError(t *testing.T) {
+	t.Parallel()
+
+	f := &evalFlags{baseline: saveBaseline(t, sizeRun(true))}
+	var buf bytes.Buffer
+	err := f.checkBaseline(&buf, sizeRun(false))
+
+	require.Error(t, err, "a regression must surface as a non-zero exit")
+	assert.Contains(t, err.Error(), "regressed against baseline")
+	assert.Contains(t, buf.String(), "Regression against baseline")
+}
+
+func TestCheckBaseline_NoRegressionSucceeds(t *testing.T) {
+	t.Parallel()
+
+	f := &evalFlags{baseline: saveBaseline(t, sizeRun(true))}
+	var buf bytes.Buffer
+	require.NoError(t, f.checkBaseline(&buf, sizeRun(true)))
+	assert.Contains(t, buf.String(), "No regression against baseline")
+}
+
+// The gate must fail closed rather than reporting success against a baseline it
+// cannot actually compare with.
+func TestCheckBaseline_FailsClosedOnAnUnusableBaseline(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "not-a-run.json")
+	require.NoError(t, os.WriteFile(path, []byte(`{"name":"not-an-eval-run"}`), 0o600))
+
+	f := &evalFlags{baseline: path}
+	var buf bytes.Buffer
+	err := f.checkBaseline(&buf, sizeRun(false))
+
+	require.ErrorIs(t, err, evaluation.ErrNoBaselineEvals)
+	assert.NotContains(t, buf.String(), "No regression",
+		"an unusable baseline must never print a reassuring verdict")
+}
+
+func TestCheckBaseline_MissingBaselineFileIsAnError(t *testing.T) {
+	t.Parallel()
+
+	f := &evalFlags{baseline: filepath.Join(t.TempDir(), "nope.json")}
+	var buf bytes.Buffer
+	err := f.checkBaseline(&buf, sizeRun(true))
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "reading baseline",
+		"a bad --baseline path must fail loudly rather than silently skipping the gate")
+}
+
+func TestEvalCmd_BaselineFlagsAreRegistered(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+	require.NotNil(t, cmd.Flags().Lookup("baseline"))
+
+	tolerance := cmd.Flags().Lookup("regression-tolerance")
+	require.NotNil(t, tolerance)
+	assert.Equal(t, "0", tolerance.DefValue, "the default gates any drop")
+}
+
+// A tolerance above 1 cannot be met by any rate movement, so it silently
+// disables the aggregate gate. Rejecting it is a startup error, not a surprise
+// discovered when a regression sails through.
+func TestEvalCmd_RejectsTooLargeTolerance(t *testing.T) {
+	t.Parallel()
+
+	cmd := newEvalCmd()
+	require.NoError(t, cmd.Flags().Set("regression-tolerance", "10"))
+	cmd.SetOut(&bytes.Buffer{})
+	cmd.SetErr(&bytes.Buffer{})
+
+	err := cmd.Args(cmd, []string{"agent.yaml"})
+	require.NoError(t, err)
+
+	err = cmd.RunE(cmd, []string{"agent.yaml"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "--regression-tolerance must be between 0 and 1")
+}

docs/features/cli/index.md

@@ -468,6 +468,8 @@ $ docker agent eval <agent-file>|<registry-ref> [<eval-dir>|./evals] [flags]
 | `--keep-containers` | `false`                              | Keep containers after evaluation (don't remove with `--rm`)                |
 | `-e, --env`         | (none)                               | Environment variables to pass to container (`KEY` or `KEY=VALUE`, repeatable) |
 | `--repeat <n>`      | `1`                                  | Number of times to repeat each evaluation (useful for computing baselines) |
+| `--baseline <file>` | (none)                               | Compare against a previously saved run JSON (`<output>/<run>.json`) and exit non-zero on regression |
+| `--regression-tolerance <n>` | `0`                         | How far an aggregate quality rate may fall before `--baseline` reports a regression (0–1) |
 
 All [runtime configuration flags](#runtime-configuration-flags) are also accepted.
 

docs/features/evaluation/index.md

@@ -168,6 +168,39 @@ $ docker agent eval <agent-file>|<registry-ref> [<eval-dir>|./evals]
 | `--keep-containers` | `false`                     | Keep containers after evaluation (don't remove with `--rm`)       |
 | `-e, --env`         | (none)                      | Environment variables to pass to container (`KEY` or `KEY=VALUE`) |
 | `--repeat`          | `1`                         | Number of times to repeat each evaluation (useful for computing baselines) |
+| `--baseline`        | (none)                      | Compare against a previously saved run JSON and exit non-zero on regression (see [Regression gate](#regression-gate)) |
+| `--regression-tolerance` | `0`                    | How far an aggregate quality rate may fall before `--baseline` reports a regression (0–1) |
+
+### Regression gate
+
+`--baseline` compares the run against a previous one and exits non-zero when
+quality regressed, so an eval suite can gate CI:
+
+```console
+$ docker agent eval ./agent.yaml --baseline results/2026-08-01-run.json
+```
+
+The baseline is the run JSON written by a previous invocation —
+`<output>/<run-name>.json` — so there is no separate artifact to produce.
+
+Four rules decide the verdict, and they are worth knowing before wiring this
+into CI:
+
+- **The tolerance governs aggregate rates only.** An LLM judge does not return
+  the same score twice, so without a tolerance the gate flaps. `--regression-tolerance 0.05`
+  lets an aggregate rate fall five points before it counts.
+- **An evaluation that passed and now fails always gates**, regardless of the
+  tolerance. That transition is the signal the gate exists to catch, so it is
+  never absorbed.
+- **Cost is reported but never gates.** A provider price change is not a quality
+  regression.
+- **An added *failing* evaluation gates** via the aggregate rate, even though no
+  existing evaluation regressed. A suite that got worse should say so — but it
+  means committing a known-failing eval needs a tolerance bump or a fix.
+
+A baseline that carries no evaluations, or a run that produced none (an
+`--only` pattern that matched nothing), is rejected rather than reported as
+passing: a gate that cannot fail is worse than no gate.
 
 ### Provider Credentials