feat: add sandbox_creation_latency metric

Adds a new histogram metric 'sandbox_creation_latency_seconds' to track
the time it takes for a Sandbox to become Ready after it has been
created.

This metric is recorded in the sandbox controller when the Sandbox's
Ready condition transitions to True.

A unit test is also added to verify that the metric is recorded
correctly.

Author: barney-s

controllers/sandbox_controller.go

@@ -22,6 +22,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -35,6 +36,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	sandboxv1alpha1 "sigs.k8s.io/agent-sandbox/api/v1alpha1"
@@ -48,11 +50,19 @@ const (
 var (
 	// Scheme for use by sandbox controllers. Registers required types for client.
 	Scheme = runtime.NewScheme()
+
+	sandboxCreationLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name: "sandbox_creation_latency_seconds",
+			Help: "Time taken from sandbox creation to sandbox being ready",
+		},
+	)
 )
 
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(Scheme))
 	utilruntime.Must(sandboxv1alpha1.AddToScheme(Scheme))
+	metrics.Registry.MustRegister(sandboxCreationLatency)
 }
 
 // SandboxReconciler reconciles a Sandbox object
@@ -225,6 +235,16 @@ func (r *SandboxReconciler) updateStatus(ctx context.Context, oldStatus *sandbox
 		return err
 	}
 
+	// record metric if we have just become ready
+	oldReadyCondition := meta.FindStatusCondition(oldStatus.Conditions, string(sandboxv1alpha1.SandboxConditionReady))
+	newReadyCondition := meta.FindStatusCondition(sandbox.Status.Conditions, string(sandboxv1alpha1.SandboxConditionReady))
+
+	if newReadyCondition != nil && newReadyCondition.Status == metav1.ConditionTrue && (oldReadyCondition == nil || oldReadyCondition.Status != metav1.ConditionTrue) {
+		latency := time.Since(sandbox.CreationTimestamp.Time)
+		sandboxCreationLatency.Observe(latency.Seconds())
+		log.Info("recorded sandbox_creation_latency_seconds", "latency", latency)
+	}
+
 	// Surface error
 	return nil
 }

controllers/sandbox_controller_test.go

@@ -15,12 +15,14 @@
 package controllers
 
 import (
+	"context"
 	"errors"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -33,6 +35,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 func newFakeClient(initialObjs ...runtime.Object) client.WithWatch {
@@ -649,3 +652,74 @@ func TestSandboxExpiry(t *testing.T) {
 		})
 	}
 }
+
+func TestReconcileMetric(t *testing.T) {
+	sandboxName := "sandbox-name"
+	sandboxNs := "sandbox-ns"
+	sb := &sandboxv1alpha1.Sandbox{}
+	sb.Name = sandboxName
+	sb.Namespace = sandboxNs
+	sb.Generation = 1
+	sb.CreationTimestamp = metav1.NewTime(time.Now().Add(-10 * time.Second))
+	sb.Spec = sandboxv1alpha1.SandboxSpec{
+		PodTemplate: sandboxv1alpha1.PodTemplate{
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name: "test-container",
+					},
+				},
+			},
+		},
+	}
+
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      sandboxName,
+			Namespace: sandboxNs,
+		},
+		Status: corev1.PodStatus{
+			Phase: corev1.PodRunning,
+			Conditions: []corev1.PodCondition{
+				{
+					Type:   corev1.PodReady,
+					Status: corev1.ConditionTrue,
+				},
+			},
+		},
+	}
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      sandboxName,
+			Namespace: sandboxNs,
+		},
+	}
+
+	r := SandboxReconciler{
+		Client: newFakeClient(sb, pod, svc),
+		Scheme: Scheme,
+	}
+
+	_, err := r.Reconcile(context.Background(), ctrl.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      sandboxName,
+			Namespace: sandboxNs,
+		},
+	})
+	require.NoError(t, err)
+
+	// Validate metric
+	metricFamilies, err := metrics.Registry.Gather()
+	require.NoError(t, err)
+	var creationLatencyMetric *dto.MetricFamily
+	for _, mf := range metricFamilies {
+		if *mf.Name == "sandbox_creation_latency_seconds" {
+			creationLatencyMetric = mf
+			break
+		}
+	}
+	require.NotNil(t, creationLatencyMetric)
+	require.Equal(t, 1, len(creationLatencyMetric.Metric))
+	require.Equal(t, uint64(1), *creationLatencyMetric.Metric[0].Histogram.SampleCount)
+	require.Greater(t, *creationLatencyMetric.Metric[0].Histogram.SampleSum, float64(0))
+}

go.mod

@@ -4,6 +4,8 @@ go 1.24.4
 
 require (
 	github.com/google/go-cmp v0.7.0
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/stretchr/testify v1.11.1
 	k8s.io/api v0.34.1
 	k8s.io/apiextensions-apiserver v0.34.1
@@ -49,8 +51,6 @@ require (
 	github.com/onsi/ginkgo/v2 v2.23.3 // indirect
 	github.com/onsi/gomega v1.37.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.1 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect