From 9859b1a3abf2aafc129f961c48a991bd38d30709 Mon Sep 17 00:00:00 2001 From: Iskren Date: Tue, 21 Apr 2026 02:52:01 +0300 Subject: [PATCH] Add custom Prometheus metrics --- internal/controller/httpbin_controller.go | 4 ++ .../controller/httpbin_controller_test.go | 13 +++++ .../httpbindeployment_controller.go | 11 ++++ internal/metrics/metrics.go | 54 +++++++++++++++++++ internal/metrics/metrics_test.go | 54 +++++++++++++++++++ 5 files changed, 136 insertions(+) create mode 100644 internal/metrics/metrics.go create mode 100644 internal/metrics/metrics_test.go diff --git a/internal/controller/httpbin_controller.go b/internal/controller/httpbin_controller.go index 8562be0..c362f25 100644 --- a/internal/controller/httpbin_controller.go +++ b/internal/controller/httpbin_controller.go @@ -23,6 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" orchestratev1alpha1 "http-operator/api/v1alpha1" + "http-operator/internal/metrics" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -81,6 +82,7 @@ func (r *HttpBinReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct "error_type", errors.IsNotFound(err), "error_forbidden", errors.IsForbidden(err), "error_invalid", errors.IsInvalid(err)) + metrics.HttpBinReconciled.WithLabelValues("error").Inc() return ctrl.Result{}, err } @@ -148,6 +150,7 @@ func (r *HttpBinReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct "HttpBinDeployment.Name", httpBinDeployment.Name, "HttpBinDeployment.Spec", httpBinDeployment.Spec) + metrics.HttpBinReconciled.WithLabelValues("created").Inc() // Deployment created successfully - return and requeue return ctrl.Result{Requeue: true}, nil } else if err != nil { @@ -191,6 +194,7 @@ func (r *HttpBinReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } + metrics.HttpBinReconciled.WithLabelValues("success").Inc() return ctrl.Result{}, nil } diff --git a/internal/controller/httpbin_controller_test.go b/internal/controller/httpbin_controller_test.go index 0070d91..0440c26 100644 --- a/internal/controller/httpbin_controller_test.go +++ b/internal/controller/httpbin_controller_test.go @@ -22,12 +22,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus/testutil" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" orchestratev1alpha1 "http-operator/api/v1alpha1" + "http-operator/internal/metrics" ) var _ = Describe("HttpBin Controller", func() { @@ -86,6 +88,8 @@ var _ = Describe("HttpBin Controller", func() { Scheme: k8sClient.Scheme(), } + before := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("created")) + result, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ NamespacedName: typeNamespacedName, }) @@ -93,6 +97,10 @@ var _ = Describe("HttpBin Controller", func() { // Check that reconciliation requests a requeue Expect(result).NotTo(Equal(reconcile.Result{})) + By("Checking that the created metric was incremented") + after := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("created")) + Expect(after - before).To(Equal(float64(1))) + By("Verifying HttpBinDeployment was created") httpBinDeployment := &orchestratev1alpha1.HttpBinDeployment{} Eventually(func() error { @@ -190,11 +198,16 @@ var _ = Describe("HttpBin Controller", func() { Expect(k8sClient.Status().Update(ctx, httpBinDeployment)).To(Succeed()) By("Reconciling again to propagate status") + beforeSuccess := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("success")) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ NamespacedName: typeNamespacedName, }) Expect(err).NotTo(HaveOccurred()) + By("Checking that the success metric was incremented") + afterSuccess := testutil.ToFloat64(metrics.HttpBinReconciled.WithLabelValues("success")) + Expect(afterSuccess - beforeSuccess).To(Equal(float64(1))) + By("Verifying HttpBin status was updated") httpBin := &orchestratev1alpha1.HttpBin{} Expect(k8sClient.Get(ctx, typeNamespacedName, httpBin)).To(Succeed()) diff --git a/internal/controller/httpbindeployment_controller.go b/internal/controller/httpbindeployment_controller.go index bdf2462..d496061 100644 --- a/internal/controller/httpbindeployment_controller.go +++ b/internal/controller/httpbindeployment_controller.go @@ -16,6 +16,7 @@ import ( gatewayApi "sigs.k8s.io/gateway-api/apis/v1" orchestratev1alpha1 "http-operator/api/v1alpha1" + "http-operator/internal/metrics" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -176,6 +177,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re _ = r.RemoteClient.Status().Update(ctx, httpBinDeployment) return ctrl.Result{}, err } + metrics.DeploymentOperations.WithLabelValues("created").Inc() } else if err != nil { logger.Error(err, "Failed to get Deployment") return ctrl.Result{}, err @@ -191,6 +193,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re _ = r.RemoteClient.Status().Update(ctx, httpBinDeployment) return ctrl.Result{}, err } + metrics.DeploymentOperations.WithLabelValues("updated").Inc() } // Handle Service in local cluster (Cluster A) @@ -212,6 +215,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re _ = r.RemoteClient.Status().Update(ctx, httpBinDeployment) return ctrl.Result{}, err } + metrics.ServiceOperations.WithLabelValues("created").Inc() } else if err != nil { logger.Error(err, "Failed to get Service") return ctrl.Result{}, err @@ -228,6 +232,7 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re _ = r.RemoteClient.Status().Update(ctx, httpBinDeployment) return ctrl.Result{}, err } + metrics.ServiceOperations.WithLabelValues("updated").Inc() } ingress := &networkingv1.Ingress{} @@ -400,9 +405,12 @@ func (r *HttpBinDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re statusNeedsUpdate = true if httpBinDeployment.Status.IsDeploymentReady { setHttpBinDeploymentStatusCondition(httpBinDeployment, metav1.ConditionTrue, orchestratev1alpha1.HttpBinDeploymentConditionTypeReady, orchestratev1alpha1.HttpBinDeploymentConditionReasonReady, "HttpBinDeployment instance is ready") + metrics.DeploymentReady.WithLabelValues(httpBinDeployment.Name).Set(1) } else { setHttpBinDeploymentStatusCondition(httpBinDeployment, metav1.ConditionFalse, orchestratev1alpha1.HttpBinDeploymentConditionTypeReady, orchestratev1alpha1.HttpBinDeploymentConditionReasonHttpBinInstanceProgressing, "HttpBinDeployment instance is not yet available") + metrics.DeploymentReady.WithLabelValues(httpBinDeployment.Name).Set(0) } + metrics.ReadyReplicas.WithLabelValues(httpBinDeployment.Name).Set(float64(deployment.Status.ReadyReplicas)) } // Update status if any changes were made @@ -490,6 +498,9 @@ func (r *HttpBinDeploymentReconciler) finalizeHttpBinDeployment(ctx context.Cont if err := r.LocalClient.Delete(ctx, deployment); err != nil && !errors.IsNotFound(err) { return err } + metrics.DeploymentOperations.WithLabelValues("deleted").Inc() + metrics.DeploymentReady.DeleteLabelValues(httpBinDeployment.Name) + metrics.ReadyReplicas.DeleteLabelValues(httpBinDeployment.Name) // Delete Service serviceName := r.getResourceName(httpBinDeployment) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..9b573f1 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + HttpBinReconciled = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "httpbin_operator_httpbin_reconciled_total", + Help: "Total number of HttpBin reconcile calls by result.", + }, + []string{"result"}, + ) + DeploymentOperations = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "httpbin_operator_deployments_total", + Help: "Total number of local Deployment operations performed.", + }, + []string{"operation"}, + ) + ServiceOperations = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "httpbin_operator_services_total", + Help: "Total number of local Service operations performed.", + }, + []string{"operation"}, + ) + DeploymentReady = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "httpbin_operator_deployment_ready", + Help: "Whether the HttpBinDeployment is ready (1) or not (0).", + }, + []string{"name"}, + ) + ReadyReplicas = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "httpbin_operator_ready_replicas", + Help: "Number of ready replicas for each HttpBinDeployment.", + }, + []string{"name"}, + ) +) + +func init() { + ctrlmetrics.Registry.MustRegister( + HttpBinReconciled, + DeploymentOperations, + ServiceOperations, + DeploymentReady, + ReadyReplicas, + ) +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..e8727c4 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "fmt" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestMetricsManual(t *testing.T) { + // Simulate HttpBin reconcile outcomes + HttpBinReconciled.WithLabelValues("success").Inc() + HttpBinReconciled.WithLabelValues("success").Inc() + HttpBinReconciled.WithLabelValues("created").Inc() + HttpBinReconciled.WithLabelValues("error").Inc() + + // Simulate Deployment operations + DeploymentOperations.WithLabelValues("created").Inc() + DeploymentOperations.WithLabelValues("created").Inc() + DeploymentOperations.WithLabelValues("updated").Inc() + DeploymentOperations.WithLabelValues("deleted").Inc() + + // Simulate Service operations + ServiceOperations.WithLabelValues("created").Inc() + ServiceOperations.WithLabelValues("updated").Inc() + + // Simulate ready state for two instances + DeploymentReady.WithLabelValues("httpbin-a").Set(1) + DeploymentReady.WithLabelValues("httpbin-b").Set(0) + ReadyReplicas.WithLabelValues("httpbin-a").Set(2) + ReadyReplicas.WithLabelValues("httpbin-b").Set(0) + + fmt.Println("\n--- httpbin_operator_httpbin_reconciled_total ---") + fmt.Printf(" success: %.0f\n", testutil.ToFloat64(HttpBinReconciled.WithLabelValues("success"))) + fmt.Printf(" created: %.0f\n", testutil.ToFloat64(HttpBinReconciled.WithLabelValues("created"))) + fmt.Printf(" error: %.0f\n", testutil.ToFloat64(HttpBinReconciled.WithLabelValues("error"))) + + fmt.Println("\n--- httpbin_operator_deployments_total ---") + fmt.Printf(" created: %.0f\n", testutil.ToFloat64(DeploymentOperations.WithLabelValues("created"))) + fmt.Printf(" updated: %.0f\n", testutil.ToFloat64(DeploymentOperations.WithLabelValues("updated"))) + fmt.Printf(" deleted: %.0f\n", testutil.ToFloat64(DeploymentOperations.WithLabelValues("deleted"))) + + fmt.Println("\n--- httpbin_operator_services_total ---") + fmt.Printf(" created: %.0f\n", testutil.ToFloat64(ServiceOperations.WithLabelValues("created"))) + fmt.Printf(" updated: %.0f\n", testutil.ToFloat64(ServiceOperations.WithLabelValues("updated"))) + + fmt.Println("\n--- httpbin_operator_deployment_ready ---") + fmt.Printf(" httpbin-a: %.0f\n", testutil.ToFloat64(DeploymentReady.WithLabelValues("httpbin-a"))) + fmt.Printf(" httpbin-b: %.0f\n", testutil.ToFloat64(DeploymentReady.WithLabelValues("httpbin-b"))) + + fmt.Println("\n--- httpbin_operator_ready_replicas ---") + fmt.Printf(" httpbin-a: %.0f\n", testutil.ToFloat64(ReadyReplicas.WithLabelValues("httpbin-a"))) + fmt.Printf(" httpbin-b: %.0f\n", testutil.ToFloat64(ReadyReplicas.WithLabelValues("httpbin-b"))) +}