Add prometheus metics component and overlay (#8)

* Refactor base install components into "core" kustomize component
* Add metrics kustomize component which installs prometheus operator,
  server and configures Kafka metrics and console
* Add core and metrics overlays
* Update install/uninstall scripts and README with new layout and
  overlay options (via OVERLAY env var)
* Add metrics overlay to smoke tests
* Increase timeout for verify scripts
* Make the wait condition for the verify scripts an optional argument

Signed-off-by: Thomas Cooper <code@tomcooper.dev>

Author: tomncooper

.github/actions/smoke-test/action.yaml

@@ -9,7 +9,7 @@ inputs:
   overlay:
     description: 'Overlay name for component verification (matches .github/config/overlays/<name>.env)'
     required: false
-    default: 'base'
+    default: 'core'
 
 runs:
   using: 'composite'
@@ -18,24 +18,29 @@ runs:
       shell: bash
       env:
         LOCAL_DIR: .
+        OVERLAY: ${{ inputs.overlay }}
         TIMEOUT: ${{ inputs.timeout }}
       run: ./install.sh
 
     - name: Verify deployments
       shell: bash
       env:
         OVERLAY: ${{ inputs.overlay }}
+        TIMEOUT: ${{ inputs.timeout }}
       run: .github/scripts/verify-install.sh
 
     - name: Run uninstall script
       shell: bash
       env:
         LOCAL_DIR: .
+        OVERLAY: ${{ inputs.overlay }}
         TIMEOUT: ${{ inputs.timeout }}
       run: ./uninstall.sh
 
     - name: Verify uninstall
       shell: bash
+      env:
+        TIMEOUT: ${{ inputs.timeout }}
       run: .github/scripts/verify-uninstall.sh
 
     - name: Debug on failure

.github/config/overlays/base.env .github/config/overlays/core.env.github/config/overlays/base.env renamed to .github/config/overlays/core.env

@@ -1,9 +1,9 @@
-# Components expected in the base (default) deployment.
-# Used by verify-install.sh, verify-uninstall.sh, and debug.sh.
+# Components expected in the core (default) deployment.
+# Used by verify-install.sh and debug.sh.
 #
 # Format:
 #   OPERATORS     - "namespace:deployment" pairs (space-separated)
-#   CUSTOM_RESOURCES - "namespace:resource" pairs (space-separated)
+#   CUSTOM_RESOURCES - "namespace:resource[:condition]" pairs (space-separated, condition defaults to Ready)
 #   NAMESPACES    - namespaces to inspect on failure (space-separated)
 
 OPERATORS="strimzi:strimzi-cluster-operator apicurio-registry:apicurio-registry-operator streamshub-console:streamshub-console-operator"

.github/config/overlays/metrics.env

@@ -0,0 +1,11 @@
+# Components expected in the metrics overlay deployment.
+# Used by verify-install.sh and debug.sh.
+#
+# Format:
+#   OPERATORS     - "namespace:deployment" pairs (space-separated)
+#   CUSTOM_RESOURCES - "namespace:resource[:condition]" pairs (space-separated, condition defaults to Ready)
+#   NAMESPACES    - namespaces to inspect on failure (space-separated)
+
+OPERATORS="strimzi:strimzi-cluster-operator apicurio-registry:apicurio-registry-operator streamshub-console:streamshub-console-operator monitoring:prometheus-operator"
+CUSTOM_RESOURCES="kafka:kafka/dev-cluster apicurio-registry:apicurioregistry3/apicurio-registry streamshub-console:console.console.streamshub.github.com/streamshub-console monitoring:prometheus.monitoring.coreos.com/prometheus:Available"
+NAMESPACES="strimzi kafka apicurio-registry streamshub-console monitoring"

.github/scripts/debug.sh

@@ -4,12 +4,12 @@
 # Reads component definitions from an overlay config file.
 #
 # Environment variables:
-#   OVERLAY - overlay name (default: "base")
+#   OVERLAY - overlay name (default: "core")
 #
 
 set +e
 
-OVERLAY="${OVERLAY:-base}"
+OVERLAY="${OVERLAY:-core}"
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 CONFIG_FILE="${SCRIPT_DIR}/../config/overlays/${OVERLAY}.env"
 

.github/scripts/verify-install.sh

@@ -4,13 +4,13 @@
 # Reads component definitions from an overlay config file.
 #
 # Environment variables:
-#   OVERLAY  - overlay name (default: "base")
+#   OVERLAY  - overlay name (default: "core")
 #   TIMEOUT  - kubectl wait timeout (default: "600s")
 #
 
 set -euo pipefail
 
-OVERLAY="${OVERLAY:-base}"
+OVERLAY="${OVERLAY:-core}"
 TIMEOUT="${TIMEOUT:-600s}"
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 CONFIG_FILE="${SCRIPT_DIR}/../config/overlays/${OVERLAY}.env"
@@ -37,7 +37,13 @@ echo ""
 
 for entry in ${CUSTOM_RESOURCES}; do
     ns="${entry%%:*}"
-    resource="${entry#*:}"
+    rest="${entry#*:}"
+    resource="${rest%:*}"
+    if [ "${rest}" != "${resource}" ]; then
+        condition="${rest##*:}"
+    else
+        condition="Ready"
+    fi
     echo "--- ${resource} (${ns}) ---"
-    kubectl wait "${resource}" --for=condition=Ready -n "${ns}" --timeout="${TIMEOUT}"
+    kubectl wait "${resource}" --for=condition="${condition}" -n "${ns}" --timeout="${TIMEOUT}"
 done

.github/workflows/integration.yaml

@@ -8,8 +8,12 @@ on:
 
 jobs:
   smoke-minikube:
-    name: smoke-minikube
+    name: smoke-minikube (${{ matrix.overlay }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        overlay: [core, metrics]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
@@ -24,10 +28,17 @@ jobs:
 
       - name: Smoke test
         uses: ./.github/actions/smoke-test
+        with:
+          timeout: 300s
+          overlay: ${{ matrix.overlay }}
 
   smoke-kind:
-    name: smoke-kind
+    name: smoke-kind (${{ matrix.overlay }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        overlay: [core, metrics]
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
@@ -46,3 +57,6 @@ jobs:
 
       - name: Smoke test
         uses: ./.github/actions/smoke-test
+        with:
+          timeout: 300s
+          overlay: ${{ matrix.overlay }}

.github/workflows/validate.yaml

@@ -16,19 +16,33 @@ jobs:
       - name: Set up kubectl
         uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4.0.1
 
-      - name: Build base layer
-        run: kubectl kustomize base/
+      - name: Build core overlay base
+        run: kubectl kustomize overlays/core/base/
 
-      - name: Build stack layer
-        run: kubectl kustomize stack/
+      - name: Build core overlay stack
+        run: kubectl kustomize overlays/core/stack/
 
-      - name: Verify quick-start labels in base
+      - name: Build metrics overlay base
+        run: kubectl kustomize overlays/metrics/base/
+
+      - name: Build metrics overlay stack
+        run: kubectl kustomize overlays/metrics/stack/
+
+      - name: Verify quick-start labels in core overlay base
+        run: |
+          kubectl kustomize overlays/core/base/ | grep -q 'app.kubernetes.io/part-of: streamshub-developer-quickstart'
+
+      - name: Verify quick-start labels in core overlay stack
+        run: |
+          kubectl kustomize overlays/core/stack/ | grep -q 'app.kubernetes.io/part-of: streamshub-developer-quickstart'
+
+      - name: Verify quick-start labels in metrics overlay base
         run: |
-          kubectl kustomize base/ | grep -q 'app.kubernetes.io/part-of: streamshub-developer-quickstart'
+          kubectl kustomize overlays/metrics/base/ | grep -q 'app.kubernetes.io/part-of: streamshub-developer-quickstart'
 
-      - name: Verify quick-start labels in stack
+      - name: Verify quick-start labels in metrics overlay stack
         run: |
-          kubectl kustomize stack/ | grep -q 'app.kubernetes.io/part-of: streamshub-developer-quickstart'
+          kubectl kustomize overlays/metrics/stack/ | grep -q 'app.kubernetes.io/part-of: streamshub-developer-quickstart'
 
   shellcheck:
     name: Lint shell scripts

README.md

@@ -15,6 +15,8 @@ A Kustomize-based repository for deploying the StreamsHub event-streaming stack
 | StreamsHub Console Operator | `streamshub-console` | Manages console instances |
 | StreamsHub Console instance | `streamshub-console` | Web UI for Kafka management |
 
+> **Optional:** The [metrics overlay](#install-with-metrics) adds Prometheus Operator, a Prometheus instance, and Kafka metrics collection via PodMonitors.
+
 ## Prerequisites
 
 - `kubectl` v1.27 or later (for Kustomize v5.0 `labels` transformer support)
@@ -39,6 +41,7 @@ The install script accepts the following environment variables:
 |----------|---------|-------------|
 | `REPO` | `streamshub/developer-quickstart` | GitHub repository path |
 | `REF` | `main` | Git ref, branch, or tag |
+| `OVERLAY` | *(empty)* | Overlay to apply (e.g. `metrics`) |
 | `TIMEOUT` | `120s` | `kubectl wait` timeout |
 
 Example with a pinned version:
@@ -47,14 +50,24 @@ Example with a pinned version:
 curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/install.sh | REF=v1.0.0 bash
 ```
 
+### Install with Metrics
+
+Deploy the stack with Prometheus metrics collection:
+
+```shell
+curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/install.sh | OVERLAY=metrics bash
+```
+
+This adds the Prometheus Operator and a Prometheus instance (namespace: `monitoring`), enables Kafka metrics via [Strimzi Metrics Reporter](https://strimzi.io/docs/operators/latest/deploying#proc-metrics-kafka-str), and wires Console to display metrics.
+
 ## Manual Install
 
 If you prefer to control each step, the stack is installed in two phases:
 
 ### Phase 1 — Operators and CRDs
 
 ```shell
-kubectl apply -k 'https://github.com/streamshub/developer-quickstart//base?ref=main'
+kubectl apply -k 'https://github.com/streamshub/developer-quickstart//overlays/core/base?ref=main'
 ```
 
 Wait for the operators to become ready:
@@ -68,7 +81,23 @@ kubectl wait --for=condition=Available deployment/streamshub-console-operator -n
 ### Phase 2 — Operands
 
 ```shell
-kubectl apply -k 'https://github.com/streamshub/developer-quickstart//stack?ref=main'
+kubectl apply -k 'https://github.com/streamshub/developer-quickstart//overlays/core/stack?ref=main'
+```
+
+### Manual Install with Metrics
+
+To include the metrics overlay, use the `overlays/metrics` paths instead of `overlays/core`.
+
+```shell
+# Phase 1
+kubectl create -k 'https://github.com/streamshub/developer-quickstart//overlays/metrics/base?ref=main'
+kubectl wait --for=condition=Available deployment/prometheus-operator -n monitoring --timeout=120s
+kubectl wait --for=condition=Available deployment/strimzi-cluster-operator -n strimzi --timeout=120s
+kubectl wait --for=condition=Available deployment/apicurio-registry-operator -n apicurio-registry --timeout=120s
+kubectl wait --for=condition=Available deployment/streamshub-console-operator -n streamshub-console --timeout=120s
+
+# Phase 2
+kubectl apply -k 'https://github.com/streamshub/developer-quickstart//overlays/metrics/stack?ref=main'
 ```
 
 ## Accessing the Console
@@ -142,6 +171,9 @@ The uninstall script handles safe teardown with shared-cluster safety checks:
 
 ```shell
 curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/uninstall.sh | bash
+
+# If installed with the metrics overlay:
+curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/uninstall.sh | OVERLAY=metrics bash
 ```
 
 The script:
@@ -156,7 +188,7 @@ The script:
 **Phase 1 — Delete operands:**
 
 ```shell
-kubectl delete -k 'https://github.com/streamshub/developer-quickstart//stack?ref=main'
+kubectl delete -k 'https://github.com/streamshub/developer-quickstart//overlays/core/stack?ref=main'
 ```
 
 Wait for all custom resources to be fully removed before proceeding.
@@ -169,9 +201,11 @@ Wait for all custom resources to be fully removed before proceeding.
 > ```
 
 ```shell
-kubectl delete -k 'https://github.com/streamshub/developer-quickstart//base?ref=main'
+kubectl delete -k 'https://github.com/streamshub/developer-quickstart//overlays/core/base?ref=main'
 ```
 
+For the metrics overlay, use `overlays/metrics/base` and `overlays/metrics/stack` instead.
+
 ### Finding Quick-Start Resources
 
 All resources carry the label `app.kubernetes.io/part-of=streamshub-developer-quickstart`:
@@ -201,7 +235,7 @@ Use the `update-version.sh` script to update component versions:
 ./update-version.sh strimzi 0.52.0
 ```
 
-Supported components: `strimzi`, `apicurio-registry`, `streamshub-console`
+Supported components: `strimzi`, `apicurio-registry`, `streamshub-console`, `prometheus-operator`
 
 ### Testing scripts locally
 
@@ -229,17 +263,25 @@ LOCAL_DIR=/home/user/repos/developer-quickstart ./install.sh
 ## Repository Structure
 
 ```
-base/                               # Phase 1: Operators & CRDs
-├── kustomization.yaml              # Composes all operator sub-components
-├── strimzi-operator/               # Strimzi Kafka Operator
-├── apicurio-registry-operator/     # Apicurio Registry Operator
-└── streamshub-console-operator/    # StreamsHub Console Operator
-
-stack/                              # Phase 2: Operands (Custom Resources)
-├── kustomization.yaml              # Composes all operand sub-components
-├── kafka/                          # Single-node Kafka cluster
-├── apicurio-registry/              # In-memory registry instance
-└── streamshub-console/             # Console instance
-
-overlays/                           # Future: variant configurations
+components/                         # Reusable Kustomize components
+├── core/                           # Core stack component
+│   ├── base/                       # Operators & CRDs
+│   │   ├── strimzi-operator/       # Strimzi Kafka Operator
+│   │   ├── apicurio-registry-operator/  # Apicurio Registry Operator
+│   │   └── streamshub-console-operator/ # StreamsHub Console Operator
+│   └── stack/                      # Operands (Custom Resources)
+│       ├── kafka/                  # Single-node Kafka cluster
+│       ├── apicurio-registry/      # In-memory registry instance
+│       └── streamshub-console/     # Console instance
+└── metrics/                        # Prometheus metrics component
+    ├── base/                       # Prometheus Operator
+    └── stack/                      # Prometheus instance, PodMonitors, patches
+
+overlays/                           # Deployable configurations
+├── core/                           # Default install (core only)
+│   ├── base/                       # Phase 1: Operators & CRDs
+│   └── stack/                      # Phase 2: Operands
+└── metrics/                        # Core + Prometheus metrics
+    ├── base/                       # Phase 1: Operators & CRDs + Prometheus Operator
+    └── stack/                      # Phase 2: Operands + Prometheus instance & monitors
 ```