add configurable client ACL init startup staggering to smooth login storm (#5021)

Author: sujay-hashicorp

.changelog/5021.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+client: Add optional startup staggering for client ACL init to spread /v1/acl/login calls and reduce login storms on large clusters. Controlled via client.aclInit.startupStagger.* values (disabled by default).
+```

charts/consul/templates/client-daemonset.yaml

@@ -518,6 +518,8 @@ spec:
       {{- if (or .Values.global.acls.manageSystemACLs (and .Values.global.tls.enabled (not .Values.global.tls.enableAutoEncrypt))) }}
       initContainers:
       {{- if .Values.global.acls.manageSystemACLs }}
+      {{- $startupStagger := default (dict "enabled" false "minSeconds" 0 "maxSeconds" 0) .Values.client.aclInit.startupStagger }}
+      {{- $staggerEnabled := and $startupStagger.enabled (gt (int $startupStagger.maxSeconds) 0) }}
       - name: client-acl-init
         image: {{ .Values.global.imageK8S }}
         {{ template "consul.imagePullPolicy" . }}
@@ -543,6 +545,23 @@ spec:
           - "/bin/sh"
           - "-ec"
           - |
+            {{- if $staggerEnabled }}
+            min_delay={{ int $startupStagger.minSeconds }}
+            max_delay={{ int $startupStagger.maxSeconds }}
+            if [ $max_delay -lt $min_delay ]; then
+              echo "client-acl-init: startupStagger.maxSeconds (${max_delay}) must be >= startupStagger.minSeconds (${min_delay})" >&2
+              exit 1
+            fi
+            range=$((max_delay - min_delay))
+            if [ $range -gt 0 ]; then
+              jitter=$((RANDOM % (range + 1)))
+            else
+              jitter=0
+            fi
+            sleep_time=$((min_delay + jitter))
+            echo "client-acl-init: staggering for ${sleep_time}s before ACL login"
+            sleep ${sleep_time}
+            {{- end }}
             exec consul-k8s-control-plane acl-init \
               -log-level={{ default .Values.global.logLevel .Values.client.logLevel }} \
               -log-json={{ .Values.global.logJSON }} \

charts/consul/test/unit/client-daemonset.bats

@@ -1356,6 +1356,41 @@ load _helpers
   [ "${actual}" = "true" ]  
 }
 
+@test "client/DaemonSet: acl-init adds startupStagger sleep when enabled" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/client-daemonset.yaml  \
+      --set 'client.enabled=true' \
+      --set 'global.acls.manageSystemACLs=true' \
+      --set 'client.aclInit.startupStagger.enabled=true' \
+      --set 'client.aclInit.startupStagger.minSeconds=10' \
+      --set 'client.aclInit.startupStagger.maxSeconds=40' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.initContainers[] | select(.name == "client-acl-init") | .command' | tee /dev/stderr)
+
+  local actual=$(echo $command | jq -r ' . | any(contains("min_delay=10"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $command | jq -r ' . | any(contains("max_delay=40"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $command | jq -r ' . | any(contains("client-acl-init: staggering for"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "client/DaemonSet: acl-init startupStagger disabled by default" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/client-daemonset.yaml  \
+      --set 'client.enabled=true' \
+      --set 'global.acls.manageSystemACLs=true' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.initContainers[] | select(.name == "client-acl-init") | .command' | tee /dev/stderr)
+
+  local actual=$(echo $command | jq -r ' . | any(contains("startupStagger"))' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
 @test "client/DaemonSet: init container is created when global.acls.manageSystemACLs=true and has correct command with Partitions enabled" {
   cd `chart_dir`
   local object=$(helm template \

charts/consul/values.yaml

@@ -1737,6 +1737,16 @@ client:
     # @recurse: false
     tlsInit: null
 
+  aclInit:
+    startupStagger:
+      # When true, adds a randomized sleep before running acl-init to avoid login storms
+      # against the Consul servers in large clusters. Defaults set to 0 for no sleep.
+      enabled: false
+      # Minimum seconds to sleep before attempting the ACL login.
+      minSeconds: 0
+      # Maximum seconds to sleep before attempting the ACL login.
+      maxSeconds: 0
+
   # A raw string of extra [JSON configuration](https://developer.hashicorp.com/consul/docs/agent/config/config-files) for Consul
   # clients. This will be saved as-is into a ConfigMap that is read by the Consul
   # client agents. This can be used to add additional configuration that