diff --git a/README.md b/README.md index 80a1979..7ba643f 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ leave `gitops-dev`, `gitops-stage` and `gitops-prod` undefined, then those steps ### Build, Push and Deploy Docker Image +#### Recommended format + +Use `gitops-namespace` and `gitops-updates`. The environment (dev/stage/prod) is derived from the git ref automatically — no need to repeat the same files three times. + ```yaml name: CD @@ -41,14 +45,40 @@ jobs: with: docker-username: ${{ vars.HARBOR_USERNAME }} docker-password: ${{ secrets.HARBOR_PASSWORD }} - docker-image: private/diablo-redbook + docker-image: sb-images/my-service gitops-token: ${{ secrets.GITOPS_TOKEN }} - gitops-dev: |- - clusters/customization/dev/mothership/diablo-redbook/diablo-redbook-helm.yaml spec.template.spec.containers.redbook.image - gitops-stage: |- - clusters/customization/stage/mothership/diablo-redbook/diablo-redbook-helm.yaml spec.template.spec.containers.redbook.image + gitops-namespace: my-service + gitops-updates: my-service-cr.yaml +``` + +The action looks up `kubernetes/namespaces///` in the GitOps repository and expands each line to a full path for every region directory found there. A service deployed only to `prod/core` will only update that directory; a customer-facing service with `prod/de1`, `prod/us1`, `prod/au1` etc. will update all of them automatically. Adding a new region to the GitOps repo requires no changes in service repos. + +The field specifier on each line is optional and resolved as follows: + +| Line format | Resolved yq field | +|---|---| +| `my-service-cr.yaml` | `spec.template.spec.containers..image` | +| `my-service-cr.yaml authentication` | `spec.template.spec.containers.authentication.image` | +| `my-service-cr.yaml spec.template.spec.initContainers.migrate.image` | used as-is | + +The second form is useful when the container name differs from the namespace (e.g. matrix builds). The third form covers init containers or any other custom yq path. Use a multiline block (`|-`) only when specifying more than one entry: + +```yaml + gitops-namespace: my-service + gitops-updates: |- + my-service-cr.yaml + my-service-cr.yaml spec.template.spec.initContainers.migrate.image +``` + +#### Explicit format (legacy / escape hatch) + +Full paths can still be specified directly. Lines starting with `kubernetes/` are passed through unchanged, so existing configurations continue to work without modification. Explicit and shorthand lines can be mixed within the same input. + +```yaml gitops-prod: |- - clusters/customization/prod/mothership/diablo-redbook/diablo-redbook-helm.yaml spec.template.spec.containers.redbook.image + kubernetes/namespaces/my-service/prod/de1/my-service-cr.yaml spec.template.spec.containers.my-service.image + kubernetes/namespaces/my-service/prod/us1/my-service-cr.yaml spec.template.spec.containers.my-service.image + kubernetes/namespaces/my-service/prod/au1/my-service-cr.yaml spec.template.spec.containers.my-service.image ``` ### Build and Push Docker Image @@ -141,9 +171,11 @@ These keys mirror the [Swarmia Deployment API](https://help.swarmia.com/settings | `gitops-user` | GitHub User for GitOps | `Staffbot` | | `gitops-email` | GitHub Email for GitOps | `staffbot@staffbase.com` | | `gitops-token` | GitHub Token for GitOps | | -| `gitops-dev` | Files which should be updated by the GitHub Action for DEV, must be relative to the root of the GitOps repository | | -| `gitops-stage` | Files which should be updated by the GitHub Action for STAGE, must be relative to the root of the GitOps repository | | -| `gitops-prod` | Files which should be updated by the GitHub Action for PROD, must be relative to the root of the GitOps repository | | +| `gitops-namespace` | Kubernetes namespace for region auto-discovery. Required when using `gitops-updates` or shorthand path format (see Usage). | | +| `gitops-updates` | Files to update for all environments. Environment derived from git ref. Replaces `gitops-dev/stage/prod` when set. | | +| `gitops-dev` | Files to update for DEV (legacy). Use `gitops-updates` instead. | | +| `gitops-stage` | Files to update for STAGE (legacy). Use `gitops-updates` instead. | | +| `gitops-prod` | Files to update for PROD (legacy). Use `gitops-updates` instead. | | | `working-directory` | The directory in which the GitOps action should be executed. The docker-file variable should be relative to working directory. | `.` | ## Outputs diff --git a/action.yml b/action.yml index 9a989f0..330d6bd 100644 --- a/action.yml +++ b/action.yml @@ -82,6 +82,12 @@ inputs: gitops-prod: description: 'Files which should be updated by the GitHub Action for PROD' required: false + gitops-namespace: + description: 'Kubernetes namespace for region auto-discovery. Required when using shorthand path format (filename + yq field without leading kubernetes/).' + required: false + gitops-updates: + description: 'Files to update for all environments. The environment (dev/stage/prod) is derived from the git ref. Replaces gitops-dev/stage/prod when set.' + required: false upwind-client-id: description: 'Upwind Client ID' required: false @@ -192,9 +198,11 @@ runs: INPUT_GITOPS_TOKEN: ${{ inputs.gitops-token }} INPUT_GITOPS_ORGANIZATION: ${{ inputs.gitops-organization }} INPUT_GITOPS_REPOSITORY: ${{ inputs.gitops-repository }} + INPUT_GITOPS_UPDATES: ${{ inputs.gitops-updates }} INPUT_GITOPS_DEV: ${{ inputs.gitops-dev }} INPUT_GITOPS_STAGE: ${{ inputs.gitops-stage }} INPUT_GITOPS_PROD: ${{ inputs.gitops-prod }} + INPUT_GITOPS_NAMESPACE: ${{ inputs.gitops-namespace }} run: ${{ github.action_path }}/scripts/update-gitops.sh - name: Emit Image Build Event to Upwind.io diff --git a/scripts/lib/gitops-functions.sh b/scripts/lib/gitops-functions.sh index 9f88ce3..8b05ec8 100755 --- a/scripts/lib/gitops-functions.sh +++ b/scripts/lib/gitops-functions.sh @@ -44,6 +44,54 @@ update_file() { yq -i '.metadata.annotations["deploy.staffbase.com/version"] = "'"${INPUT_TAG}"'"' "${file}" } +expand_with_regions() { + local file_list="$1" + local env="$2" + local namespace="$3" + local expanded="" + + while IFS= read -r line; do + [[ -z "$line" ]] && continue + + # Without a namespace, pass every line through unchanged (legacy / external repos) + if [[ -z "$namespace" ]]; then + expanded+="${line}"$'\n' + continue + fi + + # With a namespace set, explicit full paths (starting with kubernetes/) pass through unchanged + if [[ "$line" == kubernetes/* ]]; then + expanded+="${line}"$'\n' + continue + fi + + local filename field_token resolved_field + read -r filename field_token <<< "$line" + + if [[ -z "$field_token" ]]; then + resolved_field="spec.template.spec.containers.${namespace}.image" + elif [[ "$field_token" != *.* ]]; then + resolved_field="spec.template.spec.containers.${field_token}.image" + else + resolved_field="$field_token" + fi + + local regions_dir="kubernetes/namespaces/${namespace}/${env}" + if [[ -d "$regions_dir" ]]; then + for region_dir in "${regions_dir}"/*/; do + [[ -d "$region_dir" ]] || continue + local region="${region_dir%/}" + region="${region##*/}" + expanded+="${regions_dir}/${region}/${filename} ${resolved_field}"$'\n' + done + else + log_warn "Auto-discovery: directory ${regions_dir} not found, skipping" + fi + done <<< "$file_list" + + printf '%s' "$expanded" +} + process_file_updates() { local file_list="$1" local should_commit="$2" diff --git a/scripts/update-gitops.sh b/scripts/update-gitops.sh index f9037c7..36510d4 100755 --- a/scripts/update-gitops.sh +++ b/scripts/update-gitops.sh @@ -7,7 +7,9 @@ # INPUT_DOCKER_REGISTRY, INPUT_DOCKER_IMAGE, INPUT_TAG, INPUT_PUSH, # INPUT_GITOPS_USER, INPUT_GITOPS_EMAIL, # INPUT_GITOPS_TOKEN, INPUT_GITOPS_ORGANIZATION, INPUT_GITOPS_REPOSITORY -# Optional env vars: INPUT_GITOPS_DEV, INPUT_GITOPS_STAGE, INPUT_GITOPS_PROD +# Optional env vars: INPUT_GITOPS_UPDATES (preferred, applies to all envs), +# INPUT_GITOPS_DEV, INPUT_GITOPS_STAGE, INPUT_GITOPS_PROD (legacy, per-env overrides), +# INPUT_GITOPS_NAMESPACE (required when using shorthand path format) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=lib/common.sh @@ -29,22 +31,42 @@ require_env INPUT_GITOPS_REPOSITORY # shellcheck disable=SC2034 IMAGE="${INPUT_DOCKER_REGISTRY}/${INPUT_DOCKER_IMAGE}:${INPUT_TAG}" +NAMESPACE="${INPUT_GITOPS_NAMESPACE:-}" + # Configure git user git config --global user.email "${INPUT_GITOPS_EMAIL}" && git config --global user.name "${INPUT_GITOPS_USER}" -if [[ ( $GITHUB_REF == refs/heads/master || $GITHUB_REF == refs/heads/main ) && -n "${INPUT_GITOPS_STAGE:-}" ]]; then - log_info "Run update for STAGE" - process_file_updates "$INPUT_GITOPS_STAGE" "true" - -elif [[ $GITHUB_REF == refs/heads/dev && -n "${INPUT_GITOPS_DEV:-}" ]]; then - log_info "Run update for DEV" - process_file_updates "$INPUT_GITOPS_DEV" "true" +# Derive environment and commit flag from git ref +env="" +should_commit="true" +if [[ $GITHUB_REF == refs/heads/master || $GITHUB_REF == refs/heads/main ]]; then + env="stage" +elif [[ $GITHUB_REF == refs/heads/dev ]]; then + env="dev" +elif [[ $GITHUB_REF == refs/tags/* ]]; then + env="prod" +else + env="dev" + should_commit="false" +fi -elif [[ $GITHUB_REF == refs/tags/* && -n "${INPUT_GITOPS_PROD:-}" ]]; then - log_info "Run update for PROD" - process_file_updates "$INPUT_GITOPS_PROD" "true" +# Resolve file list: gitops-updates takes precedence over per-env inputs +file_list="" +if [[ -n "${INPUT_GITOPS_UPDATES:-}" ]]; then + file_list="$INPUT_GITOPS_UPDATES" +else + case "$env" in + stage) file_list="${INPUT_GITOPS_STAGE:-}" ;; + dev) file_list="${INPUT_GITOPS_DEV:-}" ;; + prod) file_list="${INPUT_GITOPS_PROD:-}" ;; + esac +fi -elif [[ -n "${INPUT_GITOPS_DEV:-}" ]]; then - log_info "Simulate update for DEV" - process_file_updates "$INPUT_GITOPS_DEV" "false" +if [[ -n "$file_list" ]]; then + if [[ "$should_commit" == "true" ]]; then + log_info "Run update for ${env^^}" + else + log_info "Simulate update for ${env^^}" + fi + process_file_updates "$(expand_with_regions "$file_list" "$env" "$NAMESPACE")" "$should_commit" fi diff --git a/tests/lib-gitops-functions.bats b/tests/lib-gitops-functions.bats index aafa1bd..311b8bd 100644 --- a/tests/lib-gitops-functions.bats +++ b/tests/lib-gitops-functions.bats @@ -141,3 +141,86 @@ file2.yaml spec.image" process_file_updates "file1.yaml spec.image" "false" ! grep -q 'git commit' "${TEST_TEMP_DIR}/git_calls.log" 2>/dev/null || true } + +# --- expand_with_regions --- + +@test "expand_with_regions passes all lines through unchanged when namespace is empty" { + result="$(expand_with_regions "manifests/app/prod/de1/deploy.yaml spec.image" "prod" "")" + [[ "$result" == *"manifests/app/prod/de1/deploy.yaml spec.image"* ]] +} + +@test "expand_with_regions passes through explicit kubernetes/ lines unchanged when namespace is empty" { + local input="kubernetes/namespaces/svc/prod/de1/svc-cr.yaml spec.image" + result="$(expand_with_regions "$input" "prod" "")" + [[ "$result" == *"kubernetes/namespaces/svc/prod/de1/svc-cr.yaml spec.image"* ]] +} + +@test "expand_with_regions expands shorthand to all discovered regions" { + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/de1" + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/us1" + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/au1" + cd "${TEST_TEMP_DIR}" + + result="$(expand_with_regions "svc-cr.yaml" "prod" "svc")" + [[ "$result" == *"kubernetes/namespaces/svc/prod/de1/svc-cr.yaml spec.template.spec.containers.svc.image"* ]] + [[ "$result" == *"kubernetes/namespaces/svc/prod/us1/svc-cr.yaml spec.template.spec.containers.svc.image"* ]] + [[ "$result" == *"kubernetes/namespaces/svc/prod/au1/svc-cr.yaml spec.template.spec.containers.svc.image"* ]] +} + +@test "expand_with_regions resolves container name shorthand to full yq path" { + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/backend/prod/de1" + cd "${TEST_TEMP_DIR}" + + result="$(expand_with_regions "authentication-cr.yaml authentication" "prod" "backend")" + [[ "$result" == *"authentication-cr.yaml spec.template.spec.containers.authentication.image"* ]] +} + +@test "expand_with_regions uses full yq path when field contains a dot" { + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/de1" + cd "${TEST_TEMP_DIR}" + + result="$(expand_with_regions "svc-cr.yaml spec.template.spec.initContainers.migrate.image" "prod" "svc")" + [[ "$result" == *"svc-cr.yaml spec.template.spec.initContainers.migrate.image"* ]] +} + +@test "expand_with_regions only discovers regions that exist in mops" { + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/internal-tool/prod/core" + cd "${TEST_TEMP_DIR}" + + result="$(expand_with_regions "internal-tool-cr.yaml spec.image" "prod" "internal-tool")" + [[ "$result" == *"prod/core/internal-tool-cr.yaml spec.image"* ]] + [[ "$result" != *"prod/de1"* ]] + [[ "$result" != *"prod/us1"* ]] +} + +@test "expand_with_regions handles mixed explicit and shorthand lines" { + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/de1" + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/us1" + cd "${TEST_TEMP_DIR}" + + input="kubernetes/namespaces/svc/prod/de1/explicit-cr.yaml spec.image +svc-cr.yaml spec.image" + result="$(expand_with_regions "$input" "prod" "svc")" + [[ "$result" == *"kubernetes/namespaces/svc/prod/de1/explicit-cr.yaml spec.image"* ]] + [[ "$result" == *"kubernetes/namespaces/svc/prod/de1/svc-cr.yaml spec.image"* ]] + [[ "$result" == *"kubernetes/namespaces/svc/prod/us1/svc-cr.yaml spec.image"* ]] +} + +@test "expand_with_regions skips empty lines" { + mkdir -p "${TEST_TEMP_DIR}/kubernetes/namespaces/svc/prod/de1" + cd "${TEST_TEMP_DIR}" + + input="svc-cr.yaml spec.image + +svc-cr.yaml spec.other.image" + result="$(expand_with_regions "$input" "prod" "svc")" + count=$(echo "$result" | grep -c "de1/svc-cr.yaml" || true) + [[ "$count" -eq 2 ]] +} + +@test "expand_with_regions warns when namespace directory does not exist" { + cd "${TEST_TEMP_DIR}" + result="$(expand_with_regions "svc-cr.yaml spec.image" "prod" "nonexistent" 2>&1)" + [[ "$result" == *"Auto-discovery"* ]] + [[ "$result" == *"not found"* ]] +} diff --git a/tests/update-gitops.bats b/tests/update-gitops.bats index 8f135df..9f8bf2b 100644 --- a/tests/update-gitops.bats +++ b/tests/update-gitops.bats @@ -17,6 +17,7 @@ setup() { export INPUT_GITOPS_TOKEN="fake-token" export INPUT_GITOPS_ORGANIZATION="Staffbase" export INPUT_GITOPS_REPOSITORY="mops" + export INPUT_GITOPS_UPDATES="" export INPUT_GITOPS_DEV="" export INPUT_GITOPS_STAGE="" export INPUT_GITOPS_PROD="" @@ -113,6 +114,126 @@ teardown() { ! grep -q 'git commit' "${TEST_TEMP_DIR}/git_calls.log" 2>/dev/null || true } +# --- gitops-updates (single input for all envs) --- + +@test "gitops-updates updates stage on main branch" { + export GITHUB_REF="refs/heads/main" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/stage/de1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_UPDATES="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Run update for STAGE" + grep -q 'kubernetes/namespaces/my-service/stage/de1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" +} + +@test "gitops-updates updates dev on dev branch" { + export GITHUB_REF="refs/heads/dev" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/dev/de1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_UPDATES="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Run update for DEV" + grep -q 'kubernetes/namespaces/my-service/dev/de1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" +} + +@test "gitops-updates updates prod on tag" { + export GITHUB_REF="refs/tags/v1.0.0" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/prod/de1" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/prod/us1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_UPDATES="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Run update for PROD" + grep -q 'kubernetes/namespaces/my-service/prod/de1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" + grep -q 'kubernetes/namespaces/my-service/prod/us1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" +} + +@test "gitops-updates simulates on feature branch without committing" { + export GITHUB_REF="refs/heads/feature/test" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/dev/de1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_UPDATES="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Simulate update for DEV" + ! grep -q 'git commit' "${TEST_TEMP_DIR}/git_calls.log" 2>/dev/null || true +} + +@test "gitops-updates takes precedence over gitops-stage" { + export GITHUB_REF="refs/heads/main" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/stage/de1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_UPDATES="my-service-cr.yaml" + export INPUT_GITOPS_STAGE="kubernetes/namespaces/other/stage/de1/other-cr.yaml spec.image" + run "$SCRIPT" + assert_success + grep -q 'my-service' "${TEST_TEMP_DIR}/yq_calls.log" + ! grep -q 'other-cr' "${TEST_TEMP_DIR}/yq_calls.log" 2>/dev/null || true +} + +# --- Shorthand auto-discovery --- + +@test "expands shorthand STAGE paths using discovered regions" { + export GITHUB_REF="refs/heads/main" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/stage/de1" + # Run the script from within the mock mops dir so directory checks work + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_STAGE="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Run update for STAGE" + grep -q 'kubernetes/namespaces/my-service/stage/de1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" +} + +@test "expands shorthand PROD paths for all discovered regions" { + export GITHUB_REF="refs/tags/v1.0.0" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/prod/de1" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/prod/us1" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/prod/au1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_PROD="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Run update for PROD" + grep -q 'kubernetes/namespaces/my-service/prod/de1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" + grep -q 'kubernetes/namespaces/my-service/prod/us1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" + grep -q 'kubernetes/namespaces/my-service/prod/au1/my-service-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" +} + +@test "shorthand simulation on feature branch does not commit" { + export GITHUB_REF="refs/heads/feature/test" + export INPUT_GITOPS_NAMESPACE="my-service" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/my-service/dev/de1" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_DEV="my-service-cr.yaml" + run "$SCRIPT" + assert_success + assert_output --partial "Simulate update for DEV" + ! grep -q 'git commit' "${TEST_TEMP_DIR}/git_calls.log" 2>/dev/null || true +} + +@test "core-only service shorthand only updates prod/core" { + export GITHUB_REF="refs/tags/v1.0.0" + export INPUT_GITOPS_NAMESPACE="internal-tool" + mkdir -p "${TEST_TEMP_DIR}/mops/kubernetes/namespaces/internal-tool/prod/core" + cd "${TEST_TEMP_DIR}/mops" + export INPUT_GITOPS_PROD="internal-tool-cr.yaml" + run "$SCRIPT" + assert_success + grep -q 'kubernetes/namespaces/internal-tool/prod/core/internal-tool-cr.yaml' "${TEST_TEMP_DIR}/yq_calls.log" + ! grep -q 'prod/de1' "${TEST_TEMP_DIR}/yq_calls.log" 2>/dev/null || true +} + # --- No files configured --- @test "does nothing when no gitops files are configured" {