Merge pull request #26 from improbable-eng/support-gcr-1

Author: 72636c

README.md

@@ -152,49 +152,74 @@ steps:
       - docker#v3.3.0
 ```
 
-### Specifying an ECR repository name
+### Changing the max cache time
 
-The plugin pushes and pulls Docker images to and from an ECR repository named
-`build-cache/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}`. You can
-optionally use a custom repository name:
+By default images are kept in ECR for up to 30 days. This can be changed by specifying a `max-age-days` parameter:
 
 ```yaml
 steps:
   - command: 'echo wow'
     plugins:
       - seek-oss/docker-ecr-cache#v1.7.0:
-          ecr-name: my-unique-repository-name
-          ecr-tags: 
-            Key: Value
-            Key2: Value2
+          max-age-days: 7
       - docker#v3.3.0
 ```
 
-### Changing the max cache time
+### Changing the name of exported variable
 
-By default images are kept in ECR for up to 30 days. This can be changed by specifying a `max-age-days` parameter:
+By default image name and computed tag are exported to the Docker buildkite plugin env variable `BUILDKITE_PLUGIN_DOCKER_IMAGE`. In order to chain the plugin with a different plugin, this can be changed by specifying a `export-env-variable` parameter:
 
 ```yaml
 steps:
   - command: 'echo wow'
     plugins:
       - seek-oss/docker-ecr-cache#v1.7.0:
-          max-age-days: 7
-      - docker#v3.3.0
+          export-env-variable: BUILDKITE_PLUGIN_MY_CUSTOM_PLUGIN_CACHE_IMAGE
+      - my-custom-plugin#v1.0.0:
 ```
 
-### Changing the name of exported variable 
+### AWS ECR specific options
 
-By default image name and computed tag are exported to the Docker buildkite plugin env variable `BUILDKITE_PLUGIN_DOCKER_IMAGE`. In order to chain the plugin with a different plugin, this can be changed by specifying a `export-env-variable` parameter:
+#### Specifying an ECR repository name
+
+The plugin pushes and pulls Docker images to and from an ECR repository named
+`build-cache/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}`. You can
+optionally use a custom repository name:
 
 ```yaml
 steps:
   - command: 'echo wow'
     plugins:
       - seek-oss/docker-ecr-cache#v1.7.0:
-          export-env-variable: BUILDKITE_PLUGIN_MY_CUSTOM_PLUGIN_CACHE_IMAGE
-      - my-custom-plugin#v1.0.0
- ```
+          ecr-name: my-unique-repository-name
+          ecr-tags:
+            Key: Value
+            Key2: Value2
+      - docker#v3.3.0
+```
+
+### GCP GCR specific configuration
+
+[Overview of Google Container Registry](https://cloud.google.com/container-registry/docs/overview)
+
+Example:
+
+```yaml
+  - command: 'echo wow'
+    plugins:
+      - seek-oss/docker-ecr-cache#v1.7.0:
+          registry-provider: gcr
+          gcp-project: foo-bar-123456
+```
+
+#### Required GCR configuration
+
+- `registry-provider`: this must be `gcr` to aim at a google container registry.
+- `gcp-project`: this must be supplied. It is the GCP project your GCR is set up inside.
+
+#### Optional GCR configuration
+
+- `registry-hostname` (default: `gcr.io`). The location your image will be stored. [See upstream docs](https://cloud.google.com/container-registry/docs/overview#registry_name) for options.
 
 ## Design
 
@@ -218,7 +243,7 @@ automatically applied to expire images after 30 days (configurable via `max-age-
 
 To run the tests of this plugin, run
 
-```
+```bash
 docker-compose run --rm tests
 ```
 

changelog.md

@@ -0,0 +1,102 @@
+login() {
+  $(aws ecr get-login --no-include-email)
+}
+
+get_registry_url() {
+  local repository_name
+  repository_name="$(get_ecr_repository_name)"
+  aws ecr describe-repositories \
+    --repository-names "${repository_name}" \
+    --output text \
+    --query 'repositories[0].repositoryUri'
+}
+
+ecr_exists() {
+  local repository_name="${1}"
+  aws ecr describe-repositories \
+    --repository-names "${repository_name}" \
+    --output text \
+    --query 'repositories[0].registryId'
+}
+
+get_ecr_arn() {
+  local repository_name="${1}"
+  aws ecr describe-repositories \
+    --repository-names "${repository_name}" \
+    --output text \
+    --query 'repositories[0].repositoryArn'
+}
+
+get_ecr_tags() {
+local result=$(cat <<EOF
+{
+    "tags": []
+}
+EOF
+)
+  while IFS='=' read -r name _ ; do
+    if [[ $name =~ ^(BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_TAGS_) ]] ; then
+      # Handle plain key=value, e.g
+      # ecr-tags:
+      #   KEY_NAME: 'key-value'
+      key_name=$(echo "${name}" | sed 's/^BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_TAGS_//')
+      key_value=$(env | grep "$name" | sed "s/^$name=//")
+      result=$(echo $result | jq ".tags[.tags| length] |= . + {\"Key\": \"${key_name}\", \"Value\": \"${key_value}\"}")
+    fi
+  done < <(env | sort)
+
+  echo $result
+}
+
+get_ecr_repository_name() {
+  echo "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_NAME:-"$(get_default_image_name)"}"
+}
+
+configure_registry_for_image_if_necessary() {
+  local repository_name
+  repository_name="$(get_ecr_repository_name)"
+  local max_age_days="${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_MAX_AGE_DAYS:-30}"
+  local ecr_tags="$(get_ecr_tags)"
+  local num_tags=$(echo $ecr_tags | jq '.tags | length')
+
+  if ! ecr_exists "${repository_name}"; then
+    aws ecr create-repository --repository-name "${repository_name}" --cli-input-json "${ecr_tags}"
+  else
+    if [ "$num_tags" -gt 0 ]; then
+      local ecr_arn=$(get_ecr_arn "${repository_name}")
+      aws ecr tag-resource --resource-arn ${ecr_arn} --cli-input-json "${ecr_tags}"
+    fi
+  fi
+
+  # As of May 2019 ECR lifecycle policies can only have one rule that targets "any"
+  # Due to this limitation, only the max_age policy is applied
+  policy_text=$(cat <<EOF
+{
+  "rules": [
+    {
+      "rulePriority": 1,
+      "description": "Expire images older than ${max_age_days} days",
+      "selection": {
+        "tagStatus": "any",
+        "countType": "sinceImagePushed",
+        "countUnit": "days",
+        "countNumber": ${max_age_days}
+      },
+      "action": {
+        "type": "expire"
+      }
+    }
+  ]
+}
+EOF
+)
+
+  # Always set the lifecycle policy to update repositories automatically
+  # created before PR #9.
+  #
+  # When using a custom repository with a restricted Buildkite role this might
+  # not succeed. Ignore the error and let the build continue.
+  aws ecr put-lifecycle-policy \
+  --repository-name "${repository_name}" \
+  --lifecycle-policy-text "${policy_text}" || true
+}

hooks/lib/ecr-registry-provider.bash

@@ -0,0 +1,20 @@
+login() {
+  # Currently assume the use of docker-credential-gcr to manage AuthN transparently.
+  echo "Plugin currently assumes that docker-credential-gcr is on PATH and configured. See https://github.com/GoogleCloudPlatform/docker-credential-gcr#configuration-and-usage if later docker pull/push fail."
+}
+
+configure_registry_for_image_if_necessary() {
+  # GCR does not have a concept of a repository for images within a registry like ECR does.
+  echo ""
+}
+
+get_registry_url() {
+  if [[ -z "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_GCP_PROJECT:-}" ]]; then
+    log_fatal "gcp-project in plugin settings must have a value." 34
+  fi
+  if [[ -z "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_HOSTNAME:-}" ]]; then
+    echoerr "registry-hostname had no value, defaulting to gcr.io"
+    BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_HOSTNAME="gcr.io"
+  fi
+  echo "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_HOSTNAME}/${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_GCP_PROJECT}/${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_NAME:-"$(get_default_image_name)"}"
+}

hooks/lib/gcr-registry-provider.bash

@@ -0,0 +1,88 @@
+echoerr() {
+  echo "$@" 1>&2;
+}
+
+log_fatal() {
+  echoerr "In $(pwd)"
+  echoerr "${@}"
+  # use the last argument as the exit code
+  exit_code="${*: -1}"
+  if [[ "${exit_code}" =~ ^[0-9]+$ ]]; then
+    exit "${exit_code}"
+  fi
+  exit 1
+}
+
+read_build_args() {
+  read_list_property 'BUILD_ARGS'
+  for arg in ${result[@]+"${result[@]}"}; do
+    build_args+=("--build-arg=${arg}")
+  done
+}
+
+# read a plugin property of type [array, string] into a Bash array. Buildkite
+# exposes a string value at BUILDKITE_PLUGIN_{NAME}_{KEY}, and array values at
+# BUILDKITE_PLUGIN_{NAME}_{KEY}_{IDX}.
+read_list_property() {
+  local base_name="BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_${1}"
+
+  result=()
+
+  if [[ -n ${!base_name:-} ]]; then
+    result+=("${!base_name}")
+  fi
+
+  while IFS='=' read -r item_name _; do
+    if [[ ${item_name} =~ ^(${base_name}_[0-9]+) ]]; then
+      result+=("${!item_name}")
+    fi
+  done < <(env | sort)
+}
+
+get_default_image_name() {
+  echo "build-cache/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}"
+}
+
+compute_tag() {
+  local docker_file="$1"
+  local sums
+
+  echoerr '--- Computing tag'
+
+  echoerr 'DOCKERFILE'
+  echoerr "+ ${docker_file}:${target:-"<target>"}"
+  sums="$(sha1sum "${docker_file}")"
+  sums+="$(echo "${target}" | sha1sum)"
+
+  echoerr 'ARCHITECTURE'
+  echoerr "+ $(uname -m)"
+  sums+="$(uname -m | sha1sum)"
+
+  echoerr 'BUILD_ARGS'
+  read_list_property 'BUILD_ARGS'
+  for arg in ${result[@]+"${result[@]}"}; do
+    echoerr "+ ${arg}"
+
+    # include underlying environment variable after echo
+    if [[ ${arg} != *=* ]]; then
+      arg+="=${!arg:-}"
+    fi
+
+    sums+="$(echo "${arg}" | sha1sum)"
+  done
+
+  # expand ** in cache-on properties
+  shopt -s globstar
+
+  echoerr 'CACHE_ON'
+  read_list_property 'CACHE_ON'
+  for glob in ${result[@]+"${result[@]}"}; do
+    echoerr "${glob}"
+    for file in ${glob}; do
+      echoerr "+ ${file}"
+      sums+="$(sha1sum "${file}")"
+    done
+  done
+
+  echo "${sums}" | sha1sum | cut -c-7
+}

hooks/lib/stdlib.bash

@@ -0,0 +1,17 @@
+login() {
+  echo "stubbed login"
+}
+
+configure_registry_for_image_if_necessary() {
+  echo "stubbed configure_registry_for_image_if_necessary"
+}
+
+get_registry_url() {
+  echo "pretend.host/path/segment/image"
+}
+
+# BATS/bats-mock does not allow stubbing a function, currently.
+# So, override it to avoid needing to repeat all the stub'd sha1sum etc inside the end-to-end tests.
+compute_tag() {
+  echo "stubbed-computed-tag"
+}