From 692e1ff8a385b7d586583dc7b6ce002e9e26d070 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 1 Jul 2026 13:46:12 -0400 Subject: [PATCH 1/9] feat: enable parallel system tests in Kokoro and trigger 4 packages --- .kokoro/system.sh | 115 +++++++++++++++++++++++- packages/django-google-spanner/setup.py | 2 + packages/google-api-core/setup.py | 2 + packages/google-cloud-ndb/setup.py | 2 + packages/sqlalchemy-bigquery/setup.py | 2 + 5 files changed, 120 insertions(+), 3 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index e0c7e71c1ad7..d57dfbf899b4 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -46,6 +46,10 @@ run_package_test() { # Inherit NOX_SESSION from environment to allow configs (like prerelease.cfg) to pass it in local NOX_SESSION="${NOX_SESSION}" + # ISOLATION: Create a unique gcloud config dir for this run + local gcloud_config_dir=$(mktemp -d -t "gcloud-config-${package_name}-XXXXXX") + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + echo "------------------------------------------------------------" echo "Configuring environment for: ${package_name}" echo "------------------------------------------------------------" @@ -94,12 +98,15 @@ run_package_test() { set -e popd > /dev/null + rm -rf "${gcloud_config_dir}" return $res } # A file for running system tests system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh" +PACKAGES_TO_TEST=() + # Run system tests for each package with directory packages/*/tests/system for path in `find 'packages' \ \( -type d -wholename 'packages/*/tests/system' \) -o \ @@ -147,10 +154,112 @@ for path in `find 'packages' \ set -e if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - # Call the function - its internal exports won't affect the next loop - run_package_test "$package_name" || RETVAL=$? + PACKAGES_TO_TEST+=("$package_name") else echo "No changes in ${package_name} and not a continuous build, skipping." fi done -exit ${RETVAL} + +# Parallel Execution Logic +MAX_JOBS=${MAX_JOBS:-4} +active_jobs=0 +declare -A job_pids +declare -A job_pkgs +failed_packages=() +passed_packages=() + +# Temporary directory for clean log segregation +LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) +# Clean up logs on exit +trap 'rm -rf "$LOG_DIR"' EXIT + +if [ ${#PACKAGES_TO_TEST[@]} -eq 0 ]; then + echo "No packages to test." + exit 0 +fi + +echo "==================================================" +echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages" +echo "Concurrency limit: ${MAX_JOBS}" +echo "==================================================" + +for pkg in "${PACKAGES_TO_TEST[@]}"; do + # Maintain concurrency limit + while [ "$active_jobs" -ge "$MAX_JOBS" ]; do + for pid in "${!job_pids[@]}"; do + if ! kill -0 "$pid" 2>/dev/null; then + wait "$pid" && status=0 || status=$? + finished_pkg=${job_pkgs[$pid]} + if [ "$status" -eq 0 ]; then + echo "✔ [SUCCESS] ${finished_pkg}" + passed_packages+=("$finished_pkg") + else + echo "✘ [FAILURE] ${finished_pkg} (Exit Code: ${status})" + failed_packages+=("$finished_pkg") + fi + unset "job_pids[$pid]" + unset "job_pkgs[$pid]" + active_jobs=$((active_jobs - 1)) + fi + done + sleep 0.1 + done + + safe_pkg_name=$(echo "$pkg" | tr '/' '_') + log_file="${LOG_DIR}/${safe_pkg_name}.log" + + echo "Spawning tests for ${pkg}..." + run_package_test "$pkg" > "$log_file" 2>&1 & + pid=$! + job_pids["$pid"]=$pid + job_pkgs["$pid"]=$pkg + active_jobs=$((active_jobs + 1)) +done + +# Reap remaining processes +while [ "$active_jobs" -gt 0 ]; do + for pid in "${!job_pids[@]}"; do + if ! kill -0 "$pid" 2>/dev/null; then + wait "$pid" && status=0 || status=$? + finished_pkg=${job_pkgs[$pid]} + if [ "$status" -eq 0 ]; then + echo "✔ [SUCCESS] ${finished_pkg}" + passed_packages+=("$finished_pkg") + else + echo "✘ [FAILURE] ${finished_pkg} (Exit Code: ${status})" + failed_packages+=("$finished_pkg") + fi + unset "job_pids[$pid]" + unset "job_pkgs[$pid]" + active_jobs=$((active_jobs - 1)) + fi + done + sleep 0.1 +done + +echo "" +echo "==================================================" +echo " TEST RUN SUMMARY " +echo "==================================================" +echo "Total tested: ${#PACKAGES_TO_TEST[@]}" +echo "Passed: ${#passed_packages[@]}" +echo "Failed: ${#failed_packages[@]}" +echo "==================================================" + +if [ ${#failed_packages[@]} -gt 0 ]; then + echo "" + echo "!!! DETAILED LOGS FOR FAILED PACKAGES !!!" + for pkg in "${failed_packages[@]}"; do + safe_pkg_name=$(echo "$pkg" | tr '/' '_') + log_file="${LOG_DIR}/${safe_pkg_name}.log" + echo "--------------------------------------------------" + echo "LOGS FOR: ${pkg}" + echo "--------------------------------------------------" + [ -f "$log_file" ] && cat "$log_file" + echo "" + done + exit 1 +fi + +echo "All tests passed successfully!" +exit 0 diff --git a/packages/django-google-spanner/setup.py b/packages/django-google-spanner/setup.py index 973dc622047e..079139850bac 100644 --- a/packages/django-google-spanner/setup.py +++ b/packages/django-google-spanner/setup.py @@ -1,4 +1,6 @@ # Copyright 2020 Google LLC +# Trigger parallel system test run + # # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file or at diff --git a/packages/google-api-core/setup.py b/packages/google-api-core/setup.py index 168877fa5bbe..1ce20aa0c671 100644 --- a/packages/google-api-core/setup.py +++ b/packages/google-api-core/setup.py @@ -1,4 +1,6 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-ndb/setup.py b/packages/google-cloud-ndb/setup.py index 8c22f5349b16..98573a0580ee 100644 --- a/packages/google-cloud-ndb/setup.py +++ b/packages/google-cloud-ndb/setup.py @@ -1,4 +1,6 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/sqlalchemy-bigquery/setup.py b/packages/sqlalchemy-bigquery/setup.py index 8d7ad005c2ee..7fb81746ef52 100644 --- a/packages/sqlalchemy-bigquery/setup.py +++ b/packages/sqlalchemy-bigquery/setup.py @@ -1,5 +1,7 @@ #!/usr/bin/env python # Copyright (c) 2017 The sqlalchemy-bigquery Authors +# Trigger parallel system test run + # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in From 461580efcb45524f4e11ec1e1c925f70902be8f9 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Wed, 1 Jul 2026 13:46:34 -0400 Subject: [PATCH 2/9] refactor: refine gcloud isolation with local variable --- .kokoro/system.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index d57dfbf899b4..95f5303ce0a1 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -48,7 +48,7 @@ run_package_test() { # ISOLATION: Create a unique gcloud config dir for this run local gcloud_config_dir=$(mktemp -d -t "gcloud-config-${package_name}-XXXXXX") - export CLOUDSDK_CONFIG="${gcloud_config_dir}" + local CLOUDSDK_CONFIG="${gcloud_config_dir}" echo "------------------------------------------------------------" echo "Configuring environment for: ${package_name}" @@ -84,7 +84,7 @@ run_package_test() { esac # Export variables for the duration of this function's sub-processes - export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION + export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION CLOUDSDK_CONFIG export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" From b8795e4f35a462e4eae5a9b10ec4cc53a4d44a70 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 7 Jul 2026 05:44:12 -0400 Subject: [PATCH 3/9] chore: add VM diagnostics and temporarily skip sqlalchemy-bigquery --- .kokoro/system.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 95f5303ce0a1..ccdfcae9a93c 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -35,6 +35,20 @@ RETVAL=0 pwd +echo "=== KOKORO VM SCOUTING ===" +echo "CPU Count:" +nproc +echo "--------------------------" +echo "Detailed CPU Info:" +lscpu | grep -E "^(Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per socket)" +echo "--------------------------" +echo "Memory Status:" +free -h +echo "--------------------------" +echo "Disk Usage:" +df -h / +echo "==========================" + run_package_test() { local package_name=$1 local package_path="packages/${package_name}" @@ -120,6 +134,11 @@ for path in `find 'packages' \ package_name=${package_name%%/*} package_path="packages/${package_name}" + if [[ "$package_name" == "sqlalchemy-bigquery" ]]; then + echo "Skipping sqlalchemy-bigquery for diagnostics." + continue + fi + # Determine if we should skip based on git diff # We always check for changes in these specific versioning/config files files_to_check=( From 54f8a72aa5a3e715228cd9ad9f52e9d8bf4a8949 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 7 Jul 2026 11:23:55 -0400 Subject: [PATCH 4/9] chore: revert sqlalchemy-bigquery trigger --- packages/sqlalchemy-bigquery/setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/sqlalchemy-bigquery/setup.py b/packages/sqlalchemy-bigquery/setup.py index 7fb81746ef52..8d7ad005c2ee 100644 --- a/packages/sqlalchemy-bigquery/setup.py +++ b/packages/sqlalchemy-bigquery/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # Copyright (c) 2017 The sqlalchemy-bigquery Authors -# Trigger parallel system test run - # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in From 4206da25cd757d2f83343d64f4c5c92213bc0d10 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Tue, 7 Jul 2026 11:24:21 -0400 Subject: [PATCH 5/9] chore: add 10 more packages to parallel test trigger --- packages/google-auth/setup.py | 4 ++-- packages/google-cloud-bigquery/setup.py | 1 + packages/google-cloud-bigtable/setup.py | 1 + packages/google-cloud-core/setup.py | 2 +- packages/google-cloud-datastore/setup.py | 1 + packages/google-cloud-error-reporting/setup.py | 1 + packages/google-cloud-firestore/setup.py | 1 + packages/google-cloud-logging/setup.py | 1 + packages/google-cloud-pubsub/setup.py | 1 + packages/google-cloud-storage/setup.py | 1 + 10 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/google-auth/setup.py b/packages/google-auth/setup.py index cf3148130d6e..ccc5b846a095 100644 --- a/packages/google-auth/setup.py +++ b/packages/google-auth/setup.py @@ -1,4 +1,5 @@ # Copyright 2014 Google Inc. +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +16,7 @@ import io import os -from setuptools import find_namespace_packages -from setuptools import setup +from setuptools import find_namespace_packages, setup cryptography_base_require = [ "cryptography >= 38.0.3", diff --git a/packages/google-cloud-bigquery/setup.py b/packages/google-cloud-bigquery/setup.py index 13aa8b6ca346..3035f8492d4d 100644 --- a/packages/google-cloud-bigquery/setup.py +++ b/packages/google-cloud-bigquery/setup.py @@ -1,4 +1,5 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-bigtable/setup.py b/packages/google-cloud-bigtable/setup.py index cc161af33de1..7e9cf84c0350 100644 --- a/packages/google-cloud-bigtable/setup.py +++ b/packages/google-cloud-bigtable/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-core/setup.py b/packages/google-cloud-core/setup.py index 360fdf4cccb5..3bf0191f2a55 100644 --- a/packages/google-cloud-core/setup.py +++ b/packages/google-cloud-core/setup.py @@ -1,4 +1,5 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +18,6 @@ import setuptools - # Package metadata. name = "google-cloud-core" diff --git a/packages/google-cloud-datastore/setup.py b/packages/google-cloud-datastore/setup.py index 26f21974303e..614353115b77 100644 --- a/packages/google-cloud-datastore/setup.py +++ b/packages/google-cloud-datastore/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-error-reporting/setup.py b/packages/google-cloud-error-reporting/setup.py index 42eeba4d2451..c152c9a73db2 100644 --- a/packages/google-cloud-error-reporting/setup.py +++ b/packages/google-cloud-error-reporting/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-firestore/setup.py b/packages/google-cloud-firestore/setup.py index 8dad33914a53..27c2ee3839a3 100644 --- a/packages/google-cloud-firestore/setup.py +++ b/packages/google-cloud-firestore/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-logging/setup.py b/packages/google-cloud-logging/setup.py index 390fb322c872..bc1dc07e67db 100644 --- a/packages/google-cloud-logging/setup.py +++ b/packages/google-cloud-logging/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-pubsub/setup.py b/packages/google-cloud-pubsub/setup.py index 66c9606ffc26..d841160d2a25 100644 --- a/packages/google-cloud-pubsub/setup.py +++ b/packages/google-cloud-pubsub/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-storage/setup.py b/packages/google-cloud-storage/setup.py index b872eff7dbd8..41a00895d1bd 100644 --- a/packages/google-cloud-storage/setup.py +++ b/packages/google-cloud-storage/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 2363d0595941d36f497f8270d3966dbb8b30d17f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 16 Jul 2026 13:25:35 -0400 Subject: [PATCH 6/9] refactor(ci): use xargs for parallel execution and add reaping function --- .kokoro/system.sh | 139 ++++++++++++++++++---------------------------- 1 file changed, 54 insertions(+), 85 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index ccdfcae9a93c..5798713dd8f4 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -52,7 +52,7 @@ echo "==========================" run_package_test() { local package_name=$1 local package_path="packages/${package_name}" - + # Declare local overrides to prevent bleeding into the next loop iteration local PROJECT_ID local GOOGLE_APPLICATION_CREDENTIALS @@ -111,11 +111,58 @@ run_package_test() { local res=$? set -e popd > /dev/null - + rm -rf "${gcloud_config_dir}" return $res } +reap_parallel_results() { + local retval=0 + local failed_count=0 + local passed_count=0 + + if [ -z "$LOG_DIR" ]; then + echo "Error: LOG_DIR is not set." + return 1 + fi + + for failed in "$LOG_DIR"/*.failed; do + if [ -f "$failed" ]; then + failed_count=$((failed_count + 1)) + fi + done + + local total_tested=${#PACKAGES_TO_TEST[@]} + passed_count=$((total_tested - failed_count)) + + echo "" + echo "==================================================" + echo " TEST RUN SUMMARY " + echo "==================================================" + echo "Total tested: $total_tested" + echo "Passed: $passed_count" + echo "Failed: $failed_count" + echo "==================================================" + + if [ "$failed_count" -gt 0 ]; then + echo "" + echo "!!! DETAILED LOGS FOR FAILED PACKAGES !!!" + for failed in "$LOG_DIR"/*.failed; do + if [ -f "$failed" ]; then + local pkg=$(basename "$failed" .failed) + echo "--------------------------------------------------" + echo "LOGS FOR: $pkg" + echo "--------------------------------------------------" + cat "$LOG_DIR/$pkg.log" + echo "" + fi + done + retval=1 + fi + return $retval +} + + # A file for running system tests system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh" @@ -181,11 +228,6 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} -active_jobs=0 -declare -A job_pids -declare -A job_pkgs -failed_packages=() -passed_packages=() # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) @@ -202,83 +244,10 @@ echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages" echo "Concurrency limit: ${MAX_JOBS}" echo "==================================================" -for pkg in "${PACKAGES_TO_TEST[@]}"; do - # Maintain concurrency limit - while [ "$active_jobs" -ge "$MAX_JOBS" ]; do - for pid in "${!job_pids[@]}"; do - if ! kill -0 "$pid" 2>/dev/null; then - wait "$pid" && status=0 || status=$? - finished_pkg=${job_pkgs[$pid]} - if [ "$status" -eq 0 ]; then - echo "✔ [SUCCESS] ${finished_pkg}" - passed_packages+=("$finished_pkg") - else - echo "✘ [FAILURE] ${finished_pkg} (Exit Code: ${status})" - failed_packages+=("$finished_pkg") - fi - unset "job_pids[$pid]" - unset "job_pkgs[$pid]" - active_jobs=$((active_jobs - 1)) - fi - done - sleep 0.1 - done - - safe_pkg_name=$(echo "$pkg" | tr '/' '_') - log_file="${LOG_DIR}/${safe_pkg_name}.log" - - echo "Spawning tests for ${pkg}..." - run_package_test "$pkg" > "$log_file" 2>&1 & - pid=$! - job_pids["$pid"]=$pid - job_pkgs["$pid"]=$pkg - active_jobs=$((active_jobs + 1)) -done - -# Reap remaining processes -while [ "$active_jobs" -gt 0 ]; do - for pid in "${!job_pids[@]}"; do - if ! kill -0 "$pid" 2>/dev/null; then - wait "$pid" && status=0 || status=$? - finished_pkg=${job_pkgs[$pid]} - if [ "$status" -eq 0 ]; then - echo "✔ [SUCCESS] ${finished_pkg}" - passed_packages+=("$finished_pkg") - else - echo "✘ [FAILURE] ${finished_pkg} (Exit Code: ${status})" - failed_packages+=("$finished_pkg") - fi - unset "job_pids[$pid]" - unset "job_pkgs[$pid]" - active_jobs=$((active_jobs - 1)) - fi - done - sleep 0.1 -done - -echo "" -echo "==================================================" -echo " TEST RUN SUMMARY " -echo "==================================================" -echo "Total tested: ${#PACKAGES_TO_TEST[@]}" -echo "Passed: ${#passed_packages[@]}" -echo "Failed: ${#failed_packages[@]}" -echo "==================================================" +export LOG_DIR +export -f run_package_test +export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR -if [ ${#failed_packages[@]} -gt 0 ]; then - echo "" - echo "!!! DETAILED LOGS FOR FAILED PACKAGES !!!" - for pkg in "${failed_packages[@]}"; do - safe_pkg_name=$(echo "$pkg" | tr '/' '_') - log_file="${LOG_DIR}/${safe_pkg_name}.log" - echo "--------------------------------------------------" - echo "LOGS FOR: ${pkg}" - echo "--------------------------------------------------" - [ -f "$log_file" ] && cat "$log_file" - echo "" - done - exit 1 -fi +printf '%s\n' "${PACKAGES_TO_TEST[@]}" | xargs -P "$MAX_JOBS" -I {} bash -c 'run_package_test "{}" > "$LOG_DIR/{}.log" 2>&1 || touch "$LOG_DIR/{}.failed"' -echo "All tests passed successfully!" -exit 0 +reap_parallel_results || RETVAL=1 From c7a9d9910c24f7be856de11aedc460088a548130 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 16 Jul 2026 13:40:02 -0400 Subject: [PATCH 7/9] refactor(ci): improve reliability with trap EXIT and explicit error handling --- .kokoro/system.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 5798713dd8f4..358b0f4179fc 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -64,6 +64,10 @@ run_package_test() { local gcloud_config_dir=$(mktemp -d -t "gcloud-config-${package_name}-XXXXXX") local CLOUDSDK_CONFIG="${gcloud_config_dir}" + # 🪤 TRAP: Ensure cleanup of THIS specific temp dir on exit of this subshell + trap 'rm -rf "$gcloud_config_dir"' EXIT + + echo "------------------------------------------------------------" echo "Configuring environment for: ${package_name}" echo "------------------------------------------------------------" @@ -101,8 +105,10 @@ run_package_test() { export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION CLOUDSDK_CONFIG export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" - gcloud config set project "$PROJECT_ID" + # 🛡️ Explicit check: Fail early if auth fails + gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || return 1 + export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}" + # Run the actual test pushd "${package_path}" > /dev/null @@ -112,7 +118,6 @@ run_package_test() { set -e popd > /dev/null - rm -rf "${gcloud_config_dir}" return $res } From 0e2a314063f0a8c08d87e7a9763208fe8613646d Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 16 Jul 2026 14:44:06 -0400 Subject: [PATCH 8/9] chore(ci): remove temporary skip for sqlalchemy-bigquery --- .kokoro/system.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 358b0f4179fc..9bea38844513 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -186,10 +186,6 @@ for path in `find 'packages' \ package_name=${package_name%%/*} package_path="packages/${package_name}" - if [[ "$package_name" == "sqlalchemy-bigquery" ]]; then - echo "Skipping sqlalchemy-bigquery for diagnostics." - continue - fi # Determine if we should skip based on git diff # We always check for changes in these specific versioning/config files From 5d1e078bb6091782975d4cd56c03fd2421d3e120 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Jul 2026 14:28:27 -0400 Subject: [PATCH 9/9] fix(ci): ensure script exits with failure code when tests fail --- .kokoro/system.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 9bea38844513..a1847410548a 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -252,3 +252,5 @@ export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR printf '%s\n' "${PACKAGES_TO_TEST[@]}" | xargs -P "$MAX_JOBS" -I {} bash -c 'run_package_test "{}" > "$LOG_DIR/{}.log" 2>&1 || touch "$LOG_DIR/{}.failed"' reap_parallel_results || RETVAL=1 + +exit ${RETVAL}