diff --git a/.kokoro/system.sh b/.kokoro/system.sh index e0c7e71c1ad7..9bea38844513 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -35,10 +35,24 @@ RETVAL=0 pwd +echo "=== KOKORO VM SCOUTING ===" +echo "CPU Count:" +nproc +echo "--------------------------" +echo "Detailed CPU Info:" +lscpu | grep -E "^(Model name|CPU\(s\)|Thread\(s\) per core|Core\(s\) per socket)" +echo "--------------------------" +echo "Memory Status:" +free -h +echo "--------------------------" +echo "Disk Usage:" +df -h / +echo "==========================" + run_package_test() { local package_name=$1 local package_path="packages/${package_name}" - + # Declare local overrides to prevent bleeding into the next loop iteration local PROJECT_ID local GOOGLE_APPLICATION_CREDENTIALS @@ -46,6 +60,14 @@ run_package_test() { # Inherit NOX_SESSION from environment to allow configs (like prerelease.cfg) to pass it in local NOX_SESSION="${NOX_SESSION}" + # ISOLATION: Create a unique gcloud config dir for this run + local gcloud_config_dir=$(mktemp -d -t "gcloud-config-${package_name}-XXXXXX") + local CLOUDSDK_CONFIG="${gcloud_config_dir}" + + # 🪤 TRAP: Ensure cleanup of THIS specific temp dir on exit of this subshell + trap 'rm -rf "$gcloud_config_dir"' EXIT + + echo "------------------------------------------------------------" echo "Configuring environment for: ${package_name}" echo "------------------------------------------------------------" @@ -80,11 +102,13 @@ run_package_test() { esac # Export variables for the duration of this function's sub-processes - export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION + export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION CLOUDSDK_CONFIG export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}" - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" - gcloud config set project "$PROJECT_ID" + # 🛡️ Explicit check: Fail early if auth fails + gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || return 1 + export CLOUDSDK_CORE_PROJECT="${PROJECT_ID}" + # Run the actual test pushd "${package_path}" > /dev/null @@ -93,13 +117,62 @@ run_package_test() { local res=$? set -e popd > /dev/null - + return $res } +reap_parallel_results() { + local retval=0 + local failed_count=0 + local passed_count=0 + + if [ -z "$LOG_DIR" ]; then + echo "Error: LOG_DIR is not set." + return 1 + fi + + for failed in "$LOG_DIR"/*.failed; do + if [ -f "$failed" ]; then + failed_count=$((failed_count + 1)) + fi + done + + local total_tested=${#PACKAGES_TO_TEST[@]} + passed_count=$((total_tested - failed_count)) + + echo "" + echo "==================================================" + echo " TEST RUN SUMMARY " + echo "==================================================" + echo "Total tested: $total_tested" + echo "Passed: $passed_count" + echo "Failed: $failed_count" + echo "==================================================" + + if [ "$failed_count" -gt 0 ]; then + echo "" + echo "!!! DETAILED LOGS FOR FAILED PACKAGES !!!" + for failed in "$LOG_DIR"/*.failed; do + if [ -f "$failed" ]; then + local pkg=$(basename "$failed" .failed) + echo "--------------------------------------------------" + echo "LOGS FOR: $pkg" + echo "--------------------------------------------------" + cat "$LOG_DIR/$pkg.log" + echo "" + fi + done + retval=1 + fi + return $retval +} + + # A file for running system tests system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh" +PACKAGES_TO_TEST=() + # Run system tests for each package with directory packages/*/tests/system for path in `find 'packages' \ \( -type d -wholename 'packages/*/tests/system' \) -o \ @@ -113,6 +186,7 @@ for path in `find 'packages' \ package_name=${package_name%%/*} package_path="packages/${package_name}" + # Determine if we should skip based on git diff # We always check for changes in these specific versioning/config files files_to_check=( @@ -147,10 +221,34 @@ for path in `find 'packages' \ set -e if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then - # Call the function - its internal exports won't affect the next loop - run_package_test "$package_name" || RETVAL=$? + PACKAGES_TO_TEST+=("$package_name") else echo "No changes in ${package_name} and not a continuous build, skipping." fi done -exit ${RETVAL} + +# Parallel Execution Logic +MAX_JOBS=${MAX_JOBS:-4} + +# Temporary directory for clean log segregation +LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) +# Clean up logs on exit +trap 'rm -rf "$LOG_DIR"' EXIT + +if [ ${#PACKAGES_TO_TEST[@]} -eq 0 ]; then + echo "No packages to test." + exit 0 +fi + +echo "==================================================" +echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages" +echo "Concurrency limit: ${MAX_JOBS}" +echo "==================================================" + +export LOG_DIR +export -f run_package_test +export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR + +printf '%s\n' "${PACKAGES_TO_TEST[@]}" | xargs -P "$MAX_JOBS" -I {} bash -c 'run_package_test "{}" > "$LOG_DIR/{}.log" 2>&1 || touch "$LOG_DIR/{}.failed"' + +reap_parallel_results || RETVAL=1 diff --git a/packages/django-google-spanner/setup.py b/packages/django-google-spanner/setup.py index 973dc622047e..079139850bac 100644 --- a/packages/django-google-spanner/setup.py +++ b/packages/django-google-spanner/setup.py @@ -1,4 +1,6 @@ # Copyright 2020 Google LLC +# Trigger parallel system test run + # # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file or at diff --git a/packages/google-api-core/setup.py b/packages/google-api-core/setup.py index 168877fa5bbe..1ce20aa0c671 100644 --- a/packages/google-api-core/setup.py +++ b/packages/google-api-core/setup.py @@ -1,4 +1,6 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-auth/setup.py b/packages/google-auth/setup.py index cf3148130d6e..ccc5b846a095 100644 --- a/packages/google-auth/setup.py +++ b/packages/google-auth/setup.py @@ -1,4 +1,5 @@ # Copyright 2014 Google Inc. +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,8 +16,7 @@ import io import os -from setuptools import find_namespace_packages -from setuptools import setup +from setuptools import find_namespace_packages, setup cryptography_base_require = [ "cryptography >= 38.0.3", diff --git a/packages/google-cloud-bigquery/setup.py b/packages/google-cloud-bigquery/setup.py index 13aa8b6ca346..3035f8492d4d 100644 --- a/packages/google-cloud-bigquery/setup.py +++ b/packages/google-cloud-bigquery/setup.py @@ -1,4 +1,5 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-bigtable/setup.py b/packages/google-cloud-bigtable/setup.py index cc161af33de1..7e9cf84c0350 100644 --- a/packages/google-cloud-bigtable/setup.py +++ b/packages/google-cloud-bigtable/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-core/setup.py b/packages/google-cloud-core/setup.py index 360fdf4cccb5..3bf0191f2a55 100644 --- a/packages/google-cloud-core/setup.py +++ b/packages/google-cloud-core/setup.py @@ -1,4 +1,5 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +18,6 @@ import setuptools - # Package metadata. name = "google-cloud-core" diff --git a/packages/google-cloud-datastore/setup.py b/packages/google-cloud-datastore/setup.py index 26f21974303e..614353115b77 100644 --- a/packages/google-cloud-datastore/setup.py +++ b/packages/google-cloud-datastore/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-error-reporting/setup.py b/packages/google-cloud-error-reporting/setup.py index 42eeba4d2451..c152c9a73db2 100644 --- a/packages/google-cloud-error-reporting/setup.py +++ b/packages/google-cloud-error-reporting/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-firestore/setup.py b/packages/google-cloud-firestore/setup.py index 8dad33914a53..27c2ee3839a3 100644 --- a/packages/google-cloud-firestore/setup.py +++ b/packages/google-cloud-firestore/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-logging/setup.py b/packages/google-cloud-logging/setup.py index 390fb322c872..bc1dc07e67db 100644 --- a/packages/google-cloud-logging/setup.py +++ b/packages/google-cloud-logging/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-ndb/setup.py b/packages/google-cloud-ndb/setup.py index 8c22f5349b16..98573a0580ee 100644 --- a/packages/google-cloud-ndb/setup.py +++ b/packages/google-cloud-ndb/setup.py @@ -1,4 +1,6 @@ # Copyright 2018 Google LLC +# Trigger parallel system test run + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-pubsub/setup.py b/packages/google-cloud-pubsub/setup.py index 66c9606ffc26..d841160d2a25 100644 --- a/packages/google-cloud-pubsub/setup.py +++ b/packages/google-cloud-pubsub/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/google-cloud-storage/setup.py b/packages/google-cloud-storage/setup.py index b872eff7dbd8..41a00895d1bd 100644 --- a/packages/google-cloud-storage/setup.py +++ b/packages/google-cloud-storage/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2026 Google LLC +# Trigger parallel system test run # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.