diff --git a/.github/actions/calculate-shard-matrix/action.yaml b/.github/actions/calculate-shard-matrix/action.yaml new file mode 100644 index 000000000000..660e725ee84f --- /dev/null +++ b/.github/actions/calculate-shard-matrix/action.yaml @@ -0,0 +1,22 @@ +name: "Calculate Shard Matrix" +description: "Calculates the dynamic test shard matrix and shard total" +outputs: + shard_matrix: + description: "JSON array of shard indices" + value: ${{ steps.set-matrix.outputs.shard_matrix }} + shard_total: + description: "Total number of shards" + value: ${{ steps.set-matrix.outputs.shard_total }} +runs: + using: "composite" + steps: + - name: Calculate Matrix + id: set-matrix + shell: bash + env: + DRY_RUN_SHARDS: true + BUILD_TYPE: presubmit + TEST_TYPE: units + GIT_DIFF_ARG: "HEAD^1" + run: | + bash ci/run_conditional_tests.sh --strict diff --git a/.github/actions/check-shard-status/action.yaml b/.github/actions/check-shard-status/action.yaml new file mode 100644 index 000000000000..37f57a2992ae --- /dev/null +++ b/.github/actions/check-shard-status/action.yaml @@ -0,0 +1,31 @@ +name: "Check Shard Status" +description: "Checks if all unit test shards passed for a given Node version" +inputs: + node-version: + required: true + description: "Node version to check" +runs: + using: "composite" + steps: + - name: Check shard status + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + }); + const nodeVersion = '${{ inputs.node-version }}'; + const shardJobs = jobs.filter(j => j.name.includes(`units (Node ${nodeVersion},`)); + if (shardJobs.length === 0) { + core.setFailed(`No shard jobs found for Node ${nodeVersion}`); + return; + } + const failed = shardJobs.filter(j => j.conclusion === 'failure' || j.conclusion === 'cancelled'); + if (failed.length > 0) { + const failedLinks = failed.map(j => `- ${j.name}: ${j.html_url}`).join('\n'); + core.setFailed(`${failed.length} shards failed for Node ${nodeVersion}:\n${failedLinks}`); + } else { + core.info(`All shards passed for Node ${nodeVersion}`); + } diff --git a/.github/actions/run-unit-tests/action.yaml b/.github/actions/run-unit-tests/action.yaml new file mode 100644 index 000000000000..69bf22693bdc --- /dev/null +++ b/.github/actions/run-unit-tests/action.yaml @@ -0,0 +1,33 @@ +name: "Run Unit Test Shard" +description: "Sets up Node.js, pnpm, and runs a unit test shard" +inputs: + node-version: + required: true + description: "Node.js version to test" + shard-total: + required: true + description: "Total number of shards" + shard-index: + required: true + description: "0-based shard index" +runs: + using: "composite" + steps: + - name: Use Node.js ${{ inputs.node-version }} + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: ${{ inputs.node-version }} + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: ^10.0.0 + - run: node --version + shell: bash + - name: Run unit tests + shell: bash + run: bash ci/run_conditional_tests.sh --strict + env: + BUILD_TYPE: presubmit + TEST_TYPE: units + SHARD_TOTAL: ${{ inputs.shard-total }} + SHARD_INDEX: ${{ inputs.shard-index }} + GIT_DIFF_ARG: HEAD^1 diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index c5de30eba54e..902233f76caa 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -5,29 +5,37 @@ on: pull_request: name: presubmit jobs: + setup: + runs-on: ubuntu-latest + outputs: + shard-matrix: ${{ steps.set-matrix.outputs.shard_matrix }} + shard-total: ${{ steps.set-matrix.outputs.shard_total }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + fetch-depth: 2 + persist-credentials: false + - id: set-matrix + uses: ./.github/actions/calculate-shard-matrix units: + needs: setup + name: units (Node ${{ matrix.node-version }}, Shard ${{ matrix.shard-index }}) runs-on: ubuntu-latest strategy: + fail-fast: false matrix: node-version: [22, 24, 26] + shard-index: ${{ fromJSON(needs.setup.outputs.shard-matrix) }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: - fetch-depth: 300 + fetch-depth: 2 persist-credentials: false - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - uses: ./.github/actions/run-unit-tests with: node-version: ${{ matrix.node-version }} - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - with: - version: ^10.0.0 - - run: node --version - - run: ci/run_conditional_tests.sh - name: Run unit tests - env: - BUILD_TYPE: presubmit - TEST_TYPE: units + shard-total: ${{ needs.setup.outputs.shard-total }} + shard-index: ${{ matrix.shard-index }} lint: runs-on: ubuntu-latest continue-on-error: true @@ -42,6 +50,25 @@ jobs: node-version: 24 - run: npm install - run: node ./bin/linter.mjs --strict - name: Run monorepo linter env: GIT_DIFF_ARG: "HEAD^1" + + # Dummy jobs to satisfy branch protection requirements for each node version + units-status: + name: units (${{ matrix.node-version }}) + needs: units + runs-on: ubuntu-latest + if: always() + strategy: + fail-fast: false + matrix: + node-version: [22, 24, 26] + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + - name: Check shard status + uses: ./.github/actions/check-shard-status + with: + node-version: ${{ matrix.node-version }} +>>>>>>> 657c33a5f3 (feat(ci): run unit-tests in parallel to speed them up) diff --git a/.github/workflows/windows-presubmit.yaml b/.github/workflows/windows-presubmit.yaml index 31105cfa46de..afd68aee344a 100644 --- a/.github/workflows/windows-presubmit.yaml +++ b/.github/workflows/windows-presubmit.yaml @@ -5,27 +5,53 @@ on: pull_request: name: presubmit-windows jobs: + setup: + runs-on: ubuntu-latest + outputs: + shard-matrix: ${{ steps.set-matrix.outputs.shard_matrix }} + shard-total: ${{ steps.set-matrix.outputs.shard_total }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + fetch-depth: 2 + persist-credentials: false + - id: set-matrix + uses: ./.github/actions/calculate-shard-matrix units: + needs: setup + name: units (Node ${{ matrix.node-version }}, Shard ${{ matrix.shard-index }}) runs-on: windows-latest strategy: + fail-fast: false matrix: node-version: [22, 24, 26] + shard-index: ${{ fromJSON(needs.setup.outputs.shard-matrix) }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: - fetch-depth: 300 + fetch-depth: 2 persist-credentials: false - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + - uses: ./.github/actions/run-unit-tests with: node-version: ${{ matrix.node-version }} - - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + shard-total: ${{ needs.setup.outputs.shard-total }} + shard-index: ${{ matrix.shard-index }} + + # Dummy jobs to satisfy branch protection requirements for each node version + units-status: + name: units (${{ matrix.node-version }}) + needs: units + runs-on: ubuntu-latest + if: always() + strategy: + fail-fast: false + matrix: + node-version: [22, 24, 26] + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: - version: ^10.0.0 - - run: node --version - - run: bash ci/run_conditional_tests.sh - name: Run windows unit tests - shell: bash - env: - BUILD_TYPE: presubmit - TEST_TYPE: units + persist-credentials: false + - name: Check shard status + uses: ./.github/actions/check-shard-status + with: + node-version: ${{ matrix.node-version }} diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh index cb6ae744a087..29f42bf888b2 100755 --- a/ci/run_conditional_tests.sh +++ b/ci/run_conditional_tests.sh @@ -36,25 +36,50 @@ if [[ "$(node -v)" == v22* ]]; then export NODE_OPTIONS="${NODE_OPTIONS} --no-experimental-require-module" fi -if [ ${BUILD_TYPE} == "presubmit" ]; then - # For presubmit build, we want to know the difference from the - # common commit in origin/main. - GIT_DIFF_ARG="origin/main..." +for arg in "$@"; do + case "${arg}" in + --strict) + STRICT=true + ;; + esac +done - # Then fetch enough history for finding the common commit. - git fetch origin main --deepen=300 +if [[ "${STRICT}" == "true" || "${STRICT}" == "1" ]]; then + if [ -z "${GIT_DIFF_ARG}" ]; then + echo "Error: STRICT mode requires GIT_DIFF_ARG to be set." >&2 + exit 1 + fi + set +e + git diff --quiet ${GIT_DIFF_ARG} + diff_status=$? + set -e + if [[ ${diff_status} -ne 0 && ${diff_status} -ne 1 ]]; then + echo "Error: STRICT mode git diff ${GIT_DIFF_ARG} failed with exit code ${diff_status}." >&2 + exit 1 + fi +else + if [ -z "${GIT_DIFF_ARG}" ]; then + if [ "${BUILD_TYPE}" == "presubmit" ]; then + # For presubmit build, we want to know the difference from the + # common commit in origin/main. + GIT_DIFF_ARG="origin/main..." -elif [ ${BUILD_TYPE} == "continuous" ]; then - # For continuous build, we want to know the difference in the last - # commit. This assumes we use squash commit when merging PRs. - GIT_DIFF_ARG="HEAD~.." + # Then fetch enough history for finding the common commit. + git fetch origin main --deepen=300 - # Then fetch one last commit for getting the diff. - git fetch origin main --deepen=1 + elif [ "${BUILD_TYPE}" == "continuous" ]; then + # For continuous build, we want to know the difference in the last + # commit. This assumes we use squash commit when merging PRs. + GIT_DIFF_ARG="HEAD~.." -else - # Run everything. - GIT_DIFF_ARG="" + # Then fetch one last commit for getting the diff. + git fetch origin main --deepen=1 + + else + # Run everything. + GIT_DIFF_ARG="" + fi + fi fi # Then detect changes in the test scripts. @@ -66,11 +91,10 @@ set -e if [[ "${changed}" -eq 0 ]]; then echo "no change detected in ci" else - echo "skipping trigger of tests for now: tracking in #7540" - # echo "change detected in ci, we should test everything" - # echo "result of git diff ${GIT_DIFF_ARG} ci:" - # git diff ${GIT_DIFF_ARG} ci - # GIT_DIFF_ARG="" + echo "change detected in ci, we should test everything" + echo "result of git diff ${GIT_DIFF_ARG} ci:" + git diff ${GIT_DIFF_ARG} ci + GIT_DIFF_ARG="" fi # Now we have a fixed list, but we can change it to autodetect if @@ -87,7 +111,7 @@ subdirs=( ) RETVAL=0 -# These following APIs need an explicit credential file to run properly (or oAuth2, which we don't support in this repo). +# These following APIs need an explicit credential file to run properly (or oAuth2, which we don't support in this repo). # When we hit these packages, we will run the "samples with credentials" trigger, which contains the credentials as an env variable tests_with_credentials="core/packages/google-auth-library-nodejs/ packages/google-analytics-admin/ packages/google-area120-tables/ packages/google-analytics-data/ packages/google-iam-credentials/ packages/google-apps-meet/ packages/google-chat/ packages/google-streetview-publish/ packages/google-cloud-developerconnect/" @@ -95,10 +119,13 @@ tests_with_credentials="core/packages/google-auth-library-nodejs/ packages/googl # Some packages are only used by our bots and automation. These packages do not need to run on Windows and # often employ platform specific code like file system interaction. Some packages may also fail # on Windows due to incompatible npm scripts. -# +# # Until these packages can be updated to be OS agnostic, we will skip them on Windows. windows_exempt_tests="core/ core/packages/ core/dev-packages/ .github/scripts/fixtures/ .github/scripts/tests/ core/packages/gapic-node-processing/ core/packages/typeless-sample-bot/ handwritten/cloud-profiler/" +# Gather all test directories into an array +test_dirs=() + for subdir in ${subdirs[@]}; do for d in `ls -d ${subdir}/*/`; do if [ -s "ignore.json" ] && jq -e ".ignored[] | select(. == \"$d\")" ignore.json > /dev/null 2>&1; then @@ -121,7 +148,7 @@ for subdir in ${subdirs[@]}; do # System tests for packages are broken and blocking PRs. # See https://github.com/googleapis/google-cloud-node/issues/7976. # - # Per https://github.com/googleapis/google-cloud-node/issues/7921, + # Per https://github.com/googleapis/google-cloud-node/issues/7921, # we are likely to permanently remove these tests in the near future. if [[ "${subdir}" == "packages" && "${TEST_TYPE}" == "system" ]]; then echo "Skipping ${TEST_TYPE} test for packages: ${d}" @@ -131,7 +158,7 @@ for subdir in ${subdirs[@]}; do # Sample tests for packages are broken/flaky and blocking PRs. # See https://github.com/googleapis/google-cloud-node/issues/7976#issuecomment-4210458096. # - # Per https://github.com/googleapis/google-cloud-node/issues/7921, + # Per https://github.com/googleapis/google-cloud-node/issues/7921, # we are likely to permanently remove these tests in the near future. if [[ "${subdir}" == "packages" && "${TEST_TYPE}" == "samples" ]]; then echo "Skipping ${TEST_TYPE} test for packages: ${d}" @@ -209,21 +236,51 @@ for subdir in ${subdirs[@]}; do fi fi if [ "${should_test}" = true ]; then - echo "running test in ${d}" - pushd ${d} - # Temporarily allow failure. - set +e - ${test_script} - ret=$? - set -e - if [ ${ret} -ne 0 ]; then - RETVAL=${ret} - # Since there are so many APIs, we should exit early if there's an error - exit ${RETVAL} - fi - popd + test_dirs+=("${d}") fi done done +# If DRY_RUN_SHARDS is set, output dynamic matrix values to GitHub Actions and exit +if [[ "${DRY_RUN_SHARDS}" == "true" ]]; then + count=${#test_dirs[@]} + if [[ $count -gt 15 ]]; then + matrix="[0, 1, 2, 3, 4]" + total="5" + else + matrix="[0]" + total="1" + fi + if [[ -n "${GITHUB_OUTPUT}" ]]; then + echo "shard_matrix=${matrix}" >> "${GITHUB_OUTPUT}" + echo "shard_total=${total}" >> "${GITHUB_OUTPUT}" + else + echo "shard_matrix=${matrix}" + echo "shard_total=${total}" + fi + exit 0 +fi + +# If SHARD_TOTAL and SHARD_INDEX are provided, we will only run a subset of the tests. +for i in "${!test_dirs[@]}"; do + d="${test_dirs[$i]}" + + if [[ -n "${SHARD_TOTAL}" && -n "${SHARD_INDEX}" ]]; then + if (( SHARD_TOTAL > 0 && i % SHARD_TOTAL != SHARD_INDEX )); then + continue + fi + fi + + echo "running test in ${d}" + pushd "${d}" >/dev/null + # Temporarily allow failure. + set +e + "${test_script}" + ret=$? + set -e + if [ ${ret} -ne 0 ]; then + exit ${ret} + fi + popd >/dev/null +done -exit ${RETVAL} +exit 0 diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 662a1184f347..3dfd98ee7a2b 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -37,6 +37,8 @@ if [ ${BUILD_TYPE} != "presubmit" ]; then export MOCHA_REPORTER_OUTPUT=${PROJECT}_sponge_log.xml export MOCHA_REPORTER_SUITENAME=${PROJECT} export MOCHA_REPORTER=xunit +else + export MOCHA_REPORTER=dot fi # Install dependencies @@ -47,8 +49,8 @@ if command -v cygpath >/dev/null 2>&1; then PNPMFILE_PATH=$(cygpath -m "${PNPMFILE_PATH}") fi -echo "pnpm install --ignore-scripts --engine-strict --prod --pnpmfile \"${PNPMFILE_PATH}\"; pnpm install --pnpmfile \"${PNPMFILE_PATH}\"" -pnpm install --ignore-scripts --engine-strict --prod --pnpmfile "${PNPMFILE_PATH}"; pnpm install --pnpmfile "${PNPMFILE_PATH}" +echo "pnpm install --engine-strict --pnpmfile \"${PNPMFILE_PATH}\"" +pnpm install --engine-strict --pnpmfile "${PNPMFILE_PATH}" retval=0 diff --git a/core/dev-packages/pack-n-play/test/fixtures/esm-package/test/test.js b/core/dev-packages/pack-n-play/test/fixtures/esm-package/test/test.js index 037babedf227..2adbf53d289c 100644 --- a/core/dev-packages/pack-n-play/test/fixtures/esm-package/test/test.js +++ b/core/dev-packages/pack-n-play/test/fixtures/esm-package/test/test.js @@ -16,7 +16,8 @@ import {packNTest} from 'pack-n-play'; import * as assert from 'assert'; import {describe, it} from 'mocha'; -describe('ESM package', () => { +describe('ESM package', function () { + this.timeout(120000); it('should support esm property', () => packNTest({ sample: { diff --git a/core/dev-packages/pack-n-play/test/fixtures/leaky/test/test.ts b/core/dev-packages/pack-n-play/test/fixtures/leaky/test/test.ts index 3a9dc7f39b39..9c56bc9c0741 100644 --- a/core/dev-packages/pack-n-play/test/fixtures/leaky/test/test.ts +++ b/core/dev-packages/pack-n-play/test/fixtures/leaky/test/test.ts @@ -16,7 +16,8 @@ import {packNTest} from 'pack-n-play'; import * as assert from 'assert'; import {describe, it} from 'mocha'; -describe('leaky tests', () => { +describe('leaky tests', function () { + this.timeout(120000); it('should fail packing n testing', async () => { await assert.rejects( packNTest({ diff --git a/core/dev-packages/pack-n-play/test/fixtures/pass/test/test.ts b/core/dev-packages/pack-n-play/test/fixtures/pass/test/test.ts index 2dd02fb27cff..50d0d07d620d 100644 --- a/core/dev-packages/pack-n-play/test/fixtures/pass/test/test.ts +++ b/core/dev-packages/pack-n-play/test/fixtures/pass/test/test.ts @@ -15,7 +15,8 @@ import {packNTest} from 'pack-n-play'; import {describe, it} from 'mocha'; -describe('passing tests', () => { +describe('passing tests', function () { + this.timeout(120000); it('should pass the test', async () => { await packNTest({ sample: { diff --git a/core/dev-packages/pack-n-play/test/test.ts b/core/dev-packages/pack-n-play/test/test.ts index da954b94ba08..06aef544b4e7 100644 --- a/core/dev-packages/pack-n-play/test/test.ts +++ b/core/dev-packages/pack-n-play/test/test.ts @@ -18,7 +18,8 @@ import execa = require('execa'); import {describe, it} from 'mocha'; describe('pack-n-play', () => { - it('should run tests', async () => { + it('should run tests', async function () { + this.timeout(600000); // 10 minutes const fixturesPath = path.resolve('./test/fixtures'); const dirs = fs .readdirSync(fixturesPath) @@ -29,7 +30,7 @@ describe('pack-n-play', () => { stdio: 'inherit', cwd: dir, }; - await execa('npm', ['install'], opts); + await execa('npm', ['install', '--no-audit', '--no-fund'], opts); await execa('npm', ['link', '../../../'], opts); await execa('npm', ['test'], opts); }