diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml new file mode 100644 index 00000000000..f9c25497220 --- /dev/null +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -0,0 +1,65 @@ +name: 'Integration Tests - Macrobenchmark (POC)' +# Proof-of-concept: run the sentry-uitest-android-macrobenchmark cold-start benchmark on Sauce +# Labs real devices. Manual trigger only — this is not yet a per-PR gate. The goal is to learn +# (a) whether Macrobenchmark runs on non-rooted cloud devices and which device guards fire, and +# (b) whether timeToInitialDisplay can be retrieved from Sauce artifacts. See the module README. +on: + workflow_dispatch: + # Temporary POC scaffolding: run on pushes to this branch so we can see it work on Sauce + # without merging to the default branch first. Remove before merge. + push: + branches: + - no/macrobenchmark-on-sauce-poc + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + macrobenchmark: + name: Macrobenchmark + runs-on: ubuntu-latest + + # we copy the secret to the env variable in order to access it in the workflow + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + + steps: + - name: Git checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: 'recursive' + + - name: 'Set up Java: 17' + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + with: + cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + + - name: Assemble target app and Macrobenchmark apk + if: env.SAUCE_USERNAME != null + run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark + + - name: Run Macrobenchmark in SauceLab + uses: saucelabs/saucectl-run-action@bc81720eb01738d9c664b07fe42621bd0014283f # pin@v3 + if: env.SAUCE_USERNAME != null + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-macrobenchmark.yml + + - name: Upload Sauce artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: macrobenchmark-artifacts + path: ./artifacts/ + if-no-files-found: warn diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml new file mode 100644 index 00000000000..6751560dc35 --- /dev/null +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -0,0 +1,43 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + concurrency: 1 + metadata: + build: sentry-uitest-android-macrobenchmark-$GITHUB_REF-$GITHUB_SHA + tags: + - benchmarks + - android + - macrobenchmark + +defaults: + timeout: 40m + +espresso: + # Target app under test: Macrobenchmark cold-starts sentry-samples-android. It must be + # release-like; the release build type is signed with the debug key so it installs on Sauce. + app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk + # Instrumentation APK: the self-instrumenting com.android.test macrobenchmark module. + testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk + +suites: + + - name: "Macrobenchmark startup (api 35)" + # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process + # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and + # permissions (it force-stops rather than `pm clear`). + devices: + - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end + +# Grab both the benchmark JSON (if Sauce collects additional test output) and the device log, so +# we can see which path actually yields timeToInitialDisplay. Macrobenchmark also logs guard +# failures (UNLOCKED, DEBUGGABLE, ...) here, which is the expected first-run signal on cloud +# hardware — see the module README. +artifacts: + download: + when: always + match: + - junit.xml + - "*.log" + - "*-benchmarkData.json" + directory: ./artifacts/