Skip to content

Commit 6e6020d

Browse files
alexghralexghr
committed
chore: run proving benchmarks nightly and weekly
1 parent 40ef9bb commit 6e6020d

8 files changed

Lines changed: 335 additions & 5 deletions

File tree

‎.github/workflows/nightly-spartan-bench.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,3 +117,120 @@ jobs:
117117
-H "Content-type: application/json" \
118118
--data "$data"
119119
fi
120+
121+
proving-benchmark:
122+
runs-on: ubuntu-latest
123+
steps:
124+
- name: Checkout
125+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
126+
# Note cron jobs will be on 'next' by default.
127+
128+
- name: Determine nightly tag
129+
id: nightly-tag
130+
run: |
131+
if [[ -n "${{ inputs.nightly_tag }}" ]]; then
132+
nightly_tag="${{ inputs.nightly_tag }}"
133+
else
134+
current_version=$(jq -r '."."' .release-please-manifest.json)
135+
nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)"
136+
fi
137+
echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT
138+
echo "Using nightly tag: $nightly_tag"
139+
140+
- name: Check if Docker image exists
141+
run: |
142+
DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
143+
echo "Checking if Docker image exists: $DOCKER_IMAGE"
144+
if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
145+
echo "Docker image exists: $DOCKER_IMAGE"
146+
else
147+
echo "Docker image does not exist: $DOCKER_IMAGE"
148+
exit 1
149+
fi
150+
151+
- name: Run proving benchmarks
152+
timeout-minutes: 150
153+
env:
154+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
155+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
156+
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
157+
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
158+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
159+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
160+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
161+
RUN_ID: ${{ github.run_id }}
162+
AWS_SHUTDOWN_TIME: 150
163+
NO_SPOT: 1
164+
run: |
165+
./.github/ci3.sh network-proving-bench prove-n-tps-fake prove-n-tps-fake "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
166+
167+
- name: Cleanup network resources
168+
if: always()
169+
env:
170+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
171+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
172+
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
173+
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
174+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
175+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
176+
NO_SPOT: 1
177+
run: ./.github/ci3.sh network-teardown prove-n-tps-fake prove-n-tps-fake
178+
179+
- name: Download benchmarks
180+
if: always()
181+
env:
182+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
183+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
184+
run: |
185+
if ./ci.sh gh-spartan-proving-bench; then
186+
echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
187+
fi
188+
189+
- name: Upload benchmarks
190+
if: always() && env.ENABLE_DEPLOY_BENCH == '1'
191+
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
192+
with:
193+
name: Spartan
194+
benchmark-data-dir-path: "bench/next"
195+
tool: "customSmallerIsBetter"
196+
output-file-path: ./bench-out/bench.json
197+
github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
198+
auto-push: true
199+
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
200+
alert-threshold: "120%"
201+
comment-on-alert: false
202+
fail-on-alert: false
203+
max-items-in-chart: 100
204+
205+
- name: Notify Slack on failure
206+
if: failure() && github.event_name != 'workflow_dispatch'
207+
env:
208+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
209+
run: |
210+
if [ -n "${SLACK_BOT_TOKEN}" ]; then
211+
read -r -d '' data <<EOF || true
212+
{
213+
"channel": "#alerts-next-scenario",
214+
"text": "Nightly Proving Benchmarks FAILED (nightly tag ${{ steps.nightly-tag.outputs.nightly_tag }}): <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
215+
}
216+
EOF
217+
curl -X POST https://slack.com/api/chat.postMessage \
218+
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
219+
-H "Content-type: application/json" \
220+
--data "$data"
221+
fi
222+
223+
status:
224+
runs-on: ubuntu-latest
225+
needs: [benchmark, proving-benchmark]
226+
if: always()
227+
steps:
228+
- name: Check benchmark results
229+
run: |
230+
if [[ "${{ needs.benchmark.result }}" != "success" || "${{ needs.proving-benchmark.result }}" != "success" ]]; then
231+
echo "One or more benchmark jobs failed"
232+
echo "benchmark: ${{ needs.benchmark.result }}"
233+
echo "proving-benchmark: ${{ needs.proving-benchmark.result }}"
234+
exit 1
235+
fi
236+
echo "All benchmark jobs succeeded"

‎.github/workflows/weekly-proving-bench.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: Weekly Real Proving Benchmark
2+
3+
on:
4+
schedule:
5+
- cron: "0 6 * * 1" # Every Monday at 6 AM UTC
6+
workflow_dispatch:
7+
inputs:
8+
nightly_tag:
9+
description: "Nightly tag to use (e.g., 2.3.4-nightly.20251209). Leave empty to auto-detect."
10+
required: false
11+
type: string
12+
13+
concurrency:
14+
group: weekly-proving-bench-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
real-proving-benchmark:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
23+
# Note cron jobs will be on 'next' by default.
24+
25+
- name: Determine nightly tag
26+
id: nightly-tag
27+
run: |
28+
if [[ -n "${{ inputs.nightly_tag }}" ]]; then
29+
nightly_tag="${{ inputs.nightly_tag }}"
30+
else
31+
current_version=$(jq -r '."."' .release-please-manifest.json)
32+
nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)"
33+
fi
34+
echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT
35+
echo "Using nightly tag: $nightly_tag"
36+
37+
- name: Check if Docker image exists
38+
run: |
39+
DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
40+
echo "Checking if Docker image exists: $DOCKER_IMAGE"
41+
if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
42+
echo "Docker image exists: $DOCKER_IMAGE"
43+
else
44+
echo "Docker image does not exist: $DOCKER_IMAGE"
45+
exit 1
46+
fi
47+
48+
- name: Run real proving benchmarks
49+
timeout-minutes: 150
50+
env:
51+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
52+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53+
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
54+
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
55+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
56+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
57+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
58+
RUN_ID: ${{ github.run_id }}
59+
AWS_SHUTDOWN_TIME: 150
60+
NO_SPOT: 1
61+
run: |
62+
./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
63+
64+
- name: Cleanup network resources
65+
if: always()
66+
env:
67+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
68+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
69+
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
70+
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
71+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
72+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
73+
NO_SPOT: 1
74+
run: ./.github/ci3.sh network-teardown prove-n-tps-real prove-n-tps-real
75+
76+
- name: Download benchmarks
77+
if: always()
78+
env:
79+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
80+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
81+
run: |
82+
if ./ci.sh gh-spartan-proving-bench; then
83+
echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
84+
fi
85+
86+
- name: Upload benchmarks
87+
if: always() && env.ENABLE_DEPLOY_BENCH == '1'
88+
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
89+
with:
90+
name: Spartan
91+
benchmark-data-dir-path: "bench/next"
92+
tool: "customSmallerIsBetter"
93+
output-file-path: ./bench-out/bench.json
94+
github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
95+
auto-push: true
96+
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
97+
alert-threshold: "120%"
98+
comment-on-alert: false
99+
fail-on-alert: false
100+
max-items-in-chart: 100
101+
102+
- name: Notify Slack on failure
103+
if: failure() && github.event_name != 'workflow_dispatch'
104+
env:
105+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
106+
run: |
107+
if [ -n "${SLACK_BOT_TOKEN}" ]; then
108+
read -r -d '' data <<EOF || true
109+
{
110+
"channel": "#alerts-next-scenario",
111+
"text": "Weekly Real Proving Benchmark FAILED (nightly tag ${{ steps.nightly-tag.outputs.nightly_tag }}): <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
112+
}
113+
EOF
114+
curl -X POST https://slack.com/api/chat.postMessage \
115+
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
116+
-H "Content-type: application/json" \
117+
--data "$data"
118+
fi

‎bootstrap.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,30 @@ case "$cmd" in
653653
bench_merge
654654
cache_upload spartan-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
655655
;;
656+
"ci-network-proving-bench")
657+
# Args: <env_file> <namespace> [docker_image]
658+
# Deploys network and runs proving benchmarks. Cleanup should be done separately.
659+
export CI=1
660+
env_file="${1:?env_file is required}"
661+
namespace="${2:?namespace is required}"
662+
docker_image="${3:-}"
663+
build
664+
# If no docker image provided, build and push to aztecdev
665+
if [ -z "$docker_image" ]; then
666+
release-image/bootstrap.sh push_pr
667+
docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
668+
fi
669+
# Set up environment and deploy using spartan
670+
export NAMESPACE="$namespace"
671+
export AZTEC_DOCKER_IMAGE="$docker_image"
672+
spartan/bootstrap.sh network_deploy "${env_file}"
673+
# Run proving benchmarks
674+
spartan/bootstrap.sh proving_bench "${env_file}"
675+
rm -rf bench-out
676+
mkdir -p bench-out
677+
bench_merge
678+
cache_upload spartan-proving-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
679+
;;
656680
"ci-network-teardown")
657681
# Args: <env_file> <namespace>
658682
# Tears down a deployed network.

‎ci.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,14 @@ case "$cmd" in
164164
export INSTANCE_POSTFIX="n-bench"
165165
bootstrap_ec2 "./bootstrap.sh ci-network-bench $*"
166166
;;
167+
network-proving-bench)
168+
# Args: <scenario> <namespace> [docker_image]
169+
# Deploys network and runs proving benchmarks.
170+
export CI_DASHBOARD="network"
171+
export JOB_ID="x-${2:?namespace is required}-network-proving-bench" CPUS=16
172+
export INSTANCE_POSTFIX="n-proving-bench"
173+
bootstrap_ec2 "./bootstrap.sh ci-network-proving-bench $*"
174+
;;
167175
network-teardown)
168176
# Args: <scenario> <namespace>
169177
export CI_DASHBOARD="network"
@@ -290,7 +298,7 @@ case "$cmd" in
290298
########################
291299
# BENCHMARK PROCESSING #
292300
########################
293-
gh-bench|gh-deploy-bench|gh-spartan-bench)
301+
gh-bench|gh-deploy-bench|gh-spartan-bench|gh-spartan-proving-bench)
294302
cache_download ${cmd#gh-}-$(git rev-parse HEAD^{tree}).tar.gz
295303
;;
296304

‎spartan/.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ tfplan
66
mnemonic.tmp
77
environments/*
88
!environments/network-defaults.yml
9-
!environments/prove-1tps.env
9+
!environments/prove-n-tps-fake.env
10+
!environments/prove-n-tps-real.env
1011
!environments/ten-tps-short-epoch.env
1112
!environments/ten-tps-long-epoch.env
1213
!environments/five-tps-short-epoch.env

‎spartan/bootstrap.sh‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,12 @@ function network_bench_cmds {
155155
done
156156
}
157157

158+
function proving_bench_cmds {
159+
local tps=1
160+
local timeout=9000 # 2.5h
161+
echo "$hash:TIMEOUT=${timeout} TPS=${tps} BENCH_OUTPUT=bench-out/prove_n_tps.${tps}tps.bench.json $root/yarn-project/end-to-end/scripts/run_test.sh simple prove_n_tps.test.ts"
162+
}
163+
158164
function network_bench {
159165
rm -rf bench-out
160166
mkdir -p bench-out
@@ -167,6 +173,18 @@ function network_bench {
167173
network_bench_cmds | parallelize 1
168174
}
169175

176+
function proving_bench {
177+
rm -rf bench-out
178+
mkdir -p bench-out
179+
180+
local env_file="$1"
181+
source_network_env $env_file
182+
183+
echo_header "spartan proving bench"
184+
gcp_auth
185+
proving_bench_cmds | parallelize 1
186+
}
187+
170188
function ensure_eth_balances {
171189
amount="$1"
172190
# if ETHEREUM_HOST is not set, use the first RPC URL
@@ -236,7 +254,7 @@ case "$cmd" in
236254
single_test $test_file
237255
;;
238256

239-
network_tests|network_tests_1|network_tests_2|network_bench)
257+
network_tests|network_tests_1|network_tests_2|network_bench|proving_bench)
240258
env_file="$1"
241259
$cmd "$env_file"
242260
;;

spartan/environments/prove-1tps.env renamed to spartan/environments/prove-n-tps-fake.env

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
NAMESPACE=${NAMESPACE:-prove-1tps}
1+
NAMESPACE=${NAMESPACE:-prove-n-tps-fake}
22
CLUSTER=aztec-gke-private
33
GCP_REGION=us-west1-a
44

55
AZTEC_EPOCH_DURATION=32
66
AZTEC_SLOT_DURATION=72
7-
AZTEC_PROOF_SUBMISSION_WINDOW=64
7+
AZTEC_PROOF_SUBMISSION_EPOCHS=1
88
AZTEC_LAG_IN_EPOCHS=1
99

1010
CREATE_ETH_DEVNET=true

0 commit comments

Comments
 (0)