fix(prover-client): increment retry count on timeout re-enqueue to prevent infinite loop

yarn-project/prover-client/src/proving_broker/proving_broker.test.ts

@@ -894,6 +894,30 @@ describe.each([
       await assertJobStatus(id, 'not-found');
     });
 
+    it('rejects jobs that time out more than maxRetries times', async () => {
+      const id = makeRandomProvingJobId();
+      await broker.enqueueProvingJob({
+        id,
+        type: ProvingRequestType.PARITY_BASE,
+        epochNumber: EpochNumber(1),
+        inputsUri: makeInputsUri(),
+      });
+
+      for (let i = 0; i < maxRetries; i++) {
+        await assertJobStatus(id, 'in-queue');
+        await getAndAssertNextJobId(id);
+        await assertJobStatus(id, 'in-progress');
+
+        await sleep(jobTimeoutMs);
+        await assertJobTransition(id, 'in-progress', i + 1 < maxRetries ? 'in-queue' : 'rejected');
+      }
+
+      await expect(broker.getProvingJobStatus(id)).resolves.toEqual({
+        status: 'rejected',
+        reason: 'Timed out',
+      });
+    });
+
     it('keeps the jobs in progress while it is alive', async () => {
       const id = makeRandomProvingJobId();
       await broker.enqueueProvingJob({

yarn-project/prover-client/src/proving_broker/proving_broker.ts

@@ -632,10 +632,26 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr
       const now = this.msTimeSource();
       const msSinceLastUpdate = now - metadata.lastUpdatedAt;
       if (msSinceLastUpdate >= this.jobTimeoutMs) {
-        this.logger.warn(`Proving job id=${id} timed out. Adding it back to the queue.`, { provingJobId: id });
         this.inProgress.delete(id);
-        this.enqueueJobInternal(item);
         this.instrumentation.incTimedOutJobs(item.type);
+
+        const retries = this.retries.get(id) ?? 0;
+        if (retries + 1 < this.maxRetries && !this.isJobStale(item)) {
+          this.logger.warn(`Proving job id=${id} timed out. Re-enqueueing (retry ${retries + 1}/${this.maxRetries}).`, {
+            provingJobId: id,
+          });
+          this.retries.set(id, retries + 1);
+          this.enqueueJobInternal(item);
+        } else {
+          this.logger.error(`Proving job id=${id} timed out after ${retries + 1} attempts. Marking as failed.`, {
+            provingJobId: id,
+          });
+          const result: ProvingJobSettledResult = { status: 'rejected', reason: 'Timed out' };
+          this.resultsCache.set(id, result);
+          this.promises.get(id)?.resolve(result);
+          this.completedJobNotifications.push(id);
+          this.instrumentation.incRejectedJobs(item.type);
+        }
       }
     }
   }