Skip to content

Fix Flaky Integration Tests #1741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
siri-varma wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Fix Flaky Integration Tests #1741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b42b5ce
feat: Add compensation workflow pattern to Spring Boot examples
Mar 13, 2026
5b53bd4
Apply suggestions from code review
siri-varma Mar 13, 2026
78b5caa
Refactor BookTripWorkflow to use CompensationHelper
siri-varma Mar 20, 2026
84ac76e
Add CompensationHelper class for managing compensations
siri-varma Mar 20, 2026
9c494d5
Await cancellation activities in BookTripWorkflow
siri-varma Apr 20, 2026
c01299d
fix things
May 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 19 additions & 15 deletions sdk-tests/src/test/java/io/dapr/it/DaprRun.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,13 @@ public void stop() throws InterruptedException, IOException {
System.out.println("Stopping dapr application ...");
try {
this.stopCommand.run();

System.out.println("Dapr application stopped.");
} catch (RuntimeException e) {
System.out.println("Could not stop app " + this.appName + ": " + e.getMessage());
if (e.getMessage() != null && e.getMessage().contains("Could not find success criteria")) {
System.out.println("App " + this.appName + " already stopped or not found (ignored).");
} else {
System.out.println("Could not stop app " + this.appName + ": " + e.getMessage());
}
}
}

Expand Down Expand Up @@ -219,8 +222,7 @@ public void waitForAppHealth(int maxWaitMilliseconds) throws InterruptedExceptio
while (System.currentTimeMillis() <= maxWait) {
try {
stub.healthCheck(Empty.getDefaultInstance());
// artursouza: workaround due to race condition with runtime's probe on app's health.
Thread.sleep(5000);
Thread.sleep(2000);
return;
} catch (Exception e) {
Thread.sleep(1000);
Expand All @@ -232,29 +234,31 @@ public void waitForAppHealth(int maxWaitMilliseconds) throws InterruptedExceptio
channel.shutdown();
}
} else {
Duration waitDuration = Duration.ofMillis(maxWaitMilliseconds);
long maxWait = System.currentTimeMillis() + maxWaitMilliseconds;
HttpClient client = HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_1_1)
.connectTimeout(waitDuration)
.connectTimeout(Duration.ofSeconds(5))
.build();
String url = "http://127.0.0.1:" + this.getAppPort() + "/health";
HttpRequest request = HttpRequest.newBuilder()
.GET()
.uri(URI.create(url))
.build();

try {
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

if (response.statusCode() != 200) {
throw new RuntimeException("error: HTTP service is not healthy.");
while (System.currentTimeMillis() <= maxWait) {
try {
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() == 200) {
Thread.sleep(2000);
return;
}
} catch (IOException e) {
// not ready yet
}
} catch (IOException e) {
throw new RuntimeException("exception: HTTP service is not healthy.");
Thread.sleep(1000);
}

// artursouza: workaround due to race condition with runtime's probe on app's health.
Thread.sleep(5000);
throw new RuntimeException("timeout: HTTP service is not healthy.");
}
}

Expand Down
9 changes: 2 additions & 7 deletions sdk-tests/src/test/java/io/dapr/it/actors/ActorReminderRecoveryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,24 +129,19 @@ public void reminderRecoveryTest(
) throws Exception {
setup(actorType);

logger.debug("Pausing 3 seconds to let gRPC connection get ready");
Thread.sleep(3000);

logger.debug("Invoking actor method 'startReminder' which will register a reminder");
proxy.invokeMethod("setReminderData", reminderDataParam).block();

proxy.invokeMethod("startReminder", reminderName).block();

logger.debug("Pausing 7 seconds to allow reminder to fire");
Thread.sleep(7000);

logger.debug("Waiting for reminder to fire at least 3 times");
final List<MethodEntryTracker> logs = new ArrayList<>();
callWithRetry(() -> {
logs.clear();
logs.addAll(fetchMethodCallLogs(proxy));
validateMethodCalls(logs, METHOD_NAME, 3);
validateMessageContent(logs, METHOD_NAME, expectedReminderStateText);
}, 5000);
}, 30000);

// Restarts runtime only.
logger.info("Stopping Dapr sidecar");
Expand Down
9 changes: 3 additions & 6 deletions sdk-tests/src/test/java/io/dapr/it/actors/ActorTimerRecoveryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ public void timerRecoveryTest() throws Exception {
true,
60000);

Thread.sleep(3000);
String actorType="MyActorTest";
logger.debug("Creating proxy builder");

Expand All @@ -68,16 +67,14 @@ public void timerRecoveryTest() throws Exception {
logger.debug("Invoking actor method 'startTimer' which will register a timer");
proxy.invokeMethod("startTimer", "myTimer").block();

logger.debug("Pausing 7 seconds to allow timer to fire");
Thread.sleep(7000);

logger.debug("Waiting for timer to fire at least 3 times");
final List<MethodEntryTracker> logs = new ArrayList<>();
callWithRetry(() -> {
logs.clear();
logs.addAll(fetchMethodCallLogs(proxy));
validateMethodCalls(logs, METHOD_NAME, 3);
validateMessageContent(logs, METHOD_NAME, "ping!");
}, 5000);
}, 30000);

// Restarts app only.
runs.left.stop();
Expand All @@ -91,7 +88,7 @@ public void timerRecoveryTest() throws Exception {
newLogs.clear();
newLogs.addAll(fetchMethodCallLogs(proxy));
validateMethodCalls(newLogs, METHOD_NAME, 3);
}, 10000);
}, 30000);

// Check that the restart actually happened by confirming the old logs are not in the new logs.
for (MethodEntryTracker oldLog: logs) {
Expand Down
95 changes: 62 additions & 33 deletions sdk-tests/src/test/java/io/dapr/it/testcontainers/jobs/DaprJobsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import io.dapr.it.testcontainers.DaprClientConfiguration;
import io.dapr.testcontainers.DaprContainer;
import io.dapr.testcontainers.DaprLogLevel;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
Expand All @@ -43,9 +44,11 @@
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.Random;
import java.util.UUID;

import static io.dapr.it.testcontainers.ContainerConstants.DAPR_RUNTIME_IMAGE_TAG;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;

@SpringBootTest(
webEnvironment = WebEnvironment.RANDOM_PORT,
Expand Down Expand Up @@ -93,63 +96,77 @@ public void setUp(){

@Test
public void testJobScheduleCreationWithDueTime() {
String jobName = "Job-" + UUID.randomUUID().toString().substring(0, 8);
DateTimeFormatter iso8601Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.withZone(ZoneOffset.UTC);

Instant currentTime = Instant.now();
daprClient.scheduleJob(new ScheduleJobRequest("Job", currentTime).setOverwrite(true)).block();
daprClient.scheduleJob(new ScheduleJobRequest(jobName, currentTime)).block();

GetJobResponse getJobResponse =
daprClient.getJob(new GetJobRequest("Job")).block();
GetJobResponse getJobResponse = Awaitility.await()
.atMost(Duration.ofSeconds(10))
.pollInterval(Duration.ofMillis(300))
.ignoreExceptions()
.until(() -> daprClient.getJob(new GetJobRequest(jobName)).block(), r -> r != null);

daprClient.deleteJob(new DeleteJobRequest("Job")).block();
daprClient.deleteJob(new DeleteJobRequest(jobName)).block();

assertNotNull(getJobResponse);
assertEquals(iso8601Formatter.format(currentTime), getJobResponse.getDueTime().toString());
assertEquals("Job", getJobResponse.getName());
assertEquals(jobName, getJobResponse.getName());
}

@Test
public void testJobScheduleCreationWithSchedule() {
String jobName = "Job-" + UUID.randomUUID().toString().substring(0, 8);
DateTimeFormatter iso8601Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.withZone(ZoneOffset.UTC);

Instant currentTime = Instant.now();
daprClient.scheduleJob(new ScheduleJobRequest("Job", JobSchedule.hourly())
.setDueTime(currentTime).setOverwrite(true)).block();
daprClient.scheduleJob(new ScheduleJobRequest(jobName, JobSchedule.hourly())
.setDueTime(currentTime)).block();

GetJobResponse getJobResponse =
daprClient.getJob(new GetJobRequest("Job")).block();
GetJobResponse getJobResponse = Awaitility.await()
.atMost(Duration.ofSeconds(10))
.pollInterval(Duration.ofMillis(300))
.ignoreExceptions()
.until(() -> daprClient.getJob(new GetJobRequest(jobName)).block(), r -> r != null);

daprClient.deleteJob(new DeleteJobRequest("Job")).block();
daprClient.deleteJob(new DeleteJobRequest(jobName)).block();

assertNotNull(getJobResponse);
assertEquals(iso8601Formatter.format(currentTime), getJobResponse.getDueTime().toString());
assertEquals(JobSchedule.hourly().getExpression(), getJobResponse.getSchedule().getExpression());
assertEquals("Job", getJobResponse.getName());
assertEquals(jobName, getJobResponse.getName());
}

@Test
public void testJobScheduleCreationWithAllParameters() {
String jobName = "Job-" + UUID.randomUUID().toString().substring(0, 8);
Instant currentTime = Instant.now();
DateTimeFormatter iso8601Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.withZone(ZoneOffset.UTC);

String cronExpression = "2 * 3 * * FRI";

daprClient.scheduleJob(new ScheduleJobRequest("Job", currentTime)
daprClient.scheduleJob(new ScheduleJobRequest(jobName, currentTime)
.setTtl(currentTime.plus(2, ChronoUnit.HOURS))
.setData("Job data".getBytes())
.setRepeat(3)
.setOverwrite(true)
.setSchedule(JobSchedule.fromString(cronExpression))).block();

GetJobResponse getJobResponse =
daprClient.getJob(new GetJobRequest("Job")).block();
GetJobResponse getJobResponse = Awaitility.await()
.atMost(Duration.ofSeconds(10))
.pollInterval(Duration.ofMillis(300))
.ignoreExceptions()
.until(() -> daprClient.getJob(new GetJobRequest(jobName)).block(), r -> r != null);

daprClient.deleteJob(new DeleteJobRequest("Job")).block();
daprClient.deleteJob(new DeleteJobRequest(jobName)).block();

assertNotNull(getJobResponse);
assertEquals(iso8601Formatter.format(currentTime), getJobResponse.getDueTime().toString());
assertEquals("2 * 3 * * FRI", getJobResponse.getSchedule().getExpression());
assertEquals("Job", getJobResponse.getName());
assertEquals(jobName, getJobResponse.getName());
assertEquals(Integer.valueOf(3), getJobResponse.getRepeats());
assertEquals("Job data", new String(getJobResponse.getData()));
assertEquals(iso8601Formatter.format(currentTime.plus(2, ChronoUnit.HOURS)),
Expand All @@ -158,48 +175,54 @@ public void testJobScheduleCreationWithAllParameters() {

@Test
public void testJobScheduleCreationWithDropFailurePolicy() {
String jobName = "Job-" + UUID.randomUUID().toString().substring(0, 8);
Instant currentTime = Instant.now();
DateTimeFormatter iso8601Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.withZone(ZoneOffset.UTC);

String cronExpression = "2 * 3 * * FRI";

daprClient.scheduleJob(new ScheduleJobRequest("Job", currentTime)
daprClient.scheduleJob(new ScheduleJobRequest(jobName, currentTime)
.setTtl(currentTime.plus(2, ChronoUnit.HOURS))
.setData("Job data".getBytes())
.setRepeat(3)
.setFailurePolicy(new DropFailurePolicy())
.setFailurePolicy(new DropFailurePolicy())
.setSchedule(JobSchedule.fromString(cronExpression))).block();

GetJobResponse getJobResponse =
daprClient.getJob(new GetJobRequest("Job")).block();
GetJobResponse getJobResponse = Awaitility.await()
.atMost(Duration.ofSeconds(10))
.pollInterval(Duration.ofMillis(300))
.ignoreExceptions()
.until(() -> daprClient.getJob(new GetJobRequest(jobName)).block(), r -> r != null);

daprClient.deleteJob(new DeleteJobRequest("Job")).block();
daprClient.deleteJob(new DeleteJobRequest(jobName)).block();

assertNotNull(getJobResponse);
assertEquals(FailurePolicyType.DROP, getJobResponse.getFailurePolicy().getFailurePolicyType());
}

@Test
public void testJobScheduleCreationWithConstantFailurePolicy() {
String jobName = "Job-" + UUID.randomUUID().toString().substring(0, 8);
Instant currentTime = Instant.now();
DateTimeFormatter iso8601Formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
.withZone(ZoneOffset.UTC);

String cronExpression = "2 * 3 * * FRI";

daprClient.scheduleJob(new ScheduleJobRequest("Job", currentTime)
daprClient.scheduleJob(new ScheduleJobRequest(jobName, currentTime)
.setTtl(currentTime.plus(2, ChronoUnit.HOURS))
.setData("Job data".getBytes())
.setRepeat(3)
.setFailurePolicy(new ConstantFailurePolicy(3)
.setDurationBetweenRetries(Duration.of(10, ChronoUnit.SECONDS)))
.setSchedule(JobSchedule.fromString(cronExpression))).block();

GetJobResponse getJobResponse =
daprClient.getJob(new GetJobRequest("Job")).block();
GetJobResponse getJobResponse = Awaitility.await()
.atMost(Duration.ofSeconds(10))
.pollInterval(Duration.ofMillis(300))
.ignoreExceptions()
.until(() -> daprClient.getJob(new GetJobRequest(jobName)).block(), r -> r != null);

daprClient.deleteJob(new DeleteJobRequest("Job")).block();
daprClient.deleteJob(new DeleteJobRequest(jobName)).block();

assertNotNull(getJobResponse);
ConstantFailurePolicy jobFailurePolicyConstant = (ConstantFailurePolicy) getJobResponse.getFailurePolicy();
assertEquals(FailurePolicyType.CONSTANT, getJobResponse.getFailurePolicy().getFailurePolicyType());
assertEquals(3, (int)jobFailurePolicyConstant.getMaxRetries());
Expand All @@ -209,17 +232,23 @@ public void testJobScheduleCreationWithConstantFailurePolicy() {

@Test
public void testDeleteJobRequest() {
String jobName = "Job-" + UUID.randomUUID().toString().substring(0, 8);
Instant currentTime = Instant.now();

String cronExpression = "2 * 3 * * FRI";

daprClient.scheduleJob(new ScheduleJobRequest("Job", currentTime)
daprClient.scheduleJob(new ScheduleJobRequest(jobName, currentTime)
.setTtl(currentTime.plus(2, ChronoUnit.HOURS))
.setData("Job data".getBytes())
.setRepeat(3)
.setOverwrite(true)
.setSchedule(JobSchedule.fromString(cronExpression))).block();

daprClient.deleteJob(new DeleteJobRequest("Job")).block();
Awaitility.await()
.atMost(Duration.ofSeconds(10))
.pollInterval(Duration.ofMillis(300))
.ignoreExceptions()
.until(() -> daprClient.getJob(new GetJobRequest(jobName)).block(), r -> r != null);

daprClient.deleteJob(new DeleteJobRequest(jobName)).block();
}
}
5 changes: 2 additions & 3 deletions sdk-tests/src/test/java/io/dapr/it/testcontainers/pubsub/outbox/DaprPubSubOutboxIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.awaitility.Awaitility;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
Expand All @@ -47,7 +46,6 @@

import static io.dapr.it.testcontainers.ContainerConstants.DAPR_RUNTIME_IMAGE_TAG;

@Disabled("Unclear why this test is failing intermittently in CI")
@SpringBootTest(
webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT,
classes = {
Expand Down Expand Up @@ -128,7 +126,8 @@ public void shouldPublishUsingOutbox() throws Exception {

client.executeStateTransaction(transactionRequest).block();

Awaitility.await().atMost(Duration.ofSeconds(10))
Awaitility.await().atMost(Duration.ofSeconds(30))
.pollInterval(Duration.ofMillis(500))
.ignoreExceptions()
.untilAsserted(() -> Assertions.assertThat(productWebhookController.getEventList()).isNotEmpty());
}
Expand Down
Loading
Loading