diff --git a/regression-test/plugins/cloud_recycler_plugin.groovy b/regression-test/plugins/cloud_recycler_plugin.groovy index 25dc6d331fc17c..3f9ca7fcab1d1a 100644 --- a/regression-test/plugins/cloud_recycler_plugin.groovy +++ b/regression-test/plugins/cloud_recycler_plugin.groovy @@ -37,6 +37,48 @@ import org.apache.hadoop.fs.LocatedFileStatus import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.RemoteIterator import org.apache.hadoop.security.UserGroupInformation +import java.util.concurrent.atomic.AtomicBoolean + +// Keep the suite-wide guard below TeamCity's 24-hour limit while allowing check_meta's +// intentional one-hour leak-check window and the existing 30-minute phase deadlines. +Suite.metaClass.enableRecyclerCaseTimeout = { long timeoutMs = 2 * 60 * 60 * 1000L -> + Suite suite = delegate as Suite + if (timeoutMs <= 0) { + throw new IllegalArgumentException("Recycler case timeout must be positive: ${timeoutMs}") + } + + Thread suiteThread = Thread.currentThread() + AtomicBoolean timedOut = new AtomicBoolean(false) + def watchdog = suite.extraThread("recycler-case-timeout-${suite.name}", true) { + try { + Thread.sleep(timeoutMs) + timedOut.set(true) + suite.getLogger().error( + "Recycler case ${suite.name} timed out after ${timeoutMs / 1000}s; interrupting the suite thread") + suiteThread.interrupt() + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt() + } + } + + suite.onSuccess { ignored -> + if (timedOut.get()) { + throw new IllegalStateException( + "Recycler case ${suite.name} timed out after ${timeoutMs / 1000}s") + } + } + suite.onFail { ignored -> + if (timedOut.get()) { + throw new IllegalStateException( + "Recycler case ${suite.name} timed out after ${timeoutMs / 1000}s") + } + } + suite.onFinish { ignored -> + watchdog.cancel(true) + } +} + +logger.info("Added 'enableRecyclerCaseTimeout' function to Suite") Suite.metaClass.triggerRecycle = { String token, String instanceId /* param */ -> // which suite invoke current function? @@ -590,4 +632,4 @@ Suite.metaClass.checkRecycleMetrics = { String recyclerHttpPort, String recycleJ sleep(5000) } } -logger.info("Added 'checkRecycleMetrics' function to Suite") \ No newline at end of file +logger.info("Added 'checkRecycleMetrics' function to Suite") diff --git a/regression-test/suites/cloud_p0/recycler/check_meta.groovy b/regression-test/suites/cloud_p0/recycler/check_meta.groovy index a9f34f19787455..ba15ad5b63191e 100644 --- a/regression-test/suites/cloud_p0/recycler/check_meta.groovy +++ b/regression-test/suites/cloud_p0/recycler/check_meta.groovy @@ -16,6 +16,8 @@ // under the License. suite("check_meta", "check_meta") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId; @@ -24,6 +26,8 @@ suite("check_meta", "check_meta") { def status = 200 def recyclerLastSuccessTime = -1 def recyclerLastFinishTime = -1 + def waitTimeoutMs = 30 * 60 * 1000L + def recyclerWaitDeadline = caseStartTime + waitTimeoutMs String jdbcUrl = context.config.jdbcUrl String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) @@ -89,6 +93,12 @@ suite("check_meta", "check_meta") { if (recyclerLastSuccessTime > caseStartTime) { break } + if (System.currentTimeMillis() >= recyclerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for recycler job after ${waitTimeoutMs / 1000}s: " + + "caseStartTime=${caseStartTime}, recyclerLastFinishTime=${recyclerLastFinishTime}, " + + "recyclerLastSuccessTime=${recyclerLastSuccessTime}") + } } while (true) assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime) diff --git a/regression-test/suites/cloud_p0/recycler/test_checker.groovy b/regression-test/suites/cloud_p0/recycler/test_checker.groovy index 6763db2c682e63..2151a946c04952 100644 --- a/regression-test/suites/cloud_p0/recycler/test_checker.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_checker.groovy @@ -41,6 +41,8 @@ import com.azure.storage.common.StorageSharedKeyCredential import java.time.Duration suite("test_checker") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId; @@ -180,6 +182,9 @@ suite("test_checker") { // Make sure to complete at least one round of checking def checkerLastSuccessTime = -1 def checkerLastFinishTime = -1 + def waitTimeoutMs = 30 * 60 * 1000L + def checkerWaitStartTime = System.currentTimeMillis() + def checkerWaitDeadline = checkerWaitStartTime + waitTimeoutMs def triggerChecker = { def triggerCheckerApi = { checkFunc -> @@ -233,6 +238,12 @@ suite("test_checker") { if (checkerLastFinishTime > caseStartTime) { break } + if (System.currentTimeMillis() >= checkerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for checker job after ${waitTimeoutMs / 1000}s: " + + "caseStartTime=${caseStartTime}, checkerLastFinishTime=${checkerLastFinishTime}, " + + "checkerLastSuccessTime=${checkerLastSuccessTime}") + } } while (true) assertTrue(checkerLastSuccessTime < checkerLastFinishTime) // Check MUST fail @@ -298,4 +309,4 @@ suite("test_checker") { } } } -} \ No newline at end of file +} diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler.groovy index 3a6e99f1778fc9..8426c7e6c51571 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; @@ -25,6 +27,8 @@ suite("test_recycler") { def caseStartTime = System.currentTimeMillis() def recyclerLastSuccessTime = -1 def recyclerLastFinishTime = -1 + def waitTimeoutMs = 30 * 60 * 1000L + def recyclerWaitDeadline = caseStartTime + waitTimeoutMs // Make sure to complete at least one round of recycling def getRecycleJobInfo = { @@ -61,6 +65,12 @@ suite("test_recycler") { if (recyclerLastFinishTime > caseStartTime) { break } + if (System.currentTimeMillis() >= recyclerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for recycler job after ${waitTimeoutMs / 1000}s: " + + "caseStartTime=${caseStartTime}, recyclerLastFinishTime=${recyclerLastFinishTime}, " + + "recyclerLastSuccessTime=${recyclerLastSuccessTime}") + } } while (true) assertEquals(recyclerLastFinishTime, recyclerLastSuccessTime) @@ -86,6 +96,8 @@ suite("test_recycler") { // Make sure to complete at least one round of checking def checkerLastSuccessTime = -1 def checkerLastFinishTime = -1 + def checkerWaitStartTime = System.currentTimeMillis() + def checkerWaitDeadline = checkerWaitStartTime + waitTimeoutMs def triggerChecker = { def triggerCheckerApi = { checkFunc -> @@ -139,6 +151,13 @@ suite("test_recycler") { if (checkerLastSuccessTime > recyclerLastSuccessTime) { break } + if (System.currentTimeMillis() >= checkerWaitDeadline) { + throw new IllegalStateException( + "Timed out waiting for checker job after ${waitTimeoutMs / 1000}s: " + + "recyclerLastSuccessTime=${recyclerLastSuccessTime}, " + + "checkerLastFinishTime=${checkerLastFinishTime}, " + + "checkerLastSuccessTime=${checkerLastSuccessTime}") + } } while (true) assertEquals(checkerLastFinishTime, checkerLastSuccessTime) } diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_expired_stage_objects.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_expired_stage_objects.groovy index 26315654178b5e..8ab1cc68e5095e 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_expired_stage_objects.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_expired_stage_objects.groovy @@ -18,6 +18,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_expired_stage_objects") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId def cloudUniqueId = context.config.cloudUniqueId diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_inverted_index.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_inverted_index.groovy index 10415f8f1abbaa..f3db70915c9f40 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_inverted_index.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_inverted_index.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_inverted_index") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_column.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_column.groovy index aed1c3b3c4cdcd..a7b6c9f5c119cd 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_column.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_column.groovy @@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import java.util.stream.Collectors suite("test_recycler_with_drop_column") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_db.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_db.groovy index f134817c254c26..95a813ba762ac2 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_db.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_db.groovy @@ -21,6 +21,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_drop_db") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; @@ -130,4 +132,3 @@ suite("test_recycler_with_drop_db") { } while (retry--) assertTrue(success) } - diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_index.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_index.groovy index 1992bd79cbdf97..f7d37077927de2 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_index.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_index.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_drop_index") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_multi_db.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_multi_db.groovy index 7bbad0ac9660f2..fda4b3a09a23d5 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_multi_db.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_multi_db.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_drop_multi_db") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; @@ -142,4 +144,3 @@ suite("test_recycler_with_drop_multi_db") { assertTrue(success) } } - diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_mv.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_mv.groovy index a35aa6ba74fe40..66d23eb0216db9 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_mv.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_mv.groovy @@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import java.util.stream.Collectors suite("test_recycler_with_drop_mv") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_partition.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_partition.groovy index fb840027367e4a..90ec965449b129 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_partition.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_partition.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_drop_partition") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_rollup.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_rollup.groovy index 4d7261d4d85594..4e6b2446b2b781 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_rollup.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_drop_rollup.groovy @@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import java.util.stream.Collectors suite("test_recycler_with_drop_rollup") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_dynamic_partition.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_dynamic_partition.groovy index 6c968755e57a3b..a53f04c5c76703 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_dynamic_partition.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_dynamic_partition.groovy @@ -17,6 +17,8 @@ import java.text.SimpleDateFormat; suite("test_recycler_with_dynamic_partition") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_internal_copy.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_internal_copy.groovy index ae7834f7e23692..ef36835b1ec116 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_internal_copy.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_internal_copy.groovy @@ -17,6 +17,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_internal_copy") { + enableRecyclerCaseTimeout() + def token = "greedisgood9999" def instanceId = context.config.instanceId; def cloudUniqueId = context.config.cloudUniqueId diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_many_partitions.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_many_partitions.groovy index 1b578a9a632e94..4eb183b60a527d 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_many_partitions.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_many_partitions.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_many_partitions") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_schema_change.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_schema_change.groovy index 382aa15ae56679..27599b783df191 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_schema_change.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_schema_change.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_schema_change") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_truncate_table.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_truncate_table.groovy index 15f18fbc009b3d..f036204bc33e4c 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_truncate_table.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_truncate_table.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_truncate_table") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId; diff --git a/regression-test/suites/cloud_p0/recycler/test_recycler_with_txn_label.groovy b/regression-test/suites/cloud_p0/recycler/test_recycler_with_txn_label.groovy index fff1a6642c4c6d..77398b598b0684 100644 --- a/regression-test/suites/cloud_p0/recycler/test_recycler_with_txn_label.groovy +++ b/regression-test/suites/cloud_p0/recycler/test_recycler_with_txn_label.groovy @@ -18,6 +18,8 @@ import groovy.json.JsonOutput import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recycler_with_txn_label") { + enableRecyclerCaseTimeout() + // create table def token = "greedisgood9999" def instanceId = context.config.instanceId;