Kubernetes integration testing using Minikube and Argo Workflows in Jenkins

DocumentsFromSnapshotMigration/build.gradle

@@ -72,7 +72,7 @@ clean.doFirst {
     delete dockerBuildDir
 }
 
-task copyDockerRuntimeJars (type: Sync) {
+tasks.register('copyDockerRuntimeJars', Sync) {
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
     description = 'Copy runtime JARs and app jar to docker build directory'
 
@@ -84,6 +84,11 @@ task copyDockerRuntimeJars (type: Sync) {
     from configurations.runtimeClasspath
     from tasks.named('jar')
     include '*.jar'
+
+    // Explicit inputs for incremental build support
+    inputs.files(configurations.runtimeClasspath)
+    inputs.files(tasks.named('jar'))
+    outputs.dir(buildDir)
 }
 
 DockerServiceProps[] dockerServices = [
@@ -94,7 +99,7 @@ DockerServiceProps[] dockerServices = [
 ] as DockerServiceProps[]
 
 for (dockerService in dockerServices) {
-    task "buildDockerImage_${dockerService.projectName}" (type: DockerBuildImage) {
+    tasks.register("buildDockerImage_${dockerService.projectName}", DockerBuildImage) {
         def hash = CommonUtils.calculateDockerHash(project.fileTree("docker/${dockerService.projectName}"))
         for (dep in dockerService.taskDependencies) {
             dependsOn dep
@@ -108,7 +113,7 @@ for (dockerService in dockerServices) {
     }
 }
 
-task buildDockerImages {
+tasks.register('buildDockerImages') {
     for (dockerService in dockerServices) {
         dependsOn "buildDockerImage_${dockerService.projectName}"
     }

RFS/build.gradle

@@ -18,28 +18,28 @@ configurations {
     lucene9
 }
 
-task shadowLucene5(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+tasks.register('shadowLucene5', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
     mergeServiceFiles()
     configurations = [project.configurations.lucene5]
     archiveClassifier.set('lucene5-shadow')
     relocate 'org.apache.lucene', 'shadow.lucene5.org.apache.lucene'
 }
 
-task shadowLucene6(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+tasks.register('shadowLucene6', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
     mergeServiceFiles()
     configurations = [project.configurations.lucene6]
     archiveClassifier.set('lucene6-shadow')
     relocate 'org.apache.lucene', 'shadow.lucene6.org.apache.lucene'
 }
 
-task shadowLucene7(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+tasks.register('shadowLucene7', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
     mergeServiceFiles()
     configurations = [project.configurations.lucene7]
     archiveClassifier.set('lucene7-shadow')
     relocate 'org.apache.lucene', 'shadow.lucene7.org.apache.lucene'
 }
 
-task shadowLucene9(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+tasks.register('shadowLucene9', com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
     mergeServiceFiles()
     archiveClassifier.set('lucene9-shadow')
     configurations = [project.configurations.lucene9]

TrafficCapture/dockerSolution/build.gradle

@@ -47,8 +47,7 @@ dockerFilesForExternalServices.each { dockerImageName, projectName ->
     // Now register the docker build task
     tasks.register("buildDockerImage_${dockerImageName}", DockerBuildImage) {
         if (projectName == migrationConsoleStagingDirName) {
-            // Depend on the base image from migrationConsole project
-            dependsOn ":migrationConsole:buildDockerImage_migrationConsoleBase"
+            CommonUtils.dependOnDockerImage(delegate, ":migrationConsole:buildDockerImage_migrationConsoleBase")
             dependsOn syncTaskProvider
 
             // Realize the task to get directory
@@ -91,15 +90,17 @@ static TaskProvider<Sync> getMigrationConsoleSyncTask(Project project, String do
             from (lib.files { lib.configurations.runtimeClasspath }) {
                 into "${applicationDestDir}/lib"
             }
-            from (lib.tasks.named('jar')) {
+            // Use map to get output files instead of task reference for configuration cache compatibility
+            from (lib.tasks.named('jar').map { it.outputs.files }) {
                 into "${applicationDestDir}/lib"
             }
         }
 
         // Sync application start scripts
         applications.each { app ->
             def applicationDestDir = "staging/${app.name}/"
-            from (app.tasks.named('startScripts').get().outputs.files) {
+            // Use map to get output files instead of task reference for configuration cache compatibility
+            from (app.tasks.named('startScripts').map { it.outputs.files }) {
                 into "${applicationDestDir}/bin"
             }
         }
@@ -108,10 +109,28 @@ static TaskProvider<Sync> getMigrationConsoleSyncTask(Project project, String do
         from (project.rootProject.layout.projectDirectory.file("VERSION"))
 
         def sourceDirName = sourceDirectoryOverrides.get(projectName, projectName)
-        from(project.layout.projectDirectory.dir("src/main/docker/${sourceDirName}").asFileTree.matching {
+        def sourceDir = project.layout.projectDirectory.dir("src/main/docker/${sourceDirName}")
+        from(sourceDir.asFileTree.matching {
             exclude("**/build/**", "**/.venv/**")
         })
 
+        // Explicit inputs for incremental build support
+        // Track the source Dockerfile and related files
+        inputs.dir(sourceDir)
+        inputs.file(project.rootProject.layout.projectDirectory.file("VERSION"))
+        // Track library and application JARs
+        // Use map to get output files instead of task reference for configuration cache compatibility
+        (libraries + applications).each { lib ->
+            inputs.files(lib.configurations.runtimeClasspath)
+            inputs.files(lib.tasks.named('jar').map { it.outputs.files })
+        }
+        // Track application start scripts
+        // Use map to get output files instead of task reference for configuration cache compatibility
+        applications.each { app ->
+            inputs.files(app.tasks.named('startScripts').map { it.outputs.files })
+        }
+        outputs.dir(destDir)
+
         // Collect assemble tasks from libraries and applications
         def assembleTasks = (libraries + applications).collect { it.tasks.named("assemble") }
         dependsOn assembleTasks
@@ -142,7 +161,7 @@ javaContainerServices.each { dockerImageName, projectName ->
 javaContainerServices.forEach { dockerImageName, gradleProjectName ->
     def projectName = project(gradleProjectName).name;
     def dockerBuildDir = "build/docker/${dockerImageName}_${projectName}"
-    task "buildDockerImage_${dockerImageName}"(type: DockerBuildImage) {
+    tasks.register("buildDockerImage_${dockerImageName}", DockerBuildImage) {
         dependsOn "createDockerfile_${dockerImageName}"
         inputDir = project.file("${dockerBuildDir}")
         images.add("migrations/${dockerImageName}:${version}".toString())
@@ -161,7 +180,7 @@ dockerCompose {
             ]
 }
 
-task buildDockerImages {
+tasks.register('buildDockerImages') {
     dependsOn tasks.named('buildDockerImage_k8s_config_map_util_scripts')
     dependsOn tasks.named('buildDockerImage_elasticsearch_searchguard')
     dependsOn tasks.named('buildDockerImage_migration_console')

build.gradle

@@ -74,7 +74,7 @@ allprojects {
     }
 }
 
-task buildDockerImages() {
+tasks.register('buildDockerImages') {
     dependsOn(':frontend:buildDockerImage')
     dependsOn(':TrafficCapture:dockerSolution:buildDockerImages')
     dependsOn(':DocumentsFromSnapshotMigration:buildDockerImages')
@@ -145,6 +145,14 @@ subprojects {
         }
     }
 
+    plugins.withId('com.bmuschko.docker-remote-api') {
+        tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage).configureEach {
+            // Declare the imageIdFile as an output so dependent tasks can track it as an input
+            // This enables proper UP_TO_DATE checking when using CommonUtils.dependOnDockerImage()
+            outputs.file(imageIdFile)
+        }
+    }
+
     tasks.withType(Javadoc).configureEach {
         options.addStringOption('Xdoclint:none', '-quiet')
     }
@@ -248,12 +256,13 @@ subprojects {
             }
         }
 
-        task javadocJar(type: Jar, dependsOn: javadoc) {
+        tasks.register('javadocJar', Jar) {
+            dependsOn javadoc
             archiveClassifier.set('javadoc')
             from javadoc.destinationDir
         }
 
-        task sourcesJar(type: Jar) {
+        tasks.register('sourcesJar', Jar) {
             archiveClassifier.set('sources')
             from sourceSets.main.allSource
             duplicatesStrategy = DuplicatesStrategy.WARN
@@ -499,13 +508,16 @@ def syncJacocoExecFiles = tasks.register("syncJacocoExecFiles", Sync) {
     dependsOn jacocoTestTasksProvider
 }
 
-tasks.register('gatherJacocoExecForMerging', Copy) {
+tasks.register('gatherJacocoExecForMerging', Sync) {
     description = 'Local troubleshooting, gathers coverage all exec files in build/jacocoMerged/*.exec into to be consumed by jacocoAggregateReport'
 
     from subprojects.collect { proj ->
         fileTree(dir: proj.buildDir, include: 'jacoco/*.exec')
     }
     into "$buildDir/jacocoMerged"
+
+    // Explicit outputs for incremental build support
+    outputs.dir("$buildDir/jacocoMerged")
 }
 
 tasks.register('mergeJacocoReports') {

buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy

@@ -1,12 +1,68 @@
 package org.opensearch.migrations.common
 
 import org.gradle.api.GradleException
-import org.gradle.api.tasks.Copy
+import org.gradle.api.Task
 import org.gradle.api.tasks.Sync
+import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.Project
 import com.bmuschko.gradle.docker.tasks.image.Dockerfile
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 
 class CommonUtils {
+
+    /**
+     * Adds a dependency on a DockerBuildImage task and tracks its imageIdFile as an input.
+     * This ensures that when the base image is rebuilt, the dependent task will also rebuild.
+     * The input property name is auto-generated from the base image task name.
+     *
+     * @param task The task that depends on the Docker image
+     * @param baseImageTaskPath The full task path (e.g., ":TrafficCapture:dockerSolution:buildDockerImage_elasticsearch_client_test_console")
+     */
+    static void dependOnDockerImage(Task task, String baseImageTaskPath) {
+        def rootProject = task.project.rootProject
+        // Parse the task path to get project path and task name
+        def lastColonIndex = baseImageTaskPath.lastIndexOf(':')
+        def projectPath = baseImageTaskPath.substring(0, lastColonIndex)
+        def taskName = baseImageTaskPath.substring(lastColonIndex + 1)
+        def targetProject = projectPath.isEmpty() ? rootProject : rootProject.project(projectPath)
+        def baseImageTaskProvider = targetProject.tasks.named(taskName, DockerBuildImage)
+        dependOnDockerImage(task, baseImageTaskProvider)
+    }
+
+    /**
+     * Adds a dependency on a DockerBuildImage task and tracks its imageIdFile as an input.
+     * This ensures that when the base image is rebuilt, the dependent task will also rebuild.
+     *
+     * @param task The task that depends on the Docker image
+     * @param baseImageTaskProvider The TaskProvider for the DockerBuildImage task to depend on
+     */
+    static void dependOnDockerImage(Task task, TaskProvider<DockerBuildImage> baseImageTaskProvider) {
+        def logger = task.project.logger
+        def baseTaskName = baseImageTaskProvider.name
+
+        task.dependsOn(baseImageTaskProvider)
+        // Use map to get the imageIdFile Provider for configuration cache compatibility
+        def imageIdFileProvider = baseImageTaskProvider.map { it.imageIdFile.get().asFile }
+        task.inputs.file(imageIdFileProvider).withPropertyName(baseTaskName)
+
+        // Add doFirst action to log the actual image ID at execution time
+        // Capture only serializable values (not the TaskProvider) for configuration cache compatibility
+        def taskPath = task.path
+        task.doFirst {
+            // Access the imageIdFile through the task's inputs to avoid capturing the TaskProvider
+            def imageIdFile = imageIdFileProvider.get()
+            if (imageIdFile.exists()) {
+                def imageId = imageIdFile.text.trim()
+                logger.lifecycle("Docker image dependency: {} depends on {} with imageId: {}",
+                        taskPath, baseTaskName, imageId)
+                logger.debug("  imageIdFile path: {}", imageIdFile.absolutePath)
+            } else {
+                logger.warn("Docker image dependency: {} depends on {} but imageIdFile does not exist: {}",
+                        taskPath, baseTaskName, imageIdFile.absolutePath)
+            }
+        }
+    }
+
     static def calculateDockerHash(def files) {
         def MessageDigest = java.security.MessageDigest
         def digest = MessageDigest.getInstance('SHA-256')
@@ -32,7 +88,8 @@ class CommonUtils {
                                                               String destProjectName, String destDir) {
         // Sync performs a copy, while also deleting items from the destination directory that are not in the source directory
         // In our case, jars of old versions were getting "stuck" and causing conflicts when the program was run
-        var dstCopyTask = dockerBuildProject.task("copyArtifact_${destProjectName}", type: Sync) {
+        // Using tasks.register() for lazy task configuration (configuration avoidance)
+        return dockerBuildProject.tasks.register("copyArtifact_${destProjectName}", Sync) {
             into destDir
             duplicatesStrategy = 'WARN'
 
@@ -41,23 +98,36 @@ class CommonUtils {
 
             // Copy jars
             // Lazily evaluate this runtimeClasspath
-            from(project.provider { sourceArtifactProject.configurations.runtimeClasspath }) {
+            from(dockerBuildProject.provider { sourceArtifactProject.configurations.runtimeClasspath }) {
                 include "*.jar"
                 into("jars")
             }
-            from(sourceArtifactProject.tasks.named('jar')) {
+            // Use map to get output files instead of task reference for configuration cache compatibility
+            from(sourceArtifactProject.tasks.named('jar').map { it.outputs.files }) {
                 include "*.jar"
                 into("jars")
             }
+
+            // Explicit inputs/outputs for incremental build support
+            inputs.file(dockerBuildProject.rootProject.layout.projectDirectory.file("VERSION"))
+            inputs.files(dockerBuildProject.provider { sourceArtifactProject.configurations.runtimeClasspath })
+            // Use map to get output files instead of task reference for configuration cache compatibility
+            inputs.files(sourceArtifactProject.tasks.named('jar').map { it.outputs.files })
+            outputs.dir(destDir)
+
+            dependsOn(sourceArtifactProject.tasks.named("assemble"))
+            dependsOn(sourceArtifactProject.tasks.named("build"))
         }
-        dstCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble"))
-        dstCopyTask.dependsOn(sourceArtifactProject.tasks.named("build"))
     }
 
     static def copyVersionFileToDockerStaging(Project project, String destProjectName, String destDir) {
-        return project.tasks.register("copyVersionFile_${destProjectName}", Copy) {
+        return project.tasks.register("copyVersionFile_${destProjectName}", Sync) {
             from(project.rootProject.layout.projectDirectory.file("VERSION"))
             into(project.layout.projectDirectory.dir(destDir))
+
+            // Explicit inputs/outputs for incremental build support
+            inputs.file(project.rootProject.layout.projectDirectory.file("VERSION"))
+            outputs.dir(project.layout.projectDirectory.dir(destDir))
         }
     }
 
@@ -71,7 +141,8 @@ class CommonUtils {
                                 String dockerImageName) {
         def projectName = sourceArtifactProject.name;
         def dockerBuildDir = "build/"+getDockerBuildDirName(dockerImageName, projectName);
-        dockerBuildProject.task("createDockerfile_${dockerImageName}", type: Dockerfile) {
+        // Using tasks.register() for lazy task configuration (configuration avoidance)
+        return dockerBuildProject.tasks.register("createDockerfile_${dockerImageName}", Dockerfile) {
             destFile = dockerBuildProject.file("${dockerBuildDir}/Dockerfile")
             dependsOn "copyArtifact_${dockerImageName}"
             if (baseImageProjectOverride) {
@@ -133,4 +204,3 @@ class CommonConfigurations {
         }
     }
 }
-

buildSrc/src/main/groovy/org/opensearch/migrations/image/RegistryImageBuildUtils.groovy

@@ -117,7 +117,8 @@ class RegistryImageBuildUtils {
         def imageTag = cfg.get("imageTag", "latest").toString()
         def contextDir = cfg.get("contextDir", ".")
         def serviceName = cfg.get("serviceName")
-        def contextPath = project.file(contextDir).path
+        def contextFile = project.file(contextDir)
+        def contextPath = contextFile.path
         def formatter = ImageRegistryFormatterFactory.getFormatter(registryEndpoint)
         def (registryDestination, cacheDestination) = formatter.getFullTargetImageIdentifier(registryEndpoint, imageName, imageTag)
 
@@ -126,6 +127,13 @@ class RegistryImageBuildUtils {
             group = "docker"
             description = "Build and push ${registryDestination} with caching"
 
+            // Add inputs for the Docker context directory to track file changes
+            inputs.dir(contextFile)
+
+            // Since outputs are pushed to a remote registry, we cannot track them locally.
+            // Always run this task to ensure the image is up-to-date with the registry.
+            outputs.upToDateWhen { false }
+
             cfg.get("requiredDependencies", []).each {
                 dependsOn it
             }