From 7cbb2b99f19cc1b67b7a07cae79a6f3d4d2f100f Mon Sep 17 00:00:00 2001 From: rayshoo Date: Sun, 31 Aug 2025 13:15:56 +0000 Subject: [PATCH 1/2] fix: IMAGE_TAG env was fixed first starting from the second architecture build. --- pkg/skaffold/build/cluster/kaniko.go | 23 +++++++++++++++++------ pkg/skaffold/build/cluster/pod.go | 8 ++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/pkg/skaffold/build/cluster/kaniko.go b/pkg/skaffold/build/cluster/kaniko.go index 95c583dd845..0d5666ee21d 100644 --- a/pkg/skaffold/build/cluster/kaniko.go +++ b/pkg/skaffold/build/cluster/kaniko.go @@ -64,17 +64,28 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace if err != nil { return "", fmt.Errorf("error processing generated env variables from image uri: %w", err) } - env, err := evaluateEnv(artifact.Env, generatedEnvs...) + + kaCopy := *artifact + + evaluatedEnv, err := evaluateEnv(artifact.Env, generatedEnvs...) if err != nil { return "", fmt.Errorf("unable to evaluate env variables: %w", err) } - artifact.Env = env - buildArgs, err := docker.EvalBuildArgsWithEnv(b.cfg.Mode(), kaniko.GetContext(artifact, workspace), artifact.DockerfilePath, artifact.BuildArgs, requiredImages, envMapFromVars(artifact.Env)) + kaCopy.Env = append([]v1.EnvVar(nil), evaluatedEnv...) + + buildArgs, err := docker.EvalBuildArgsWithEnv( + b.cfg.Mode(), + kaniko.GetContext(&kaCopy, workspace), + kaCopy.DockerfilePath, + kaCopy.BuildArgs, + requiredImages, + envMapFromVars(kaCopy.Env), + ) if err != nil { return "", fmt.Errorf("unable to evaluate build args: %w", err) } - artifact.BuildArgs = buildArgs + kaCopy.BuildArgs = buildArgs client, err := kubernetesclient.DefaultClient() if err != nil { @@ -82,7 +93,7 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace } pods := client.CoreV1().Pods(b.Namespace) - podSpec, err := b.kanikoPodSpec(artifact, tag, platforms) + podSpec, err := b.kanikoPodSpec(&kaCopy, tag, platforms) if err != nil { return "", err } @@ -106,7 +117,7 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace } }() - if err := b.setupKanikoBuildContext(ctx, out, workspace, artifactName, artifact, pods, pod.Name); err != nil { + if err := b.setupKanikoBuildContext(ctx, out, workspace, artifactName, &kaCopy, pods, pod.Name); err != nil { return "", fmt.Errorf("copying sources: %w", err) } diff --git a/pkg/skaffold/build/cluster/pod.go b/pkg/skaffold/build/cluster/pod.go index 75d329dd2f3..9022a6963f1 100644 --- a/pkg/skaffold/build/cluster/pod.go +++ b/pkg/skaffold/build/cluster/pod.go @@ -141,12 +141,8 @@ func (b *Builder) kanikoPodSpec(artifact *latest.KanikoArtifact, tag string, pla if pod.Spec.NodeSelector == nil { pod.Spec.NodeSelector = make(map[string]string) } - if _, found := pod.Spec.NodeSelector[nodeArchitectureLabel]; !found { - pod.Spec.NodeSelector[nodeArchitectureLabel] = platforms.Platforms[0].Architecture - } - if _, found := pod.Spec.NodeSelector[nodeOperatingSystemLabel]; !found { - pod.Spec.NodeSelector[nodeOperatingSystemLabel] = platforms.Platforms[0].OS - } + pod.Spec.NodeSelector[nodeArchitectureLabel] = platforms.Platforms[0].Architecture + pod.Spec.NodeSelector[nodeOperatingSystemLabel] = platforms.Platforms[0].OS } // Add used-defines Volumes From 1bedb77f045b2151b5ca58ecba66b339f580d0d2 Mon Sep 17 00:00:00 2001 From: rayshoo Date: Tue, 2 Sep 2025 03:50:44 +0000 Subject: [PATCH 2/2] fix: use --custom-platform in Kaniko to pull correct arch for multi-platform base images Kaniko was defaulting to amd64 when building with multi-platform base images, causing runtime errors on arm64. Adding --custom-platform ensures the correct architecture image is pulled during the build, so arm64 images now build and run successfully. --- pkg/skaffold/build/cluster/pod.go | 3 +++ pkg/skaffold/build/cluster/pod_test.go | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/skaffold/build/cluster/pod.go b/pkg/skaffold/build/cluster/pod.go index 9022a6963f1..7435f8ebfb1 100644 --- a/pkg/skaffold/build/cluster/pod.go +++ b/pkg/skaffold/build/cluster/pod.go @@ -43,6 +43,9 @@ func (b *Builder) kanikoPodSpec(artifact *latest.KanikoArtifact, tag string, pla if err != nil { return nil, fmt.Errorf("building args list: %w", err) } + if len(platforms.Platforms) == 1 { + args = append(args, fmt.Sprintf("--custom-platform=%s/%s", platforms.Platforms[0].OS, platforms.Platforms[0].Architecture)) + } vm := v1.VolumeMount{ Name: kaniko.DefaultEmptyDirName, diff --git a/pkg/skaffold/build/cluster/pod_test.go b/pkg/skaffold/build/cluster/pod_test.go index 3a218ee5f0a..80c8e6ae72f 100644 --- a/pkg/skaffold/build/cluster/pod_test.go +++ b/pkg/skaffold/build/cluster/pod_test.go @@ -17,6 +17,7 @@ limitations under the License. package cluster import ( + "fmt" "testing" specs "github.com/opencontainers/image-spec/specs-go/v1" @@ -265,6 +266,10 @@ func TestKanikoPodSpec(t *testing.T) { }, } matcher := platform.Matcher{Platforms: []specs.Platform{{OS: "linux", Architecture: "arm64"}}} + args := []string{"--destination", "tag", "--dockerfile", "Dockerfile", "--context", "dir:///kaniko/buildcontext", "--destination", "gcr.io/foo/bar:test-1", "--destination", "gcr.io/foo/bar:test-2"} + if len(matcher.Platforms) == 1 { + args = append(args, fmt.Sprintf("--custom-platform=%s/%s", matcher.Platforms[0].OS, matcher.Platforms[0].Architecture)) + } pod, _ := builder.kanikoPodSpec(artifact, "tag", matcher) expectedPod := &v1.Pod{ @@ -305,7 +310,7 @@ func TestKanikoPodSpec(t *testing.T) { Containers: []v1.Container{{ Name: kaniko.DefaultContainerName, Image: "image", - Args: []string{"--destination", "tag", "--dockerfile", "Dockerfile", "--context", "dir:///kaniko/buildcontext", "--destination", "gcr.io/foo/bar:test-1", "--destination", "gcr.io/foo/bar:test-2"}, + Args: args, ImagePullPolicy: v1.PullIfNotPresent, Env: []v1.EnvVar{{ Name: "UPSTREAM_CLIENT_TYPE",