From 3030f14a3513302c8bc059e6cba89f10061d645c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 8 Apr 2026 15:59:14 -0600 Subject: [PATCH] fix: use min instead of max when capping write buffer size to Int range (#3914) COMET_SHUFFLE_WRITE_BUFFER_SIZE is a Long (bytesConf) but the protobuf field is int32, so the value must be capped at Int.MaxValue. The code used .max(Int.MaxValue) which always returns Int.MaxValue (~2GB) regardless of the configured value. Should be .min(Int.MaxValue) to preserve smaller values while capping at the Int range. --- .../sql/comet/execution/shuffle/CometNativeShuffleWriter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index 3fc222bd19..d704d3fd88 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -191,7 +191,7 @@ class CometNativeShuffleWriter[K, V]( shuffleWriterBuilder.setCompressionLevel( CometConf.COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL.get) shuffleWriterBuilder.setWriteBufferSize( - CometConf.COMET_SHUFFLE_WRITE_BUFFER_SIZE.get().max(Int.MaxValue).toInt) + CometConf.COMET_SHUFFLE_WRITE_BUFFER_SIZE.get().min(Int.MaxValue).toInt) outputPartitioning match { case p if isSinglePartitioning(p) =>