branch-4.1: [fix](timestamptz) Preserve correct DST fold branch to go cross the transition point #63034

be/src/core/value/timestamptz_value.cpp

@@ -133,4 +133,36 @@ void TimestampTzValue::convert_local_to_utc(const cctz::time_zone& local_time_zo
                                dt.microsecond());
 }
 
+int TimestampTzValue::utc_offset(const cctz::time_zone& local_time_zone) const {
+    cctz::civil_second utc_cs(_utc_dt.year(), _utc_dt.month(), _utc_dt.day(), _utc_dt.hour(),
+                              _utc_dt.minute(), _utc_dt.second());
+    cctz::time_point<cctz::seconds> utc_tp = cctz::convert(utc_cs, cctz::utc_time_zone());
+    return local_time_zone.lookup(utc_tp).offset;
+}
+
+void TimestampTzValue::convert_local_to_utc(const cctz::time_zone& local_time_zone,
+                                            const DateV2Value<DateTimeV2ValueType>& dt,
+                                            int preferred_offset) {
+    cctz::civil_second local_cs(dt.year(), dt.month(), dt.day(), dt.hour(), dt.minute(),
+                                dt.second());
+    const auto lookup = local_time_zone.lookup(local_cs);
+    cctz::time_point<cctz::seconds> local_tp = cctz::convert(local_cs, local_time_zone);
+
+    if (lookup.kind == cctz::time_zone::civil_lookup::REPEATED) {
+        const auto pre_offset = local_time_zone.lookup(lookup.pre).offset;
+        const auto post_offset = local_time_zone.lookup(lookup.post).offset;
+        if (preferred_offset == pre_offset) {
+            local_tp = lookup.pre;
+        } else if (preferred_offset == post_offset) {
+            local_tp = lookup.post;
+        }
+    }
+
+    auto utc_cs = cctz::convert(local_tp, cctz::utc_time_zone());
+    _utc_dt.unchecked_set_time((uint16_t)utc_cs.year(), (uint8_t)utc_cs.month(),
+                               (uint8_t)utc_cs.day(), (uint8_t)utc_cs.hour(),
+                               (uint8_t)utc_cs.minute(), (uint8_t)utc_cs.second(),
+                               dt.microsecond());
+}
+
 } // namespace doris

be/src/core/value/timestamptz_value.h

@@ -151,6 +151,11 @@ class TimestampTzValue {
     void convert_local_to_utc(const cctz::time_zone& local_time_zone,
                               const DateV2Value<DateTimeV2ValueType>& dt);
 
+    int utc_offset(const cctz::time_zone& local_time_zone) const;
+
+    void convert_local_to_utc(const cctz::time_zone& local_time_zone,
+                              const DateV2Value<DateTimeV2ValueType>& dt, int preferred_offset);
+
     TimestampTzValue& operator++() {
         ++_utc_dt;
         return *this;
@@ -197,4 +202,4 @@ struct std::hash<doris::TimestampTzValue> {
         auto int_val = v.to_date_int_val();
         return doris::HashUtil::hash(&int_val, sizeof(int_val), 0);
     }
-};
+};

be/src/exprs/function/function_datetime_floor_ceil.cpp

@@ -772,18 +772,13 @@ struct DateTimeFloorCeilCore {
         // For TimestampTzValue on date-based units, convert result from local time back to UTC
         if constexpr (need_tz_conversion) {
             if (result) {
-                cctz::civil_second local_result_cs(ts_res.year(), ts_res.month(), ts_res.day(),
-                                                   ts_res.hour(), ts_res.minute(), ts_res.second());
-                cctz::time_point<cctz::sys_seconds> local_tp = cctz::convert(local_result_cs, tz);
-                auto utc_result_cs = cctz::convert(local_tp, cctz::utc_time_zone());
-
-                ts_origin.unchecked_set_time(static_cast<uint16_t>(utc_result_cs.year()),
-                                             static_cast<uint8_t>(utc_result_cs.month()),
-                                             static_cast<uint8_t>(utc_result_cs.day()),
-                                             static_cast<uint8_t>(utc_result_cs.hour()),
-                                             static_cast<uint8_t>(utc_result_cs.minute()),
-                                             static_cast<uint8_t>(utc_result_cs.second()),
-                                             ts_res.microsecond());
+                DateV2Value<DateTimeV2ValueType> local_result(ts_res.to_date_int_val());
+                if constexpr (Flag::Unit == HOUR || Flag::Unit == MINUTE) {
+                    const int preferred_offset = ts_arg.utc_offset(tz);
+                    ts_origin.convert_local_to_utc(tz, local_result, preferred_offset);
+                } else {
+                    ts_origin.convert_local_to_utc(tz, local_result);
+                }
             }
         }
 

be/src/exprs/function/function_other_types_to_date.cpp

@@ -551,7 +551,13 @@ struct DateTrunc {
                 DateV2Value<DateTimeV2ValueType> local_dt;
                 dt.convert_utc_to_local(timezone, local_dt);
                 local_dt.template datetime_trunc<Unit>();
-                dt.convert_local_to_utc(timezone, local_dt);
+                if constexpr (Unit == TimeUnit::SECOND || Unit == TimeUnit::MINUTE ||
+                              Unit == TimeUnit::HOUR) {
+                    const int preferred_offset = dt.utc_offset(timezone);
+                    dt.convert_local_to_utc(timezone, local_dt, preferred_offset);
+                } else {
+                    dt.convert_local_to_utc(timezone, local_dt);
+                }
             } else {
                 dt.template datetime_trunc<Unit>();
             }

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java

@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.exceptions.CastException;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
+import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker;
 import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.DateTimeType;
 import org.apache.doris.nereids.types.DateTimeV2Type;
@@ -147,10 +148,19 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti
             return new DateTimeLiteral((DateTimeType) targetType, datetime.year, datetime.month, datetime.day,
                     datetime.hour, datetime.minute, datetime.second, datetime.microSecond);
         } else if (targetType.isTimeStampTzType()) {
+            // Explicit offsets must not round-trip through session local time; that loses the selected
+            // branch in DST fold hours. Wildcard targets still need a concrete scale before parsing.
+            TimeStampTzType timeStampTzType = (TimeStampTzType) targetType;
+            if (timeStampTzType.getScale() < 0) {
+                timeStampTzType = TimeStampTzType.forTypeFromString(value);
+            }
+            if (DateTimeChecker.hasTimeZone(value)) {
+                return new TimestampTzLiteral(timeStampTzType, value);
+            }
             DateTimeV2Literal expression = castToDateTime(DateTimeV2Type.MAX, strictCast, true);
             expression = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz(expression,
                     new StringLiteral(ConnectContext.get().getSessionVariable().timeZone), new StringLiteral("UTC")));
-            return new TimestampTzLiteral((TimeStampTzType) targetType, expression.year, expression.month,
+            return new TimestampTzLiteral(timeStampTzType, expression.year, expression.month,
                     expression.day, expression.hour, expression.minute, expression.second, expression.microSecond);
         } else if (targetType.isDateTimeV2Type()) {
             return castToDateTime(targetType, strictCast, true);

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java

@@ -24,12 +24,18 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.CharType;
 import org.apache.doris.nereids.types.DataType;
 import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.StringType;
 import org.apache.doris.nereids.types.TimeStampTzType;
+import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.util.DateUtils;
 import org.apache.doris.qe.ConnectContext;
 
 import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.Objects;
 
 /**
@@ -168,6 +174,15 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti
         if (targetType.isTimeStampTzType()) {
             return new TimestampTzLiteral((TimeStampTzType) targetType,
                     year, month, day, hour, minute, second, microSecond);
+        } else if (targetType.isCharType()) {
+            String desc = getStringValueInSessionTimeZone();
+            if (((CharType) targetType).getLen() >= desc.length()) {
+                return new CharLiteral(desc, ((CharType) targetType).getLen());
+            }
+        } else if (targetType.isVarcharType()) {
+            return new VarcharLiteral(getStringValueInSessionTimeZone(), ((VarcharType) targetType).getLen());
+        } else if (targetType instanceof StringType) {
+            return new StringLiteral(getStringValueInSessionTimeZone());
         } else if (targetType.isDateTimeV2Type()) {
             DateTimeV2Literal dtV2Lit = new DateTimeV2Literal((DateTimeV2Type) targetType,
                     year, month, day, hour, minute, second, microSecond);
@@ -180,6 +195,29 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti
         throw new AnalysisException(String.format("Cast from %s to %s not supported", this, targetType));
     }
 
+    private String getStringValueInSessionTimeZone() {
+        ZoneId sessionZone = DateUtils.getTimeZone();
+        ZonedDateTime localDateTime = toJavaDateType().atZone(ZoneId.of("UTC")).withZoneSameInstant(sessionZone);
+        String offset = localDateTime.getOffset().getId();
+        if ("Z".equals(offset)) {
+            offset = "+00:00";
+        }
+        return formatDateTime(localDateTime.toLocalDateTime()) + offset;
+    }
+
+    private String formatDateTime(LocalDateTime dateTime) {
+        String base = String.format("%04d-%02d-%02d %02d:%02d:%02d", dateTime.getYear(),
+                dateTime.getMonthValue(), dateTime.getDayOfMonth(), dateTime.getHour(),
+                dateTime.getMinute(), dateTime.getSecond());
+        int scale = getDataType().getScale();
+        if (scale <= 0) {
+            return base;
+        }
+        long scaledMicroSecond = (dateTime.getNano() / 1000)
+                / (int) Math.pow(10, TimeStampTzType.MAX_SCALE - scale);
+        return base + "." + String.format("%0" + scale + "d", scaledMicroSecond);
+    }
+
     public Expression plusDays(long days) {
         return fromJavaDateType(toJavaDateType().plusDays(days), getDataType().getScale());
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java

@@ -642,15 +642,26 @@ public static Optional<Expression> characterLiteralTypeCoercion(String value, Da
                     && DateTimeChecker.isValidDateTime(value)) {
                 ret = DateTimeLiteral.parseDateTimeLiteral(value, true).orElse(null);
             } else if (dataType.isTimeStampTzType() && DateTimeChecker.isValidDateTime(value)) {
-                DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) DateTimeLiteral
-                                .parseDateTimeLiteral(value, true).orElse(null);
-                if (dtV2Lit != null) {
-                    dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz(
-                            dtV2Lit,
-                            new StringLiteral(ConnectContext.get().getSessionVariable().timeZone),
-                            new StringLiteral("UTC")));
-                    ret = new TimestampTzLiteral(dtV2Lit.getYear(), dtV2Lit.getMonth(), dtV2Lit.getDay(),
-                            dtV2Lit.getHour(), dtV2Lit.getMinute(), dtV2Lit.getSecond(), dtV2Lit.getMicroSecond());
+                if (DateTimeChecker.hasTimeZone(value)) {
+                    // Signature search can pass TIMESTAMPTZ(*) here. TimestampTzLiteral rounds by scale,
+                    // so derive a concrete scale from the literal before preserving its explicit offset.
+                    TimeStampTzType timeStampTzType = (TimeStampTzType) dataType;
+                    if (timeStampTzType.getScale() < 0) {
+                        timeStampTzType = TimeStampTzType.forTypeFromString(value);
+                    }
+                    ret = new TimestampTzLiteral(timeStampTzType, value);
+                } else {
+                    DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) DateTimeLiteral
+                                    .parseDateTimeLiteral(value, true).orElse(null);
+                    if (dtV2Lit != null) {
+                        dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz(
+                                dtV2Lit,
+                                new StringLiteral(ConnectContext.get().getSessionVariable().timeZone),
+                                new StringLiteral("UTC")));
+                        ret = new TimestampTzLiteral(dtV2Lit.getYear(), dtV2Lit.getMonth(), dtV2Lit.getDay(),
+                                dtV2Lit.getHour(), dtV2Lit.getMinute(), dtV2Lit.getSecond(),
+                                dtV2Lit.getMicroSecond());
+                    }
                 }
             } else if ((dataType.isDateV2Type() || dataType.isDateType()) && DateTimeChecker.isValidDateTime(value)) {
                 Result<DateLiteral, AnalysisException> parseResult = DateV2Literal.parseDateLiteral(value, true);

fe/fe-core/src/test/java/org/apache/doris/nereids/util/TypeCoercionUtilsTest.java

@@ -63,10 +63,12 @@
 import org.apache.doris.nereids.types.SmallIntType;
 import org.apache.doris.nereids.types.StringType;
 import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.types.TimeStampTzType;
 import org.apache.doris.nereids.types.TimeV2Type;
 import org.apache.doris.nereids.types.TinyIntType;
 import org.apache.doris.nereids.types.VarcharType;
 import org.apache.doris.nereids.types.coercion.IntegralType;
+import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.ImmutableList;
 import org.junit.jupiter.api.Assertions;
@@ -286,6 +288,22 @@ public void testCharacterLiteralTypeCoercion() {
         // datetime
         Assertions.assertEquals(DateTimeV2Type.SYSTEM_DEFAULT,
                                 TypeCoercionUtils.characterLiteralTypeCoercion("2020-02-02", DateTimeType.INSTANCE).get().getDataType());
+        // timestamptz wildcard
+        Assertions.assertEquals(TimeStampTzType.SYSTEM_DEFAULT,
+                TypeCoercionUtils.characterLiteralTypeCoercion("2023-08-17T01:41:18Z", TimeStampTzType.WILDCARD)
+                        .get().getDataType());
+        // No-zone TIMESTAMPTZ coercion uses the session timezone to define the local civil time.
+        ConnectContext connectContext = new ConnectContext();
+        connectContext.getSessionVariable().setTimeZone("Asia/Shanghai");
+        connectContext.setThreadLocalInfo();
+        try {
+            // timestamptz without explicit timezone keeps the literal scale during signature search
+            Assertions.assertEquals(TimeStampTzType.SYSTEM_DEFAULT,
+                    TypeCoercionUtils.characterLiteralTypeCoercion("2004-12-31", TimeStampTzType.MAX)
+                            .get().getDataType());
+        } finally {
+            ConnectContext.remove();
+        }
     }
 
     @Test

regression-test/data/datatype_p0/timestamptz/test_timestamptz_dst_fold.out

@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	pre_fold_utc	2024-11-03 01:05:00.000000-04:00
+2	post_fold_utc	2024-11-03 01:05:00.000000-05:00
+3	pre_explicit	2024-11-03 01:05:00.000000-04:00
+4	post_explicit	2024-11-03 01:05:00.000000-05:00
+
+-- !sql --
+2	post_fold_utc
+4	post_explicit
+
+-- !sql --
+4	2024-11-03 01:00:00.000000-05:00	2024-11-03 01:10:00.000000-05:00	2024-11-03 01:00:00.000000-05:00	2024-11-03 01:05:00.000000-05:00	2024-11-03 01:05:00.000000-05:00	2024-11-03 01:00:00.000000-05:00
+
+-- !sql --
+2024-11-03 06:05:00.000000+00:00	2024-11-03 06:05:00.000000+00:00	2024-11-03 06:00:00.000000+00:00
+

regression-test/data/datatype_p0/timestamptz/test_timestamptz_storage_agg_key.out

@@ -458,6 +458,15 @@
 2025-12-12 12:12:12.123456+08:00	3023-09-09 16:09:09.000000+08:00	3023-09-09 16:09:09.000000+08:00	1023-09-09 16:09:09.000000+08:00	3023-09-09 16:09:09.000000+08:00
 
 -- !scale_not_in --
+0000-01-01 08:00:00.123456+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00
+0000-01-01 08:00:00.999999+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00
+2025-12-12 12:12:12.000000+08:00	3023-09-09 16:09:09.000000+08:00	2023-09-09 16:09:09.000000+08:00	2023-09-09 16:09:09.000000+08:00	2023-09-09 16:09:09.000000+08:00
+2025-12-12 12:12:12.000001+08:00	3023-09-09 16:09:09.000000+08:00	3023-09-09 16:09:09.000000+08:00	1023-09-09 16:09:09.000000+08:00	3023-09-09 16:09:09.000000+08:00
+2025-12-12 12:12:12.999999+08:00	3023-09-09 16:09:09.000000+08:00	3023-09-09 16:09:09.000000+08:00	1023-09-09 16:09:09.000000+08:00	3023-09-09 16:09:09.000000+08:00
+9999-12-31 23:59:59.000000+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00	0000-01-01 08:00:00.000000+08:00	2023-04-05 07:59:59.000000+08:00
+9999-12-31 23:59:59.000001+08:00	0000-01-01 08:00:00.000000+08:00	9999-12-31 23:59:59.999999+08:00	0000-01-01 08:00:00.000000+08:00	9999-12-31 23:59:59.999999+08:00
+9999-12-31 23:59:59.123456+08:00	0000-01-01 08:00:00.000000+08:00	9999-12-31 23:59:59.999999+08:00	0000-01-01 08:00:00.000000+08:00	9999-12-31 23:59:59.999999+08:00
+9999-12-31 23:59:59.999999+08:00	0000-01-01 08:00:00.000000+08:00	9999-12-31 23:59:59.999999+08:00	0000-01-01 08:00:00.000000+08:00	9999-12-31 23:59:59.999999+08:00
 
 -- !scale_is_null --
 \N	2025-12-12 12:12:12.000000+08:00	2025-12-12 12:12:12.000000+08:00	0000-01-01 08:00:00.000000+08:00	2025-12-12 12:12:12.000000+08:00

regression-test/data/doc/sql-manual/sql-functions/doc_date_functions_test.out

@@ -2097,7 +2097,7 @@ da fanadur
 1196389819
 
 -- !unix_timestamp_4 --
-1196386219.000000
+1196386219
 
 -- !unix_timestamp_5 --
 1196389819.000000

regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.out

@@ -121,6 +121,7 @@
 
 -- !get_format_time_4 --
 \N
+
 -- !sql_addtime1 --
 2023-10-14T22:35:22
 
@@ -154,10 +155,10 @@
 2025-06-30T17:15:30.999999
 
 -- !sql_addtime9 --
-2025-10-10T17:24:36+08:00
+2025-10-10 17:24:36+08:00
 
 -- !sql_addtime10 --
-2025-10-10T17:24:36.123457+08:00
+2025-10-10 17:24:36.123457+08:00
 
 -- !sql_addtime11 --
 44:22:32
@@ -220,3 +221,4 @@
 
 -- !sql_subtime14 --
 0001-01-01 07:59:59.999999+08:00
+