MDEV-39451 Accounting for truncation to value_max_length in the ntile comparison

mysql-test/main/win_ntile.result

@@ -511,3 +511,13 @@ update t1 set c3= 1 where pk = 1;
 select c1, c2, c3, ntile(c3) over (partition by c2 order by pk) from t1;
 ERROR HY000: Argument of NTILE must be greater than 0
 drop table t1;
+#
+# MDEV-39451 Floating point exception: division by zero in Item_sum_ntile::val_int
+#
+CREATE TABLE t (c2 TEXT CHARACTER SET 'Binary' COLLATE 'Binary');
+INSERT INTO t VALUES (REPEAT('a',1026)),(REPEAT('a',1026));
+SELECT NTILE(2)OVER (PARTITION BY c2 ORDER BY c2) FROM t;
+NTILE(2)OVER (PARTITION BY c2 ORDER BY c2)
+1
+2
+DROP TABLE t;

mysql-test/main/win_ntile.test

@@ -212,3 +212,12 @@ update t1 set c3= 1 where pk = 1;
 select c1, c2, c3, ntile(c3) over (partition by c2 order by pk) from t1;
 
 drop table t1;
+
+--echo #
+--echo # MDEV-39451 Floating point exception: division by zero in Item_sum_ntile::val_int
+--echo #
+
+CREATE TABLE t (c2 TEXT CHARACTER SET 'Binary' COLLATE 'Binary');
+INSERT INTO t VALUES (REPEAT('a',1026)),(REPEAT('a',1026));
+SELECT NTILE(2)OVER (PARTITION BY c2 ORDER BY c2) FROM t;
+DROP TABLE t;

sql/item_buff.cc

@@ -101,7 +101,10 @@ bool Cached_item_str::cmp(void)
 
 int Cached_item_str::cmp_read_only()
 {
-  String *res= item->val_str(&tmp_value);
+  String res;
+  if (String *tmp_res= item->val_str(&tmp_value))
+    res.set(tmp_res->ptr(), MY_MIN(tmp_res->length(), value_max_length),
+            tmp_res->charset());
 
   if (null_value)
   {
@@ -113,7 +116,7 @@ int Cached_item_str::cmp_read_only()
   if (item->null_value)
     return 1;
 
-  return sortcmp(&value, res, item->collation.collation);
+  return sortcmp(&value, &res, item->collation.collation);
 }
 
 

sql/item_windowfunc.h

@@ -685,11 +685,17 @@ class Item_sum_ntile : public Item_sum_int,
 
   longlong val_int() override
   {
-    if (get_row_count() == 0)
+    if (partition_row_count_ == 0)
     {
       null_value= true;
       return 0;
     }
+    /*
+      The current row count in the partition should not exceed the
+      total row count of the partition
+    */
+    DBUG_ASSERT(current_row_count_ <= partition_row_count_);
+    DBUG_ASSERT(current_row_count_ > 0);
 
     longlong num_quantiles= get_num_quantiles();
 
@@ -701,9 +707,14 @@ class Item_sum_ntile : public Item_sum_int,
     }
     n_old_val_= static_cast<ulonglong>(num_quantiles);
     null_value= false;
-    ulonglong quantile_size = get_row_count() / num_quantiles;
-    ulonglong extra_rows = get_row_count() - quantile_size * num_quantiles;
+    ulonglong quantile_size = partition_row_count_ / num_quantiles;
+    ulonglong extra_rows = partition_row_count_ - quantile_size * num_quantiles;
 
+    /*
+      Say there are n extra rows i.e. extra_rows == n, then each of
+      these n rows is placed in the first n tiles, effectively
+      incrementing the size of the first n tiles by 1.
+    */
     if (current_row_count_ <= extra_rows * (quantile_size + 1))
       return (current_row_count_ - 1) / (quantile_size + 1) + 1;