MDEV-41020 A MEMORY table refuses a row it just held

mysql-test/main/func_gconcat.result

@@ -863,7 +863,7 @@ group_concat(distinct a, c)
 00,01,10,11,31
 select group_concat(distinct a, c order by a) from t1;
 group_concat(distinct a, c order by a)
-00,01,11,10,31
+01,00,11,10,31
 select group_concat(distinct a, c) from t1;
 group_concat(distinct a, c)
 00,01,10,11,31
@@ -1169,21 +1169,20 @@ LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
 1024
 Warnings:
 Warning	1260	Row 2 was cut by group_concat()
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
 SET group_concat_max_len= 499999;
 SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 WHERE f2 = 0;
 LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
 499999
 Warnings:
-Warning	1260	Row 1 was cut by group_concat()
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
 SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
 LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
 499999
 499999
 499999
 Warnings:
-Warning	1260	Row 1 was cut by group_concat()
-Warning	1260	Row 2 was cut by group_concat()
-Warning	1260	Row 3 was cut by group_concat()
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
 INSERT INTO t1 VALUES (REPEAT('a', 499999), 3), (REPEAT('b', 500000), 4);
 SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
 LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
@@ -1193,10 +1192,7 @@ LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
 499999
 499999
 Warnings:
-Warning	1260	Row 1 was cut by group_concat()
-Warning	1260	Row 2 was cut by group_concat()
-Warning	1260	Row 3 was cut by group_concat()
-Warning	1260	Row 5 was cut by group_concat()
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
 DROP TABLE t1;
 SET group_concat_max_len= DEFAULT;
 set session group_concat_max_len=1024;

mysql-test/main/gconcat_cut_note.result

@@ -0,0 +1,63 @@
+CREATE TABLE t1 (grp INT, t TEXT);
+INSERT INTO t1 SELECT seq, REPEAT(CHAR(64 + seq), 200) FROM seq_1_to_3;
+SET SESSION group_concat_max_len= 100;
+#
+# One row per group, so the result of each group is exactly the cut
+# value and is never longer than the limit. Nothing is missing from
+# any answer, and the three cut values give one note.
+#
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
+grp	len
+1	100
+2	100
+3	100
+Warnings:
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
+#
+# Two aggregates in one statement, so one note each. The brackets
+# JSON_ARRAYAGG() puts around the value push its result past the
+# limit, so that one loses data as well and warns for it.
+#
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS gc_len,
+LENGTH(JSON_ARRAYAGG(t ORDER BY t)) AS ja_len FROM t1 GROUP BY grp;
+grp	gc_len	ja_len
+1	100	102
+2	100	102
+3	100	102
+Warnings:
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
+Warning	1260	Row 1 was cut by json_arrayagg()
+Note	4265	Some values were cut while processing json_arrayagg(). Increase group_concat_max_len if you want to avoid the cut
+Warning	1260	Row 2 was cut by json_arrayagg()
+Warning	1260	Row 3 was cut by json_arrayagg()
+#
+# The same statement run again gets its own note.
+#
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
+grp	len
+1	100
+2	100
+3	100
+Warnings:
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
+#
+# All three rows in one group. Now the result is cut as well, and
+# values the user asked for really are missing, so the warning is
+# given for that on top of the note.
+#
+SELECT LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1;
+len
+100
+Warnings:
+Warning	1260	Row 2 was cut by group_concat()
+Note	4265	Some values were cut while processing group_concat(). Increase group_concat_max_len if you want to avoid the cut
+#
+# Nothing is cut, so nothing is said.
+#
+SET SESSION group_concat_max_len= DEFAULT;
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
+grp	len
+1	200
+2	200
+3	200
+DROP TABLE t1;

mysql-test/main/gconcat_cut_note.test

@@ -0,0 +1,47 @@
+#
+# A TEXT value longer than group_concat_max_len is cut on its way into
+# blob_storage, while the group is being built. Whether that changed the
+# answer is not known: the result may have been cut in the same place
+# anyway. So it is a note, and one note is enough for the statement.
+#
+--source include/have_sequence.inc
+
+CREATE TABLE t1 (grp INT, t TEXT);
+INSERT INTO t1 SELECT seq, REPEAT(CHAR(64 + seq), 200) FROM seq_1_to_3;
+
+SET SESSION group_concat_max_len= 100;
+
+--echo #
+--echo # One row per group, so the result of each group is exactly the cut
+--echo # value and is never longer than the limit. Nothing is missing from
+--echo # any answer, and the three cut values give one note.
+--echo #
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
+
+--echo #
+--echo # Two aggregates in one statement, so one note each. The brackets
+--echo # JSON_ARRAYAGG() puts around the value push its result past the
+--echo # limit, so that one loses data as well and warns for it.
+--echo #
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS gc_len,
+       LENGTH(JSON_ARRAYAGG(t ORDER BY t)) AS ja_len FROM t1 GROUP BY grp;
+
+--echo #
+--echo # The same statement run again gets its own note.
+--echo #
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
+
+--echo #
+--echo # All three rows in one group. Now the result is cut as well, and
+--echo # values the user asked for really are missing, so the warning is
+--echo # given for that on top of the note.
+--echo #
+SELECT LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1;
+
+--echo #
+--echo # Nothing is cut, so nothing is said.
+--echo #
+SET SESSION group_concat_max_len= DEFAULT;
+SELECT grp, LENGTH(GROUP_CONCAT(t ORDER BY t)) AS len FROM t1 GROUP BY grp;
+
+DROP TABLE t1;

mysql-test/main/gconcat_distinct_rewalk.result

@@ -0,0 +1,93 @@
+#
+# A plain aggregate, asked once and then asked twice. A HAVING
+# clause on the alias is the shortest statement that asks twice.
+#
+CREATE TABLE t1 (g INT, a VARCHAR(10));
+INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f');
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1;
+v
+
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+
+#
+# Two conditions ask for it a third time.
+#
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%' AND v NOT LIKE 'zz%';
+v
+
+#
+# DISTINCT reaches the walk by the other route.
+#
+SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+
+#
+# One group spends the offset exactly and the other does not, so
+# both answers show up in the same statement.
+#
+SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1
+GROUP BY g HAVING v LIKE '%' ORDER BY g;
+g	v
+1	
+2	
+#
+# An offset that stops inside the group is not affected, the walk
+# having written a row, and neither is a group with no LIMIT.
+#
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+b,c
+SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+a,b,c,d
+DROP TABLE t1;
+#
+# The same replay against a duplicate filter that has spilled to
+# disk, where the second walk is not merely wrong but unsupported.
+#
+CREATE TABLE t2 (g INT, a VARCHAR(100));
+INSERT INTO t2 SELECT seq MOD 4, LPAD(seq, 100, '0') FROM seq_1_to_2000;
+SELECT COUNT(*) AS rows_in_table, COUNT(DISTINCT a) AS distinct_values FROM t2;
+rows_in_table	distinct_values
+2000	2000
+SET @@tmp_memory_table_size=0;
+SELECT g, GROUP_CONCAT(DISTINCT a LIMIT 5 OFFSET 1000) AS gc
+FROM t2 GROUP BY g HAVING gc <> 'x';
+g	gc
+0	
+1	
+2	
+3	
+SELECT g, GROUP_CONCAT(DISTINCT a ORDER BY a LIMIT 5 OFFSET 1000) AS gc
+FROM t2 GROUP BY g HAVING gc <> 'x';
+g	gc
+0	
+1	
+2	
+3	
+#
+# An offset that skips only part of the group is not affected, and
+# the answer must not depend on whether the filter spilled.
+#
+SELECT g, LENGTH(GROUP_CONCAT(DISTINCT a ORDER BY a LIMIT 2 OFFSET 3)) AS gc_len
+FROM t2 GROUP BY g HAVING gc_len > 0;
+g	gc_len
+0	201
+1	201
+2	201
+3	201
+SET @@tmp_memory_table_size=DEFAULT;
+SELECT g, LENGTH(GROUP_CONCAT(DISTINCT a ORDER BY a LIMIT 2 OFFSET 3)) AS gc_len
+FROM t2 GROUP BY g HAVING gc_len > 0;
+g	gc_len
+0	201
+1	201
+2	201
+3	201
+DROP TABLE t2;

mysql-test/main/gconcat_distinct_rewalk.test

@@ -0,0 +1,75 @@
+#
+# Nothing says how many times a statement asks for the result of a group,
+# and the answer must not depend on it. When an OFFSET skips every row of
+# the group, the walk writes nothing, and the aggregate must still record
+# that it has run. Otherwise the next caller walks again with the offset
+# already spent, which replays the group into a buffer that was handed
+# over once already, and, once the duplicate filter has spilled to disk,
+# walks a Unique that can only be walked once.
+#
+--source include/have_sequence.inc
+
+--echo #
+--echo # A plain aggregate, asked once and then asked twice. A HAVING
+--echo # clause on the alias is the shortest statement that asks twice.
+--echo #
+CREATE TABLE t1 (g INT, a VARCHAR(10));
+INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f');
+
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1;
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+
+--echo #
+--echo # Two conditions ask for it a third time.
+--echo #
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%' AND v NOT LIKE 'zz%';
+
+--echo #
+--echo # DISTINCT reaches the walk by the other route.
+--echo #
+SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+
+--echo #
+--echo # One group spends the offset exactly and the other does not, so
+--echo # both answers show up in the same statement.
+--echo #
+SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1
+  GROUP BY g HAVING v LIKE '%' ORDER BY g;
+
+--echo #
+--echo # An offset that stops inside the group is not affected, the walk
+--echo # having written a row, and neither is a group with no LIMIT.
+--echo #
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+DROP TABLE t1;
+
+--echo #
+--echo # The same replay against a duplicate filter that has spilled to
+--echo # disk, where the second walk is not merely wrong but unsupported.
+--echo #
+CREATE TABLE t2 (g INT, a VARCHAR(100));
+INSERT INTO t2 SELECT seq MOD 4, LPAD(seq, 100, '0') FROM seq_1_to_2000;
+SELECT COUNT(*) AS rows_in_table, COUNT(DISTINCT a) AS distinct_values FROM t2;
+
+SET @@tmp_memory_table_size=0;
+SELECT g, GROUP_CONCAT(DISTINCT a LIMIT 5 OFFSET 1000) AS gc
+  FROM t2 GROUP BY g HAVING gc <> 'x';
+SELECT g, GROUP_CONCAT(DISTINCT a ORDER BY a LIMIT 5 OFFSET 1000) AS gc
+  FROM t2 GROUP BY g HAVING gc <> 'x';
+
+--echo #
+--echo # An offset that skips only part of the group is not affected, and
+--echo # the answer must not depend on whether the filter spilled.
+--echo #
+SELECT g, LENGTH(GROUP_CONCAT(DISTINCT a ORDER BY a LIMIT 2 OFFSET 3)) AS gc_len
+  FROM t2 GROUP BY g HAVING gc_len > 0;
+SET @@tmp_memory_table_size=DEFAULT;
+SELECT g, LENGTH(GROUP_CONCAT(DISTINCT a ORDER BY a LIMIT 2 OFFSET 3)) AS gc_len
+  FROM t2 GROUP BY g HAVING gc_len > 0;
+DROP TABLE t2;