diff --git a/mysql-test/main/rowid_filter_innodb,ahi.rdiff b/mysql-test/main/rowid_filter_innodb,ahi.rdiff index 9d0b7fc137c3c..48fee5cc52d07 100644 --- a/mysql-test/main/rowid_filter_innodb,ahi.rdiff +++ b/mysql-test/main/rowid_filter_innodb,ahi.rdiff @@ -1,5 +1,5 @@ --- rowid_filter_innodb.result -+++ rowid_filter_innodb,ahi.result ++++ rowid_filter_innodb,ahi.reject @@ -2,7 +2,7 @@ SET @save_stats_persistent=@@GLOBAL.innodb_stats_persistent; SET GLOBAL innodb_stats_persistent=0; @@ -9,20 +9,20 @@ DROP DATABASE IF EXISTS dbt3_s001; CREATE DATABASE dbt3_s001; use dbt3_s001; -@@ -2059,7 +2059,7 @@ +@@ -2061,7 +2061,7 @@ "r_table_time_ms": "REPLACED", "r_other_time_ms": "REPLACED", "r_engine_stats": { -- "pages_accessed": 90 +- "pages_accessed": 81 + "pages_accessed": 51 }, "filtered": "REPLACED", "r_total_filtered": 2.43902439, -@@ -2227,7 +2227,7 @@ +@@ -2229,7 +2229,7 @@ "r_table_time_ms": "REPLACED", "r_other_time_ms": "REPLACED", "r_engine_stats": { -- "pages_accessed": 90 +- "pages_accessed": 81 + "pages_accessed": 49 }, "filtered": "REPLACED", diff --git a/mysql-test/main/rowid_filter_innodb.result b/mysql-test/main/rowid_filter_innodb.result index bb01a1cfd78b1..f385ecb6d9c4e 100644 --- a/mysql-test/main/rowid_filter_innodb.result +++ b/mysql-test/main/rowid_filter_innodb.result @@ -2061,7 +2061,7 @@ ANALYZE "r_table_time_ms": "REPLACED", "r_other_time_ms": "REPLACED", "r_engine_stats": { - "pages_accessed": 90 + "pages_accessed": 81 }, "filtered": "REPLACED", "r_total_filtered": 2.43902439, @@ -2229,7 +2229,7 @@ ANALYZE "r_table_time_ms": "REPLACED", "r_other_time_ms": "REPLACED", "r_engine_stats": { - "pages_accessed": 90 + "pages_accessed": 81 }, "filtered": "REPLACED", "r_total_filtered": 2.43902439, diff --git a/mysql-test/suite/innodb/r/non_covering_sec_idx_scan.result b/mysql-test/suite/innodb/r/non_covering_sec_idx_scan.result new file mode 100644 index 0000000000000..4e304158ab828 --- /dev/null +++ b/mysql-test/suite/innodb/r/non_covering_sec_idx_scan.result @@ -0,0 +1,259 @@ +create table t1 ( +pk int not null primary key, +domain_grp int, +val int +) engine=innodb row_format=dynamic; +insert into t1 select seq, mod(seq,10), seq from seq_1_to_10000; +create index domain_idx on t1(domain_grp); +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +set @js='$out_scan'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_FULL_SCAN; +PAGES_ACCESSED_FULL_SCAN +23 +set @js='$out_idx'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SEC_INDEX; +PAGES_ACCESSED_SEC_INDEX +1028 +drop table t1; +create table t2 ( +pk varchar(500) character set utf8mb4 collate utf8mb4_general_ci +not null primary key, +domain_grp int, +val int +) engine=innodb row_format=dynamic; +insert into t2 select lpad(seq,500,'0'), mod(seq,4), seq from seq_1_to_3000; +create index domain_idx on t2(domain_grp); +analyze table t2 persistent for all; +Table Op Msg_type Msg_text +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status OK +select stat_value into @leaf from mysql.innodb_index_stats +where database_name='test' and table_name='t2' and index_name='PRIMARY' + and stat_name='n_leaf_pages'; +select @leaf > 100 as THREE_LEVEL_TREE; +THREE_LEVEL_TREE +1 +set @js='$out_scan2'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_FULL_SCAN_3LEVEL; +PAGES_ACCESSED_FULL_SCAN_3LEVEL +110 +set @js='$out_idx2'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SEC_INDEX_3LEVEL; +PAGES_ACCESSED_SEC_INDEX_3LEVEL +1007 +drop table t2; +create table t3 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t3 select seq, mod(seq*997+13,10007), seq from seq_1_to_10000; +create index k_idx on t3(k); +analyze table t3 persistent for all; +Table Op Msg_type Msg_text +test.t3 analyze status Engine-independent statistics collected +test.t3 analyze status OK +set @js='$out_uncorr'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_UNCORRELATED; +PAGES_ACCESSED_UNCORRELATED +15780 +drop table t3; +create table t4 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t4 select seq, 2*seq-1, seq from seq_1_to_1000; +insert into t4 select 1000000+seq, 2*seq, seq from seq_1_to_1000; +create index k_idx on t4(k); +analyze table t4 persistent for all; +Table Op Msg_type Msg_text +test.t4 analyze status Engine-independent statistics collected +test.t4 analyze status OK +set @js='$out_interleaved'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_INTERLEAVED; +PAGES_ACCESSED_INTERLEAVED +2012 +drop table t4; +create table t5 ( +pk int not null primary key, +k int unsigned, +val int +) engine=innodb row_format=dynamic; +insert into t5 select seq, crc32(seq), seq from seq_1_to_10000; +create index k_idx on t5(k); +analyze table t5 persistent for all; +Table Op Msg_type Msg_text +test.t5 analyze status Engine-independent statistics collected +test.t5 analyze status OK +set @js='$out_shuffled'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SHUFFLED; +PAGES_ACCESSED_SHUFFLED +20006 +drop table t5; +create table t6 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t6 select seq, mod(seq,10), seq from seq_1_to_10000; +alter table t6 add column c int default 42, algorithm=instant; +create index k_idx on t6(k); +select count(val) as ROWS_INSTANT, sum(val) as SUM_INSTANT, min(c) as DEFAULT_C +from t6 force index(k_idx) where k=3; +ROWS_INSTANT SUM_INSTANT DEFAULT_C +1000 4998000 42 +drop table t6; +create table t7 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=redundant; +insert into t7 select seq, mod(seq,10), seq from seq_1_to_10000; +alter table t7 add column c int default 42, algorithm=instant; +create index k_idx on t7(k); +select count(val) as ROWS_REDUNDANT, sum(val) as SUM_REDUNDANT +from t7 force index(k_idx) where k=3; +ROWS_REDUNDANT SUM_REDUNDANT +1000 4998000 +drop table t7; +create table t8 ( +pk int not null primary key, +k int, +val int, +drop_me int +) engine=innodb row_format=dynamic; +insert into t8 select seq, mod(seq,10), seq, seq from seq_1_to_10000; +alter table t8 drop column drop_me, algorithm=instant; +create index k_idx on t8(k); +select count(val) as ROWS_DROPPED, sum(val) as SUM_DROPPED +from t8 force index(k_idx) where k=3; +ROWS_DROPPED SUM_DROPPED +1000 4998000 +drop table t8; +create table t9 ( +pk int not null primary key, +k int, +val int, +drop_me int +) engine=innodb row_format=redundant; +insert into t9 select seq, mod(seq,10), seq, seq from seq_1_to_10000; +alter table t9 drop column drop_me, algorithm=instant; +create index k_idx on t9(k); +select count(val) as ROWS_REDUNDANT_DROPPED, sum(val) as SUM_REDUNDANT_DROPPED +from t9 force index(k_idx) where k=3; +ROWS_REDUNDANT_DROPPED SUM_REDUNDANT_DROPPED +1000 4998000 +drop table t9; +create table t10 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=compressed key_block_size=8; +insert into t10 select seq, mod(seq,10), seq from seq_1_to_10000; +create index k_idx on t10(k); +select count(val) as ROWS_COMPRESSED, sum(val) as SUM_COMPRESSED +from t10 force index(k_idx) where k=3; +ROWS_COMPRESSED SUM_COMPRESSED +1000 4998000 +drop table t10; +create table t11 ( +k int, +val int +) engine=innodb row_format=dynamic; +insert into t11 select mod(seq,10), seq from seq_1_to_10000; +create index k_idx on t11(k); +select count(val) as ROWS_ROW_ID, sum(val) as SUM_ROW_ID +from t11 force index(k_idx) where k=3; +ROWS_ROW_ID SUM_ROW_ID +1000 4998000 +drop table t11; +create table t12 ( +pk int not null primary key, +k int, +val varchar(255) +) engine=innodb row_format=dynamic; +insert into t12 select seq, seq, repeat('x',30) from seq_1_to_10000; +create index k_idx on t12(k); +update t12 force index(k_idx) set val=repeat('y',200) where k > 0; +select count(*) as ROWS_GROWN, sum(pk) as SUM_PK from t12 +where val=repeat('y',200); +ROWS_GROWN SUM_PK +10000 50005000 +update t12 force index(k_idx) set val=repeat('x',30) where k > 0; +select count(*) as ROWS_SHRUNK, sum(pk) as SUM_PK from t12 +where val=repeat('x',30); +ROWS_SHRUNK SUM_PK +10000 50005000 +check table t12; +Table Op Msg_type Msg_text +test.t12 check status OK +drop table t12; +create table t13 ( +pk int not null primary key, +k bigint unsigned, +val int +) engine=innodb row_format=dynamic; +insert into t13 select seq, crc32(seq), seq from seq_1_to_3000; +insert into t13 select 3000+seq, 4294967296+seq, seq from seq_1_to_3000; +create index k_idx on t13(k); +analyze table t13 persistent for all; +Table Op Msg_type Msg_text +test.t13 analyze status Engine-independent statistics collected +test.t13 analyze status OK +set @js='$out_split'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_LATE_LOCALITY; +PAGES_ACCESSED_LATE_LOCALITY +9289 +drop table t13; +create table t14 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t14 select seq, seq, seq from seq_1_to_10000; +create index k_idx on t14(k); +analyze table t14 persistent for all; +Table Op Msg_type Msg_text +test.t14 analyze status Engine-independent statistics collected +test.t14 analyze status OK +set @js='$out_cov'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_COVERING; +PAGES_ACCESSED_COVERING +20 +set @js='$out_cov_x'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_COVERING_FOR_UPDATE; +PAGES_ACCESSED_COVERING_FOR_UPDATE +10042 +drop table t14; +create table t15 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t15 select seq, seq, seq from seq_1_to_10000; +create index k_idx on t15(k); +start transaction with consistent snapshot; +connect con2,localhost,root,,test; +update t15 set k=k+100000 where pk <= 5000; +disconnect con2; +connection default; +select count(k) as ROWS_SNAPSHOT, sum(k) as SUM_SNAPSHOT +from t15 force index(k_idx) where k > 0; +ROWS_SNAPSHOT SUM_SNAPSHOT +10000 50005000 +commit; +drop table t15; diff --git a/mysql-test/suite/innodb/r/non_covering_sec_idx_scan_debug.result b/mysql-test/suite/innodb/r/non_covering_sec_idx_scan_debug.result new file mode 100644 index 0000000000000..9b81b463394ab --- /dev/null +++ b/mysql-test/suite/innodb/r/non_covering_sec_idx_scan_debug.result @@ -0,0 +1,260 @@ +# clustered leaf hints disabled +create table t1 ( +pk int not null primary key, +domain_grp int, +val int +) engine=innodb row_format=dynamic; +insert into t1 select seq, mod(seq,10), seq from seq_1_to_10000; +create index domain_idx on t1(domain_grp); +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +set @js='$out_scan'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_FULL_SCAN; +PAGES_ACCESSED_FULL_SCAN +23 +set @js='$out_idx'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SEC_INDEX; +PAGES_ACCESSED_SEC_INDEX +2006 +drop table t1; +create table t2 ( +pk varchar(500) character set utf8mb4 collate utf8mb4_general_ci +not null primary key, +domain_grp int, +val int +) engine=innodb row_format=dynamic; +insert into t2 select lpad(seq,500,'0'), mod(seq,4), seq from seq_1_to_3000; +create index domain_idx on t2(domain_grp); +analyze table t2 persistent for all; +Table Op Msg_type Msg_text +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status OK +select stat_value into @leaf from mysql.innodb_index_stats +where database_name='test' and table_name='t2' and index_name='PRIMARY' + and stat_name='n_leaf_pages'; +select @leaf > 100 as THREE_LEVEL_TREE; +THREE_LEVEL_TREE +1 +set @js='$out_scan2'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_FULL_SCAN_3LEVEL; +PAGES_ACCESSED_FULL_SCAN_3LEVEL +110 +set @js='$out_idx2'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SEC_INDEX_3LEVEL; +PAGES_ACCESSED_SEC_INDEX_3LEVEL +2291 +drop table t2; +create table t3 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t3 select seq, mod(seq*997+13,10007), seq from seq_1_to_10000; +create index k_idx on t3(k); +analyze table t3 persistent for all; +Table Op Msg_type Msg_text +test.t3 analyze status Engine-independent statistics collected +test.t3 analyze status OK +set @js='$out_uncorr'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_UNCORRELATED; +PAGES_ACCESSED_UNCORRELATED +20020 +drop table t3; +create table t4 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t4 select seq, 2*seq-1, seq from seq_1_to_1000; +insert into t4 select 1000000+seq, 2*seq, seq from seq_1_to_1000; +create index k_idx on t4(k); +analyze table t4 persistent for all; +Table Op Msg_type Msg_text +test.t4 analyze status Engine-independent statistics collected +test.t4 analyze status OK +set @js='$out_interleaved'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_INTERLEAVED; +PAGES_ACCESSED_INTERLEAVED +4006 +drop table t4; +create table t5 ( +pk int not null primary key, +k int unsigned, +val int +) engine=innodb row_format=dynamic; +insert into t5 select seq, crc32(seq), seq from seq_1_to_10000; +create index k_idx on t5(k); +analyze table t5 persistent for all; +Table Op Msg_type Msg_text +test.t5 analyze status Engine-independent statistics collected +test.t5 analyze status OK +set @js='$out_shuffled'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SHUFFLED; +PAGES_ACCESSED_SHUFFLED +20020 +drop table t5; +create table t6 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t6 select seq, mod(seq,10), seq from seq_1_to_10000; +alter table t6 add column c int default 42, algorithm=instant; +create index k_idx on t6(k); +select count(val) as ROWS_INSTANT, sum(val) as SUM_INSTANT, min(c) as DEFAULT_C +from t6 force index(k_idx) where k=3; +ROWS_INSTANT SUM_INSTANT DEFAULT_C +1000 4998000 42 +drop table t6; +create table t7 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=redundant; +insert into t7 select seq, mod(seq,10), seq from seq_1_to_10000; +alter table t7 add column c int default 42, algorithm=instant; +create index k_idx on t7(k); +select count(val) as ROWS_REDUNDANT, sum(val) as SUM_REDUNDANT +from t7 force index(k_idx) where k=3; +ROWS_REDUNDANT SUM_REDUNDANT +1000 4998000 +drop table t7; +create table t8 ( +pk int not null primary key, +k int, +val int, +drop_me int +) engine=innodb row_format=dynamic; +insert into t8 select seq, mod(seq,10), seq, seq from seq_1_to_10000; +alter table t8 drop column drop_me, algorithm=instant; +create index k_idx on t8(k); +select count(val) as ROWS_DROPPED, sum(val) as SUM_DROPPED +from t8 force index(k_idx) where k=3; +ROWS_DROPPED SUM_DROPPED +1000 4998000 +drop table t8; +create table t9 ( +pk int not null primary key, +k int, +val int, +drop_me int +) engine=innodb row_format=redundant; +insert into t9 select seq, mod(seq,10), seq, seq from seq_1_to_10000; +alter table t9 drop column drop_me, algorithm=instant; +create index k_idx on t9(k); +select count(val) as ROWS_REDUNDANT_DROPPED, sum(val) as SUM_REDUNDANT_DROPPED +from t9 force index(k_idx) where k=3; +ROWS_REDUNDANT_DROPPED SUM_REDUNDANT_DROPPED +1000 4998000 +drop table t9; +create table t10 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=compressed key_block_size=8; +insert into t10 select seq, mod(seq,10), seq from seq_1_to_10000; +create index k_idx on t10(k); +select count(val) as ROWS_COMPRESSED, sum(val) as SUM_COMPRESSED +from t10 force index(k_idx) where k=3; +ROWS_COMPRESSED SUM_COMPRESSED +1000 4998000 +drop table t10; +create table t11 ( +k int, +val int +) engine=innodb row_format=dynamic; +insert into t11 select mod(seq,10), seq from seq_1_to_10000; +create index k_idx on t11(k); +select count(val) as ROWS_ROW_ID, sum(val) as SUM_ROW_ID +from t11 force index(k_idx) where k=3; +ROWS_ROW_ID SUM_ROW_ID +1000 4998000 +drop table t11; +create table t12 ( +pk int not null primary key, +k int, +val varchar(255) +) engine=innodb row_format=dynamic; +insert into t12 select seq, seq, repeat('x',30) from seq_1_to_10000; +create index k_idx on t12(k); +update t12 force index(k_idx) set val=repeat('y',200) where k > 0; +select count(*) as ROWS_GROWN, sum(pk) as SUM_PK from t12 +where val=repeat('y',200); +ROWS_GROWN SUM_PK +10000 50005000 +update t12 force index(k_idx) set val=repeat('x',30) where k > 0; +select count(*) as ROWS_SHRUNK, sum(pk) as SUM_PK from t12 +where val=repeat('x',30); +ROWS_SHRUNK SUM_PK +10000 50005000 +check table t12; +Table Op Msg_type Msg_text +test.t12 check status OK +drop table t12; +create table t13 ( +pk int not null primary key, +k bigint unsigned, +val int +) engine=innodb row_format=dynamic; +insert into t13 select seq, crc32(seq), seq from seq_1_to_3000; +insert into t13 select 3000+seq, 4294967296+seq, seq from seq_1_to_3000; +create index k_idx on t13(k); +analyze table t13 persistent for all; +Table Op Msg_type Msg_text +test.t13 analyze status Engine-independent statistics collected +test.t13 analyze status OK +set @js='$out_split'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_LATE_LOCALITY; +PAGES_ACCESSED_LATE_LOCALITY +12016 +drop table t13; +create table t14 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t14 select seq, seq, seq from seq_1_to_10000; +create index k_idx on t14(k); +analyze table t14 persistent for all; +Table Op Msg_type Msg_text +test.t14 analyze status Engine-independent statistics collected +test.t14 analyze status OK +set @js='$out_cov'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_COVERING; +PAGES_ACCESSED_COVERING +20 +set @js='$out_cov_x'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_COVERING_FOR_UPDATE; +PAGES_ACCESSED_COVERING_FOR_UPDATE +20020 +drop table t14; +create table t15 ( +pk int not null primary key, +k int, +val int +) engine=innodb row_format=dynamic; +insert into t15 select seq, seq, seq from seq_1_to_10000; +create index k_idx on t15(k); +start transaction with consistent snapshot; +connect con2,localhost,root,,test; +update t15 set k=k+100000 where pk <= 5000; +disconnect con2; +connection default; +select count(k) as ROWS_SNAPSHOT, sum(k) as SUM_SNAPSHOT +from t15 force index(k_idx) where k > 0; +ROWS_SNAPSHOT SUM_SNAPSHOT +10000 50005000 +commit; +drop table t15; diff --git a/mysql-test/suite/innodb/t/non_covering_sec_idx_scan.test b/mysql-test/suite/innodb/t/non_covering_sec_idx_scan.test new file mode 100644 index 0000000000000..47c7b56d7f50f --- /dev/null +++ b/mysql-test/suite/innodb/t/non_covering_sec_idx_scan.test @@ -0,0 +1,435 @@ +# +# MDEV-32286 ANALYZE displays a huge number of InnoDB secondary index pages_accessed +# +# The .result records raw pages_accessed values, so every server option they +# depend on is fixed by the test itself, because mariadb-test-run appends the +# options of its own --mysqld arguments after any that a test file requests, +# and the last value on the command line wins. The page size decides the tree +# shape, so a run of another page size skips. The row format decides the +# record layout, so every table names its own. The whole working set must stay +# resident, so that no access path depends on eviction timing, and it fits the +# buffer pool of a default run; a run with a smaller pool skips. The adaptive +# hash index, whose hash guesses reach a record without reading the pages that +# a descent reads, is set below, where no command line reaches it. +--source include/have_innodb.inc +--source include/have_innodb_16k.inc +--source include/have_sequence.inc + +if (`select @@global.innodb_buffer_pool_size < 8*1024*1024`) +{ + --skip Test requires innodb_buffer_pool_size of 8M or more +} + +# Kept out of the .result, so that a build without the adaptive hash index, +# where the variable does not exist, records the same result. +--disable_query_log +let $adaptive_hash_index= `select count(*) from information_schema.global_variables + where variable_name = 'innodb_adaptive_hash_index'`; +if ($adaptive_hash_index) +{ + set @save_adaptive_hash_index= @@global.innodb_adaptive_hash_index; + set global innodb_adaptive_hash_index= 0; +} +--enable_query_log + +create table t1 ( + pk int not null primary key, + domain_grp int, + val int +) engine=innodb row_format=dynamic; + +insert into t1 select seq, mod(seq,10), seq from seq_1_to_10000; + +create index domain_idx on t1(domain_grp); + +analyze table t1 persistent for all; + +# "val" is outside domain_idx, so each matching row needs a clustered-index +# lookup either way. +let $out_scan=`analyze format=json select sql_no_cache val from t1 ignore index(domain_idx) where domain_grp=3`; +evalp set @js='$out_scan'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_FULL_SCAN; + +# ~1000 rows share key 3 +let $out_idx=`analyze format=json select sql_no_cache val from t1 force index(domain_idx) where domain_grp=3`; +evalp set @js='$out_idx'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SEC_INDEX; + +drop table t1; + +# Wide primary key: the clustered index needs a 3-level tree. The character +# set and collation are pinned, because they decide the maximum key length +# and the comparison rules, and with them the recorded counts. +create table t2 ( + pk varchar(500) character set utf8mb4 collate utf8mb4_general_ci + not null primary key, + domain_grp int, + val int +) engine=innodb row_format=dynamic; + +insert into t2 select lpad(seq,500,'0'), mod(seq,4), seq from seq_1_to_3000; + +create index domain_idx on t2(domain_grp); + +analyze table t2 persistent for all; + +# A node pointer for this 500-byte key takes about 507 bytes, so a 16k root +# page holds about 32 of them: a leaf count far above that cannot hang off a +# single root, and the tree has at least three levels. The 'size' statistic +# cannot show this, because it counts the pages that both segments reserve, +# the free ones included, and so exceeds the leaf count of a two-level tree +# as well. +select stat_value into @leaf from mysql.innodb_index_stats + where database_name='test' and table_name='t2' and index_name='PRIMARY' + and stat_name='n_leaf_pages'; +select @leaf > 100 as THREE_LEVEL_TREE; + +let $out_scan2=`analyze format=json select sql_no_cache val from t2 ignore index(domain_idx) where domain_grp=3`; +evalp set @js='$out_scan2'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_FULL_SCAN_3LEVEL; + +# ~750 rows share key 3 +let $out_idx2=`analyze format=json select sql_no_cache val from t2 force index(domain_idx) where domain_grp=3`; +evalp set @js='$out_idx2'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SEC_INDEX_3LEVEL; + +drop table t2; + +# Secondary index decorrelated from clustered key order by a multiplier: the +# scan walks the clustered index in two slowly drifting strides, so +# consecutive rows keep sharing a clustered leaf for a while. +create table t3 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=dynamic; + +insert into t3 select seq, mod(seq*997+13,10007), seq from seq_1_to_10000; + +create index k_idx on t3(k); + +analyze table t3 persistent for all; + +let $out_uncorr=`analyze format=json select sql_no_cache val from t3 force index(k_idx) where k between 0 and 10006`; +evalp set @js='$out_uncorr'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_UNCORRELATED; + +drop table t3; + +# Two well separated clustered key ranges, interleaved in secondary key order: +# consecutive rows alternate between two clustered leaves, so the whole scan +# needs no more than two of them at a time. +create table t4 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=dynamic; + +insert into t4 select seq, 2*seq-1, seq from seq_1_to_1000; +insert into t4 select 1000000+seq, 2*seq, seq from seq_1_to_1000; + +create index k_idx on t4(k); + +analyze table t4 persistent for all; + +let $out_interleaved=`analyze format=json select sql_no_cache val from t4 force index(k_idx) where k between 1 and 2000`; +evalp set @js='$out_interleaved'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_INTERLEAVED; + +drop table t4; + +# Secondary key order shuffled against clustered key order, over five times +# as many clustered leaves as there are slots. Some lookups do land on a +# remembered leaf, but too few in a row: the miss counter reaches its limit +# early in the scan and the slots stop being tested on every row. The count +# therefore shows that the scan abandoned pays almost nothing, which is what +# this case is for, rather than what a hinted scan saves. +create table t5 ( + pk int not null primary key, + k int unsigned, + val int +) engine=innodb row_format=dynamic; + +insert into t5 select seq, crc32(seq), seq from seq_1_to_10000; + +create index k_idx on t5(k); + +analyze table t5 persistent for all; + +let $out_shuffled=`analyze format=json select sql_no_cache val from t5 force index(k_idx) where k > 0`; +evalp set @js='$out_shuffled'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_SHUFFLED; + +drop table t5; + +# The cases below cover the record formats and key shapes that the remembered +# leaves have to read. They check query results rather than counts, because a +# count would only repeat what t1 measures over a record layout that the +# counts do not depend on. +# +# Instant ALTER TABLE leaves a metadata pseudo-record as the first user record +# of the leftmost clustered leaf, which a remembered leaf then copies and +# compares its keys against. Nothing between that ALTER TABLE and the query +# may rebuild the clustered index, here or in the ROW_FORMAT=REDUNDANT case +# below: a rebuild drops the record, and the case then measures what t1 +# already measures, with the same query result to show it. +create table t6 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=dynamic; + +insert into t6 select seq, mod(seq,10), seq from seq_1_to_10000; + +alter table t6 add column c int default 42, algorithm=instant; + +create index k_idx on t6(k); + +select count(val) as ROWS_INSTANT, sum(val) as SUM_INSTANT, min(c) as DEFAULT_C + from t6 force index(k_idx) where k=3; + +drop table t6; + +# ROW_FORMAT=REDUNDANT, with a metadata pseudo-record as well: the keys are +# copied out of the other record format. +create table t7 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=redundant; + +insert into t7 select seq, mod(seq,10), seq from seq_1_to_10000; + +alter table t7 add column c int default 42, algorithm=instant; + +create index k_idx on t7(k); + +select count(val) as ROWS_REDUNDANT, sum(val) as SUM_REDUNDANT + from t7 force index(k_idx) where k=3; + +drop table t7; + +# The same, for a column dropped instantly rather than added. That sets +# dict_table_t::instant and leaves an alter metadata pseudo-record, whose +# header carries an added-field count and a metadata BLOB that the add +# metadata record above does not, so rec_copy_prefix_to_buf() converts a +# different header when a remembered leaf copies its first record. +# ALGORITHM=INSTANT is the assertion: without it the record would not exist. +create table t8 ( + pk int not null primary key, + k int, + val int, + drop_me int +) engine=innodb row_format=dynamic; + +insert into t8 select seq, mod(seq,10), seq, seq from seq_1_to_10000; + +alter table t8 drop column drop_me, algorithm=instant; + +create index k_idx on t8(k); + +select count(val) as ROWS_DROPPED, sum(val) as SUM_DROPPED + from t8 force index(k_idx) where k=3; + +drop table t8; + +# The dropped column under ROW_FORMAT=REDUNDANT. +create table t9 ( + pk int not null primary key, + k int, + val int, + drop_me int +) engine=innodb row_format=redundant; + +insert into t9 select seq, mod(seq,10), seq, seq from seq_1_to_10000; + +alter table t9 drop column drop_me, algorithm=instant; + +create index k_idx on t9(k); + +select count(val) as ROWS_REDUNDANT_DROPPED, sum(val) as SUM_REDUNDANT_DROPPED + from t9 force index(k_idx) where k=3; + +drop table t9; + +# ROW_FORMAT=COMPRESSED: the keys are copied out of a third record layout, +# and its page can carry a page_zip_des_t. Whether the buffer pool holds such +# a page without the uncompressed frame that a hint needs is not a state a +# test can ask for, so this case does not reach that rejection; it covers the +# record layout. +create table t10 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=compressed key_block_size=8; + +insert into t10 select seq, mod(seq,10), seq from seq_1_to_10000; + +create index k_idx on t10(k); + +select count(val) as ROWS_COMPRESSED, sum(val) as SUM_COMPRESSED + from t10 force index(k_idx) where k=3; + +drop table t10; + +# No user primary key: the clustered index is keyed by the hidden row id, so +# a remembered key is that single field. +create table t11 ( + k int, + val int +) engine=innodb row_format=dynamic; + +insert into t11 select mod(seq,10), seq from seq_1_to_10000; + +create index k_idx on t11(k); + +select count(val) as ROWS_ROW_ID, sum(val) as SUM_ROW_ID + from t11 force index(k_idx) where k=3; + +drop table t11; + +# Leaves that move while they are read. The secondary key order follows the +# clustered one, so every row is looked up through a remembered leaf, and the +# writes of the same statement grow each row from 30 to 200 bytes, which +# splits those leaves, and shrink it back, which merges and frees them. A +# remembered range then promises a leaf that the latched page denies, and a +# page that has left the tree keeps contents that only the check for a freed +# page rejects. Being a locking read, it also stores the position of a cursor +# that a hint placed. +create table t12 ( + pk int not null primary key, + k int, + val varchar(255) +) engine=innodb row_format=dynamic; + +insert into t12 select seq, seq, repeat('x',30) from seq_1_to_10000; + +create index k_idx on t12(k); + +update t12 force index(k_idx) set val=repeat('y',200) where k > 0; +select count(*) as ROWS_GROWN, sum(pk) as SUM_PK from t12 + where val=repeat('y',200); + +update t12 force index(k_idx) set val=repeat('x',30) where k > 0; +select count(*) as ROWS_SHRUNK, sum(pk) as SUM_PK from t12 + where val=repeat('x',30); + +check table t12; + +drop table t12; + +# Locality that appears only half way through the scan, which is what the +# retry that starts the miss count again is for. The secondary key order +# visits 3000 shuffled rows first, which makes the scan give the slots up, +# and then 3000 whose order follows the clustered one. crc32() cannot reach +# 2**32, so the second group sorts wholly above the first. Without the retry +# the scan would stay unhinted over that second group and keep no more than +# the few hits it made before it gave up, so nearly the whole saving that +# this count records is the retry's. A count again, placed after the cases +# above because it needs a table of its own shape. +create table t13 ( + pk int not null primary key, + k bigint unsigned, + val int +) engine=innodb row_format=dynamic; + +insert into t13 select seq, crc32(seq), seq from seq_1_to_3000; +insert into t13 select 3000+seq, 4294967296+seq, seq from seq_1_to_3000; + +create index k_idx on t13(k); + +analyze table t13 persistent for all; + +let $out_split=`analyze format=json select sql_no_cache val from t13 force index(k_idx) where k >= 0`; +evalp set @js='$out_split'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_LATE_LOCALITY; + +drop table t13; + +# A locking read cannot be covering. Every column that this query reads is in +# k_idx, so the first count is a scan of the secondary index alone, with no +# clustered-index lookup at all, and the hints cannot change it. The same scan +# under FOR UPDATE needs one lookup per row, because an exclusive select lock +# type makes ha_innobase::build_template() retrieve the whole clustered index +# record whatever the index holds, and the remembered leaves answer those +# lookups like any others: the secondary key order follows the clustered one +# here. ha_innobase::external_lock() decides the lock type per statement and +# not from the transaction state, so this case starts no transaction. +create table t14 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=dynamic; + +insert into t14 select seq, seq, seq from seq_1_to_10000; + +create index k_idx on t14(k); + +analyze table t14 persistent for all; + +let $out_cov=`analyze format=json select sql_no_cache k from t14 force index(k_idx) where k > 0`; +evalp set @js='$out_cov'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_COVERING; + +let $out_cov_x=`analyze format=json select sql_no_cache k from t14 force index(k_idx) where k > 0 for update`; +evalp set @js='$out_cov_x'; +set @out=(select json_extract(@js,'$**.r_engine_stats.pages_accessed')); +select cast(json_extract(@out,'$[0]') as UNSIGNED) as PAGES_ACCESSED_COVERING_FOR_UPDATE; + +drop table t14; + +# A covering scan reaches a clustered-index record as well, when the secondary +# leaf carries a PAGE_MAX_TRX_ID that the read view cannot see: the visibility +# test of row_search_mvcc() then enters the lookup past the check of +# need_to_access_clustered, whatever the query reads. A lookup whose clustered +# record is too new for the read view then rebuilds an older version of it +# through row_sel_build_prev_vers_for_mysql(), which +# row_sel_sec_rec_is_for_clust_rec() matches against the secondary entry. No +# other case here has a concurrent writer, so this is the only one that makes +# a remembered leaf answer such a lookup. +# The update below moves half the rows to another key range, so the older +# transaction meets both verdicts in one scan: it accepts the delete-marked +# entries that its version still carries, and rejects the new entries that no +# version of it does. Its result is the snapshot it opened on, and no record +# from a remembered leaf other than the right one produces that. A result +# rather than a count, because a count here follows the write pattern and +# the depth of the version rebuilds, and so reports what the writer did +# rather than what the hints save. +create table t15 ( + pk int not null primary key, + k int, + val int +) engine=innodb row_format=dynamic; + +insert into t15 select seq, seq, seq from seq_1_to_10000; + +create index k_idx on t15(k); + +start transaction with consistent snapshot; + +--connect (con2,localhost,root,,test) +update t15 set k=k+100000 where pk <= 5000; +--disconnect con2 + +--connection default +select count(k) as ROWS_SNAPSHOT, sum(k) as SUM_SNAPSHOT + from t15 force index(k_idx) where k > 0; +commit; + +drop table t15; + +--disable_query_log +if ($adaptive_hash_index) +{ + set global innodb_adaptive_hash_index= @save_adaptive_hash_index; +} +--enable_query_log diff --git a/mysql-test/suite/innodb/t/non_covering_sec_idx_scan_debug.test b/mysql-test/suite/innodb/t/non_covering_sec_idx_scan_debug.test new file mode 100644 index 0000000000000..1af8cccf76286 --- /dev/null +++ b/mysql-test/suite/innodb/t/non_covering_sec_idx_scan_debug.test @@ -0,0 +1,20 @@ +# +# MDEV-32286 ANALYZE displays a huge number of InnoDB secondary index pages_accessed +# +# The same shapes with the clustered leaf hints turned off, which only a debug +# build can do. The .result is a line for line counterpart of the one of +# non_covering_sec_idx_scan, so a diff of the two files is what the hints save. +# +--source include/have_debug.inc +--echo # clustered leaf hints disabled +--disable_query_log +# Global, and not of this session: an embedded server answers the backticks of +# a let on another thread than the rest, and only a global DBUG setting reaches +# both. +set @save_debug_dbug= @@global.debug_dbug; +set global debug_dbug='+d,ib_no_clust_leaf_hint'; +--enable_query_log +--source suite/innodb/t/non_covering_sec_idx_scan.test +--disable_query_log +set global debug_dbug= @save_debug_dbug; +--enable_query_log diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index dbbc77c55a5a0..2f97df160a4df 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1614,6 +1614,101 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode, goto search_loop; } +bool btr_cur_t::try_leaf_hint(const dtuple_t *tuple, page_id_t hint_page_id, + mtr_t *mtr) +{ + /* A clustered index only. buf_page_try_get() omits the change buffer + merge that buf_page_get_low() performs on a page whose state is + IBUF_EXIST, which would drop buffered entries of a secondary index leaf, + and the checks below accept FIL_PAGE_RTREE, which search_leaf() rejects. + Neither can happen where nothing is buffered and no page is an R-tree. */ + ut_ad(index()->is_primary()); + + /* A complete unique key. The search below takes a match on every compared + field as the answer without examining the successor record, which only a + key that cannot repeat allows: the PAGE_CUR_LE match of a repeating key can + be on a later leaf. */ + ut_ad(dtuple_get_n_fields_cmp(tuple) == dict_index_get_n_unique(index())); + + /* hint_page_id was not read from a latched parent page, so it may now + precede the caller's already-latched secondary-index leaf in the + B-tree latching order: never block on its latch (page latches have no + deadlock detection) and never read it from disk. */ + buf_block_t *const block= buf_page_try_get(hint_page_id, mtr); + if (!block) + return false; + + const page_t *const page= block->page.frame; + if (block->page.is_freed() || !fil_page_index_page_check(page) || + !page_is_leaf(page) || + !!page_is_comp(page) != index()->table->not_redundant() || + btr_page_get_index_id(page) != index()->id) + { + /* Stale hint or, for search_leaf()'s own checks, corruption; we cannot + tell here, so fall back either way, and the full descent still reports a + corrupt live leaf. is_freed() is the guard that descent omits: a freed + but unreused page keeps old contents that pass the other checks. */ + mtr->release_last_page(); + return false; + } + + page_cur.block= block; + /* The byte counts stay 0 instead of being computed: the search below does + not report them, and their only reader is the adaptive hash index, which + the hints stand down for. */ + up_match= 0; + up_bytes= 0; + low_match= 0; + low_bytes= 0; + if (page_cur_search_with_match(tuple, PAGE_CUR_LE, &up_match, &low_match, + &page_cur, nullptr) || + page_rec_is_infimum(page_cur.rec)) + { + /* Corruption, or tuple precedes every record on this page: its + predecessor, if any, is on an earlier leaf. */ + mtr->release_last_page(); + return false; + } + + if (page_has_next(page) && low_match < dtuple_get_n_fields_cmp(tuple)) + { + /* The record found is strictly less than tuple: PAGE_CUR_LE lands on + the greatest record <= tuple, and a full match would have made + low_match == n_fields_cmp. If it is the last user record of a leaf + with a right sibling, the true match may be on a later leaf; we cannot + resolve that from here, so reject the hint. The rightmost leaf needs + no such check: its last record is that match for any larger tuple. */ + const rec_t *const next_rec= page_rec_get_next_const(page_cur.rec); + if (UNIV_UNLIKELY(!next_rec) || page_rec_is_supremum(next_rec)) + { + mtr->release_last_page(); + return false; + } + } + + /* Unlike search_leaf(), this feeds no btr_search_info_update(): a hit + already provides the direct leaf access the adaptive hash index would. */ + + /* Age the page as the buf_page_get_gen() of a descent would, which + buf_page_try_get() does not do: a leaf that a correlated scan reads once + per row must not look less recently used than one reached by descent. */ + buf_page_make_young_if_needed(&block->page); + + /* search_leaf() also sets tree_height, which the hint cannot know because + it never walks the levels above the leaf. The value that the last descent + of this cursor left stands, and it is a real height of this tree: a hint is + only tried where a descent of this cursor already remembered a leaf of this + index. That matters because the value can travel: apart from search_leaf() + itself, its readers are the extent reservations of the pessimistic insert, + update and delete, which btr_pcur_copy_stored_position() reaches by copying + the whole cursor into the cursor of an update node. None of the three ever + sees a value that a hint left: each holds the index latch, which this + cursor can only take by descending again under BTR_MODIFY_TREE, and that + descent sets tree_height afresh. */ + flag= BTR_CUR_BINARY; + return true; +} + ATTRIBUTE_COLD void mtr_t::index_lock_upgrade() { auto &slot= m_memo[get_savepoint() - 1]; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e78f348bb2f0e..2d7a388d41f42 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -16021,6 +16021,13 @@ ha_innobase::reset() /* This is a statement level counter. */ m_prebuilt->autoinc_last_value = 0; + /* The clustered leaf hints are scoped to one statement. The slots + keep their key buffers, which are already sized for this table, and + their contents, which the next statement overwrites before reading: + a slot is only read once an insertion has counted it. */ + m_prebuilt->clust_leaf_hint_n = 0; + m_prebuilt->clust_leaf_hint_miss = 0; + m_prebuilt->skip_locked = false; return(0); } diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 9bc03449e1cd7..e249ae3b34f5f 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -763,6 +763,28 @@ struct btr_cur_t { will be set correctly. */ bool check_mismatch(const dtuple_t &tuple, bool ge, ulint comp) noexcept; #endif + + /** Try a PAGE_CUR_LE, BTR_SEARCH_LEAF lookup directly on a previously + remembered leaf page instead of descending from the root. + The hint is a guess, never derived from a latched parent page, so the + page is acquired via a non-blocking, no-I/O buf_page_try_get() and a + miss (stale hint) is expected, not corruption; the caller falls back + to a normal search. + Unlike search_leaf(), no index()->lock is acquired: that latch protects + the tree structure, which only a descent reads, while the contents of a + leaf page are protected by the page latch alone, and this never + dereferences a node pointer. A caller cannot rely on that latch after + search_leaf() either: it releases the latch for BTR_SEARCH_LEAF before + it returns, so both leave the caller the leaf latch alone. + For a clustered index only: the page is acquired without the change + buffer merge that a secondary index leaf can need. + @param tuple key to search for: a complete unique key of the index, + compared over its n_fields_cmp fields + @param hint_page_id remembered leaf page id + @param mtr mini-transaction + @return whether the cursor was positioned on the hinted leaf page */ + bool try_leaf_hint(const dtuple_t *tuple, page_id_t hint_page_id, + mtr_t *mtr); }; /** Modify the delete-mark flag of a record. diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 63858f25f023e..15850245b50fd 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -456,6 +456,60 @@ struct mysql_row_templ_t { #define ROW_PREBUILT_ALLOCATED 78540783 #define ROW_PREBUILT_FREED 26423527 +/** A remembered clustered-index leaf page and the key range it covered when +it was remembered, for btr_cur_t::try_leaf_hint(). The keys are copies, +because the page is unlatched between two lookups. + +The range is a filter only: a stale one (the page was split, merged, or a +record was inserted below its old minimum) can cost a wasted probe or a +needless descent, never a wrong result, because the checks that +btr_cur_t::try_leaf_hint() makes on the latched page remain the sole +authority. It therefore needs no invalidation protocol and no +modify_clock guard. */ +struct clust_leaf_hint_t { + const rec_t* first; /*!< copy of the leaf's first user + record, truncated to the key fields */ + const rec_t* last; /*!< copy of the leaf's last user + record, truncated to the key fields */ + byte* first_buf; /*!< buffer owning first */ + ulint first_buf_size; /*!< allocated size of first_buf */ + byte* last_buf; /*!< buffer owning last */ + ulint last_buf_size; /*!< allocated size of last_buf */ + rec_offs* first_offs; /*!< rec_get_offsets() of first */ + rec_offs* last_offs; /*!< rec_get_offsets() of last. Both + arrays are made where the copies are, + so that a lookup compares against a + slot without parsing its records + again; they are sized for the key of + this index and never grow */ + uint32_t page_no; /*!< the remembered leaf page number in + the clustered index's own tablespace; + page 0 is the FSP header, never a + leaf */ + uint16_t n_core_fields; /*!< dict_index_t::n_core_fields when + first and last were copied. The copies + must be interpreted with the value that + was in force, and that value can change + under a reader that holds no more than a + shared metadata lock, in either + direction, so a slot whose value no + longer matches is discarded rather than + read; see + row_sel_clust_leaf_hint_covers() */ + bool rightmost; /*!< whether the leaf had no right + sibling when it was remembered */ +}; + +/** Number of clustered leaf pages that a handle remembers, kept in most +recently used order. It buys the access patterns that alternate between a +few leaves, which one slot cannot serve at all; it cannot buy a scan that is +random over a table with many more leaves than this, at any size. Every +lookup that the hints do not answer scans them all, comparing at most two +keys per slot and only one where the key sorts below the range, so the count +is kept small enough for that scan to stay under the page-local searches of +the descent that follows it. */ +#define CLUST_LEAF_HINT_SLOTS 4 + /** A struct for (sometimes lazily) prebuilt structures in an Innobase table handle used within MySQL; these are used to save CPU time. */ @@ -574,6 +628,15 @@ struct row_prebuilt_t { sel/upd/del */ lock_mode select_lock_type;/*!< LOCK_NONE, LOCK_S, or LOCK_X */ bool skip_locked; /*!< TL_{READ,WRITE}_SKIP_LOCKED */ + uint8_t clust_leaf_hint_n;/*!< how many leading slots of + clust_leaf_hint are in use; zeroed per + statement in ha_innobase::reset(), + matching autoinc_last_value */ + uint16_t clust_leaf_hint_miss;/*!< consecutive lookups that no + slot of clust_leaf_hint answered; see + CLUST_LEAF_HINT_MAX_MISSES in + row0sel.cc. Zeroed per statement with + clust_leaf_hint_n */ lock_mode stored_select_lock_type;/*!< this field is used to remember the original select_lock_type that was decided in ha_innodb.cc, @@ -695,6 +758,19 @@ struct row_prebuilt_t { /** The MySQL table object */ TABLE* m_mysql_table; + /** CLUST_LEAF_HINT_SLOTS clustered leaves remembered from the + Row_sel_get_clust_rec_for_mysql() lookups of this statement, most + recently used first, or NULL if no lookup of this handle has + descended yet. Allocated from heap on the first descent, so that a + handle that never needs a clustered lookup allocates nothing, and one + that does pays for the slots once, not per statement; the key buffers, + which are already sized for this table, outlive the statement and are + released in row_prebuilt_free(). + Declared last, apart from the counters that go with it, so that every + field that precedes it keeps the offset, and with it the cache line, + that it had before this pointer existed. */ + clust_leaf_hint_t* clust_leaf_hint; + /** Get template by dict_table_t::cols[] number */ const mysql_row_templ_t* get_template_by_col(ulint col) const { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 3948d0312f4d6..6ef0978eee4f5 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -940,6 +940,17 @@ void row_prebuilt_free(row_prebuilt_t *prebuilt) ut_free(prebuilt->mysql_template); + if (prebuilt->clust_leaf_hint) { + /* The slots are on prebuilt->heap, but the buffers that + rec_copy_prefix_to_buf() allocated for their keys are not. + Every slot is freed, not only the ones in use: a slot that + was discarded or evicted keeps the buffers it owned. */ + for (ulint i = 0; i < CLUST_LEAF_HINT_SLOTS; i++) { + ut_free(prebuilt->clust_leaf_hint[i].first_buf); + ut_free(prebuilt->clust_leaf_hint[i].last_buf); + } + } + if (prebuilt->ins_graph) { que_graph_free_recursive(prebuilt->ins_graph); } diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 193b2b56db223..27377b24cee14 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -73,6 +73,27 @@ to que_run_threads: this is to allow canceling runaway queries */ #define SEL_EXHAUSTED 1 #define SEL_RETRY 2 +/** Consecutive lookups answered by no remembered clustered leaf after which a +scan is taken to have too little locality to pay for the slots of +row_prebuilt_t::clust_leaf_hint. It then stops testing them and stops +refreshing them, which is the larger half of their cost: two key copies for +every lookup they do not answer. + +What a hit saves is one buffer pool access and one page-local search for each +level above the leaf, which is little where those pages are resident, so the +trade is won only near the top of the answer rate. A scan that answers nearly +every lookup runs a third faster, one that answers two lookups in three pays +a few percent of its time for a third fewer page accesses, and one that +answers half of them or fewer only pays. A run of this length is not reached +by the first and is reached in the first rows by the last. */ +#define CLUST_LEAF_HINT_MAX_MISSES 8 + +/** How often the clustered leaf hints are tested again once +CLUST_LEAF_HINT_MAX_MISSES has been reached, in lookups. A scan whose order +becomes correlated only later recovers after at most this many rows, instead +of losing the hints for the rest of the statement. */ +#define CLUST_LEAF_HINT_RETRY 1024 + /********************************************************************//** Returns TRUE if the user-defined column in a secondary index record is alphabetically the same as the corresponding BLOB column in the clustered @@ -3353,6 +3374,256 @@ class Row_sel_get_clust_rec_for_mysql dtuple_t **vrow, mtr_t *mtr); }; +/** Determine whether a key can be on the leaf that a clustered leaf hint +remembers, comparing the key against the copies of that leaf's boundary +records. This decides a miss without any buffer pool access, so an +uncorrelated scan pays no page access for the hint it cannot use. +@param hint a non-empty clustered leaf hint +@param tuple key to search for +@param index the clustered index +@return whether the hinted leaf is worth probing */ +static bool row_sel_clust_leaf_hint_covers(const clust_leaf_hint_t &hint, + const dtuple_t *tuple, + const dict_index_t *index) +{ + ut_ad(hint.page_no); + + /* The copies can only be interpreted with the dict_index_t::n_core_fields + that was in force when they were made, and that value can change under a + reader that holds no more than a shared metadata lock, in either + direction: a delete that empties a single-page table invokes + dict_index_t::clear_instant_alter(), which raises it to n_fields where + instant ADD COLUMN alone was used, and lowers it past the columns that a + generic instant ALTER TABLE dropped. A slot that such a change has + outlived is therefore no candidate. The next descent refreshes it. */ + if (UNIV_UNLIKELY(hint.n_core_fields != index->n_core_fields)) + return false; + + /* This is the field count that row_sel_clust_leaf_hint_remember() copied, + and the one that its offsets describe. */ + ut_ad(dtuple_get_n_fields_cmp(tuple) == dict_index_get_n_unique(index)); + + /* On the leftmost leaf of a table that was subjected to instant ALTER + TABLE, the first user record is the metadata pseudo-record. + cmp_dtuple_rec_with_match_low() settles that comparison from + REC_INFO_MIN_REC_FLAG alone, which rec_copy_prefix_to_buf() preserves, + and reports every key as sorting above it: the correct lower bound for + the leaf that precedes all others. */ + bool covers= cmp_dtuple_rec(tuple, hint.first, index, hint.first_offs) >= 0; + + /* A key above the last record of the rightmost leaf still belongs to + that leaf, mirroring the page_has_next() test in + btr_cur_t::try_leaf_hint(). */ + if (covers && !hint.rightmost) + covers= cmp_dtuple_rec(tuple, hint.last, index, hint.last_offs) <= 0; + + return covers; +} + +/** Move a clustered leaf hint to another position of the most recently used +order, shifting every slot in between by one. The slot travels with the key +buffers that it owns, so that the displaced slot supplies buffers instead of +leaking them. +@param hints the slot array +@param from the slot to move +@param to where to move it */ +static void row_sel_clust_leaf_hint_move(clust_leaf_hint_t *hints, ulint from, + ulint to) +{ + ut_ad(from < CLUST_LEAF_HINT_SLOTS); + ut_ad(to < CLUST_LEAF_HINT_SLOTS); + if (from == to) + return; + const clust_leaf_hint_t moved= hints[from]; + if (from > to) + memmove(hints + to + 1, hints + to, (from - to) * sizeof *hints); + else + memmove(hints + from, hints + from + 1, (to - from) * sizeof *hints); + hints[to]= moved; +} + +/** Decide whether the clustered leaf hints take part in this lookup. +Both halves of their cost are governed here: the test of the slots before +the descent, and the copies that refresh them after it. A scan that has +given up must pay for neither. +@param prebuilt prebuilt struct of the handle +@return whether the slots are to be tested and refreshed */ +static bool row_sel_clust_leaf_hint_armed(row_prebuilt_t *prebuilt) +{ + DBUG_EXECUTE_IF("ib_no_clust_leaf_hint", return false;); + + const unsigned misses= prebuilt->clust_leaf_hint_miss; + if (misses < CLUST_LEAF_HINT_MAX_MISSES) + return true; + + if (misses % CLUST_LEAF_HINT_RETRY) + { + /* This scan has shown that it has no locality to exploit. */ + prebuilt->clust_leaf_hint_miss= uint16_t(misses + 1); + return false; + } + + /* One lookup in CLUST_LEAF_HINT_RETRY starts the count again, so that a + scan whose order becomes correlated only later recovers, after at most + that many rows, instead of losing the hints for the rest of the + statement. The trial that this begins is what makes the recovery + possible: the slots hold the leaves of the row where the scan gave up, + which nothing has refreshed since, so it takes a miss that remembers the + leaf the scan is on now before a later lookup can be answered. */ + prebuilt->clust_leaf_hint_miss= 0; + return true; +} + +/** Try the leaves that this statement remembered, most recently used first. +@param prebuilt prebuilt struct of the handle +@param index the clustered index +@param mtr mini-transaction +@return whether prebuilt->clust_pcur was positioned on a remembered leaf */ +static bool row_sel_clust_leaf_hint_search(row_prebuilt_t *prebuilt, + const dict_index_t *index, + mtr_t *mtr) +{ + const ulint n= prebuilt->clust_leaf_hint_n; + ut_ad(n <= CLUST_LEAF_HINT_SLOTS); + const unsigned misses= prebuilt->clust_leaf_hint_miss; + ut_ad(misses < CLUST_LEAF_HINT_MAX_MISSES); + + clust_leaf_hint_t *const hints= prebuilt->clust_leaf_hint; + ut_ad(hints || !n); + + for (ulint i= 0; i < n; i++) + { + if (!row_sel_clust_leaf_hint_covers(hints[i], prebuilt->clust_ref, index)) + continue; + + /* Two ranges can cover the same key only if one of them is stale, so + there is nothing to gain from looking past the first candidate: the + descent resolves whatever this one cannot. */ + if (prebuilt->clust_pcur->btr_cur.try_leaf_hint( + prebuilt->clust_ref, + page_id_t(index->table->space_id, hints[i].page_no), mtr)) + { + prebuilt->clust_leaf_hint_miss= 0; + row_sel_clust_leaf_hint_move(hints, i, 0); + return true; + } + + /* The range promised this leaf and the latched page denied it, so the + slot is stale. Discard it past the end of the used slots, where its key + buffers are the ones that the next insertion takes over. This counts as + a miss, like a key that no range covered: only a page that answered is + locality. A page that never answers, as a ROW_FORMAT=COMPRESSED page + that the buffer pool holds without an uncompressed frame never does, + would otherwise hold the counter at zero and be probed once per row for + the whole statement. */ + prebuilt->clust_leaf_hint_n= uint8_t(n - 1); + row_sel_clust_leaf_hint_move(hints, i, n - 1); + break; + } + + prebuilt->clust_leaf_hint_miss= uint16_t(misses + 1); + return false; +} + +/** Remember the clustered leaf that a descent landed on, together with the +keys of its first and last user record, at the front of the most recently +used order. The key buffers grow in place and travel with their slot, so a +scan allocates at most twice per slot, and nothing per row. +@param prebuilt prebuilt struct of the handle +@param block the clustered index leaf page the cursor is positioned on +@param index the clustered index */ +static void row_sel_clust_leaf_hint_remember(row_prebuilt_t *prebuilt, + const buf_block_t *block, + const dict_index_t *index) +{ + const page_t *const page= block->page.frame; + ut_ad(page_is_leaf(page)); + + const rec_t *const first= page_rec_get_next_const(page_get_infimum_rec(page)); + const rec_t *const last= page_rec_get_prev_const(page_get_supremum_rec(page)); + + if (UNIV_UNLIKELY(!first || !last || page_rec_is_supremum(first) || + page_rec_is_infimum(last))) + /* An empty page, which only the root of an empty tree can be, or a + corrupted record list; there is nothing worth remembering. */ + return; + + const ulint n_fields= dict_index_get_n_unique(index); + clust_leaf_hint_t *hints= prebuilt->clust_leaf_hint; + if (!hints) + { + /* Allocated on the first descent rather than with the handle, so that a + handle that never needs a clustered lookup allocates nothing. + + The offsets arrays are allocated here with the slots and never grow, + unlike the key buffers, because their size follows the key field count + of the index and not the length of a key: rec_get_offsets() describes at + most the n_fields fields that it is asked for. */ + const ulint n_offs= n_fields + (1 + REC_OFFS_HEADER_SIZE); + hints= static_cast + (mem_heap_zalloc(prebuilt->heap, CLUST_LEAF_HINT_SLOTS * sizeof *hints)); + rec_offs *offs= static_cast + (mem_heap_alloc(prebuilt->heap, + 2 * CLUST_LEAF_HINT_SLOTS * n_offs * sizeof *offs)); + for (ulint i= 0; i < CLUST_LEAF_HINT_SLOTS; i++) + { + rec_offs_set_n_alloc(offs, n_offs); + hints[i].first_offs= offs; + offs+= n_offs; + rec_offs_set_n_alloc(offs, n_offs); + hints[i].last_offs= offs; + offs+= n_offs; + } + prebuilt->clust_leaf_hint= hints; + } + + const ulint n= prebuilt->clust_leaf_hint_n; + ut_ad(n <= CLUST_LEAF_HINT_SLOTS); + const uint32_t page_no= block->page.id().page_no(); + ut_ad(page_no); + + /* This leaf can be remembered already, because a lookup that no range + covered descends without consulting any page, and the live range of a leaf + grows past the remembered one where a record is inserted above its last, or + where a sibling merges into it. Refresh that slot, rather than spend a + second one of the few on the same page. */ + ulint from= n < CLUST_LEAF_HINT_SLOTS ? n : CLUST_LEAF_HINT_SLOTS - 1; + bool remembered= false; + for (ulint i= 0; i < n; i++) + if (hints[i].page_no == page_no) + { + from= i; + remembered= true; + break; + } + + /* Move the slot to the front, taking the least recently used one once the + array is full. The slot that is displaced, or refreshed, is the one whose + buffers the copies below reuse, which makes the eviction exact and free. */ + row_sel_clust_leaf_hint_move(hints, from, 0); + if (!remembered && n < CLUST_LEAF_HINT_SLOTS) + prebuilt->clust_leaf_hint_n= uint8_t(n + 1); + + clust_leaf_hint_t &hint= hints[0]; + hint.n_core_fields= index->n_core_fields; + /* The offsets go with the copy they describe: a slot travels with both, + and rec_copy_prefix_to_buf() can move a copy when it grows its buffer. + prebuilt->heap is passed for a growth that the sizing above rules out, + so that an array which did grow would still outlive the statement. */ + hint.first= rec_copy_prefix_to_buf(first, index, n_fields, &hint.first_buf, + &hint.first_buf_size); + hint.first_offs= rec_get_offsets(hint.first, index, hint.first_offs, + hint.n_core_fields, n_fields, + &prebuilt->heap); + hint.last= rec_copy_prefix_to_buf(last, index, n_fields, &hint.last_buf, + &hint.last_buf_size); + hint.last_offs= rec_get_offsets(hint.last, index, hint.last_offs, + hint.n_core_fields, n_fields, + &prebuilt->heap); + hint.rightmost= !page_has_next(page); + hint.page_no= page_no; +} + /*********************************************************************//** Retrieves the clustered index record corresponding to a record in a non-clustered index. Does the necessary locking. Used in the MySQL @@ -3397,9 +3668,57 @@ Row_sel_get_clust_rec_for_mysql::operator()( clust_index = dict_table_get_first_index(sec_index->table); prebuilt->clust_pcur->btr_cur.page_cur.index = clust_index; - dberr_t err = btr_pcur_open_with_no_init(prebuilt->clust_ref, + /* The rows of a non-covering secondary-index scan often share a few + clustered leaf pages, so try the leaves that this statement already + visited before descending again. The remembered key ranges decide the + uncorrelated case in memory, so a scan that the hints cannot serve + pays neither a buffer pool access nor a pages_accessed for them. + + A probe would come before the descent, and therefore also before the + adaptive hash index guess that the descent tries first. The two solve + the same problem, and the guess solves it better: it lands directly + on the record, with no page-local search and no buffer pool access to + charge, where a hint hit costs both. So where the adaptive hash index + is enabled, the hints stay out of its way entirely, neither used nor + collected. + + The flag is read again at every lookup, so a change in the middle of + a statement takes effect at the next lookup, and the slots that the + change leaves behind are reset in ha_innobase::reset(). The test is + coarse, because the flag is global: an index that the adaptive hash + index does not serve loses the hints too. Both are heuristics, so the + price of either answer is a descent, never a wrong result. */ +#ifdef BTR_CUR_HASH_ADAPT + const bool use_hints = !btr_search.enabled; +#else + const bool use_hints = true; +#endif /* BTR_CUR_HASH_ADAPT */ + dberr_t err; + const bool hints_armed = use_hints + && row_sel_clust_leaf_hint_armed(prebuilt); + + if (hints_armed + && row_sel_clust_leaf_hint_search(prebuilt, clust_index, mtr)) { + err = DB_SUCCESS; + /* Set what btr_pcur_open_with_no_init() below would set for + the same (PAGE_CUR_LE, BTR_SEARCH_LEAF) arguments, except + trx_if_known, which is assigned unconditionally further + down. */ + prebuilt->clust_pcur->latch_mode + = BTR_LATCH_MODE_WITHOUT_INTENTION(BTR_SEARCH_LEAF); + prebuilt->clust_pcur->search_mode = PAGE_CUR_LE; + prebuilt->clust_pcur->pos_state = BTR_PCUR_IS_POSITIONED; + } else { + err = btr_pcur_open_with_no_init(prebuilt->clust_ref, PAGE_CUR_LE, BTR_SEARCH_LEAF, prebuilt->clust_pcur, mtr); + if (hints_armed && err == DB_SUCCESS) { + row_sel_clust_leaf_hint_remember( + prebuilt, + prebuilt->clust_pcur->btr_cur.page_cur.block, + clust_index); + } + } if (UNIV_UNLIKELY(err != DB_SUCCESS)) { return err; }