Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion proto/tikv.proto
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,8 @@ message GroupTagRecordItem {
uint64 network_in_bytes = 5;
uint64 network_out_bytes = 6;
uint64 logical_read_bytes = 7;
uint64 logical_write_bytes = 8;
uint64 logical_write_bytes = 8;
// RocksDB block reads attributed to the foreground request. This is not
// equivalent to device-level read IOPS.
uint64 rocksdb_block_read_count = 9;
}
3 changes: 3 additions & 0 deletions src/sinks/topsql_data_deltalake/arch.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ pub struct TopSQLDataDeltaLakeConfig {
- **SQL Digest Grouping**: Group by SQL digest
- **Time Partitioning**: Partition by execution time
- **Schema Optimization**: Optimized schema for TopSQL data
- **Detailed TiKV I/O**: Stores logical reads, logical writes, and
`topsql_rocksdb_block_read_count`; historical rows without the block-read column remain
compatible through nullable schema evolution
- **Keyspace-based Routing**: When `enable_keyspace_cluster_mapping = true`, `base_path` must already contain `org=xxx/cluster=xxx` template segments; the sink resolves keyspace via PD and replaces those template values with the routed `org` / `cluster`
- **Component-based Path Layout**: TopSQL data is partitioned by `component=<tidb|tikv|topru>` and `instance=<id>`

Expand Down
23 changes: 21 additions & 2 deletions src/sinks/topsql_data_deltalake/processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ use crate::sources::topsql_v2::upstream::consts::{
LABEL_TAG_LABEL, LABEL_TIMESTAMPS, LABEL_USER, METRIC_NAME_CPU_TIME_MS, METRIC_NAME_EXEC_COUNT,
METRIC_NAME_EXEC_DURATION, METRIC_NAME_LOGICAL_READ_BYTES, METRIC_NAME_LOGICAL_WRITE_BYTES,
METRIC_NAME_NETWORK_IN_BYTES, METRIC_NAME_NETWORK_OUT_BYTES, METRIC_NAME_READ_KEYS,
METRIC_NAME_STMT_DURATION_COUNT, METRIC_NAME_STMT_DURATION_SUM_NS, METRIC_NAME_STMT_EXEC_COUNT,
METRIC_NAME_TOTAL_RU, METRIC_NAME_WRITE_KEYS, SOURCE_TABLE_TOPRU,
METRIC_NAME_ROCKSDB_BLOCK_READ_COUNT, METRIC_NAME_STMT_DURATION_COUNT,
METRIC_NAME_STMT_DURATION_SUM_NS, METRIC_NAME_STMT_EXEC_COUNT, METRIC_NAME_TOTAL_RU,
METRIC_NAME_WRITE_KEYS, SOURCE_TABLE_TOPRU,
};

use lazy_static::lazy_static;
Expand Down Expand Up @@ -165,6 +166,13 @@ lazy_static! {
"is_nullable": true
}),
);
schema_info.insert(
METRIC_NAME_ROCKSDB_BLOCK_READ_COUNT.into(),
serde_json::json!({
"mysql_type": "bigint",
"is_nullable": true
}),
);
// tikv region specific fields
schema_info.insert(
LABEL_REGION_ID.into(),
Expand Down Expand Up @@ -732,6 +740,17 @@ mod tests {
)
}

#[test]
fn test_topsql_schema_contains_rocksdb_block_read_count() {
assert_eq!(
TOPSQL_SCHEMA.get(METRIC_NAME_ROCKSDB_BLOCK_READ_COUNT),
Some(&serde_json::json!({
"mysql_type": "bigint",
"is_nullable": true
}))
);
}

#[test]
fn test_build_table_path_with_meta_route_for_s3() {
let (sink, _) = TopSQLDeltaLakeSink::new_for_test(
Expand Down
1 change: 1 addition & 0 deletions src/sources/topsql/upstream/tikv/mock_upstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ impl ResourceMeteringPubSub for MockResourceMeteringPubSubServer {
network_out_bytes: 0,
logical_read_bytes: 0,
logical_write_bytes: 0,
rocksdb_block_read_count: 0,
}],
})),
})])) as Self::SubscribeStream,
Expand Down
2 changes: 2 additions & 0 deletions src/sources/topsql/upstream/tikv/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ impl UpstreamEventParser for ResourceUsageRecordParser {
network_out_bytes: 0, // Not supported in topsql v1
logical_read_bytes: 0, // Not supported in topsql v1
logical_write_bytes: 0, // Not supported in topsql v1
rocksdb_block_read_count: 0, // Not supported in topsql v1
};
match digest_items.get_mut(&psd.resource_group_tag) {
None => {
Expand Down Expand Up @@ -393,6 +394,7 @@ mod tests {
network_out_bytes: i.network_out_bytes,
logical_read_bytes: i.logical_read_bytes,
logical_write_bytes: i.logical_write_bytes,
rocksdb_block_read_count: 0,
})
.collect(),
})),
Expand Down
10 changes: 10 additions & 0 deletions src/sources/topsql_v2/arch.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,16 @@ TiDB subscriptions emit four log-event families:

For TiDB-originated events, `keyspace` is propagated when the upstream payload includes `keyspace_name`, including `topsql_sql_meta` and `topsql_plan_meta`.

TiKV subscriptions emit `tikv_topsql` and `tikv_topregion` events. In addition to CPU,
key counts, network bytes, and logical read/write bytes, these events include
`topsql_rocksdb_block_read_count`. The value is the foreground request's RocksDB block-read
count and must not be interpreted as device-level read IOPS.

Per-second Top N filtering keeps the union of records selected independently by CPU, combined
network traffic, logical reads, logical writes, and RocksDB block reads. Metrics from evicted
records and upstream `others` records are merged without dropping any dimension, and the same
fields are preserved during downsampling.

## Dependencies

- Same as TopSQL v1
Expand Down
1 change: 1 addition & 0 deletions src/sources/topsql_v2/upstream/consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub const METRIC_NAME_NETWORK_IN_BYTES: &str = "topsql_network_in_bytes";
pub const METRIC_NAME_NETWORK_OUT_BYTES: &str = "topsql_network_out_bytes";
pub const METRIC_NAME_LOGICAL_READ_BYTES: &str = "topsql_logical_read_bytes";
pub const METRIC_NAME_LOGICAL_WRITE_BYTES: &str = "topsql_logical_write_bytes";
pub const METRIC_NAME_ROCKSDB_BLOCK_READ_COUNT: &str = "topsql_rocksdb_block_read_count";
pub const METRIC_NAME_STMT_EXEC_COUNT: &str = "topsql_stmt_exec_count";
pub const METRIC_NAME_STMT_DURATION_SUM_NS: &str = "topsql_stmt_duration_sum_ns";
pub const METRIC_NAME_STMT_DURATION_COUNT: &str = "topsql_stmt_duration_count";
Expand Down
1 change: 1 addition & 0 deletions src/sources/topsql_v2/upstream/tikv/mock_upstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ impl ResourceMeteringPubSub for MockResourceMeteringPubSubServer {
network_out_bytes: 0,
logical_read_bytes: 0,
logical_write_bytes: 0,
rocksdb_block_read_count: 0,
}],
})),
})])) as Self::SubscribeStream,
Expand Down
Loading
Loading