From 4da79f379dc5f3234637f6e52ba121ac37cd61dc Mon Sep 17 00:00:00 2001 From: seks99x Date: Tue, 1 Sep 2026 04:47:26 +0300 Subject: [PATCH 1/2] Add 4 KiB logical block footprint tracking to stats This adds a 'Number of 4 KiB logical blocks touched' metric to the --stats output to decouple network delta payload from actual local file modifications. Previously, a small amount of literal data scattered across a file (especially with --inplace) could result in a massive number of local file write operations with no visibility, and large sparse files masked their true write opreations (ignoring punch holes). Technical details: - Implemented a stateful block tracker in the receiver that calculates touched 4K boundaries using file offsets and lengths, including strict lseek awareness to accurately skip sparse file holes. - Enforced strict per-file lifetime state with a reset hook inside receive_data(), successfully mitigating POSIX file descriptor (FD) recycling state leaks. - Created MSG_BLOCK_STATS multiplex message to tunnel the block footprint safely out of the isolated receiver process and relay it over the network. - Bumped PROTOCOL_VERSION to 33 and SUBPROTOCOL_VERSION to 8392 for safe PR testing. - Added test suite covering contiguous, scattered, zero-byte, sparse file (hole-skipping), multi-file (FD reuse), batch mode, and maximum-I/O boundary conditions. --- fileio.c | 42 ++++++ io.c | 11 ++ main.c | 8 ++ receiver.c | 6 + rsync.1.md | 6 + rsync.h | 4 +- testsuite/write-touched-blocks_test.py | 171 +++++++++++++++++++++++++ 7 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 testsuite/write-touched-blocks_test.py diff --git a/fileio.c b/fileio.c index c4cf57829..5925c9333 100644 --- a/fileio.c +++ b/fileio.c @@ -40,6 +40,9 @@ OFF_T preallocated_len = 0; static OFF_T sparse_seek = 0; static OFF_T sparse_past_write = 0; +static int last_tracked_fd = -1; +static int64 last_touched_blk = -1; + int sparse_end(int f, OFF_T size, int updating_basis_or_equiv) { int ret = 0; @@ -161,6 +164,8 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int continue; } if (i > start) { + if (!use_seek) + track_block_touches(f, offset + start, i - start); if (emit_sparse_span(f, use_seek, buf + start, i - start) < 0) return -1; sparse_past_write = offset + i; @@ -170,6 +175,8 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int start = i; } if (end > start) { + if (!use_seek) + track_block_touches(f, offset + start, end - start); if (emit_sparse_span(f, use_seek, buf + start, end - start) < 0) return -1; } @@ -201,6 +208,38 @@ int flush_write_file(int f) return ret; } +void reset_block_tracker(void) +{ + last_tracked_fd = -1; + last_touched_blk = -1; +} + +void track_block_touches(int f, OFF_T offset, int32 len) +{ + if (len <= 0) + return; + extern struct stats stats; + if (f != last_tracked_fd) { + last_tracked_fd = f; + last_touched_blk = -1; + } + int64 start_blk = offset / 4096; + int64 end_blk = start_blk + (((offset % 4096) + len - 1) / 4096); + int64 blocks_to_add = 0; + if (start_blk > last_touched_blk) + blocks_to_add = (end_blk - start_blk + 1); + else if (end_blk > last_touched_blk) + blocks_to_add = (end_blk - last_touched_blk); + if (blocks_to_add > 0) { + if (INT64_MAX - stats.touched_blocks_4k < blocks_to_add) + stats.touched_blocks_4k = INT64_MAX; + else + stats.touched_blocks_4k += blocks_to_add; + } + if (end_blk > last_touched_blk) + last_touched_blk = end_blk; +} + /* write_file does not allow incomplete writes. It loops internally * until len bytes are written or errno is set. Note that use_seek and * offset are only used in sparse processing (see write_sparse()). */ @@ -208,6 +247,9 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len) { int ret = 0; + if (!use_seek && sparse_files == 0) { + track_block_touches(f, offset, len); + } while (len > 0) { int r1; if (sparse_files > 0) { diff --git a/io.c b/io.c index 451d9b450..fd1a1c64a 100644 --- a/io.c +++ b/io.c @@ -1692,6 +1692,17 @@ static void read_a_msg(void) raw_read_buf((char*)&stats.total_read, sizeof stats.total_read); iobuf.in_multiplexed = 1; break; + case MSG_BLOCK_STATS: { + if (msg_bytes != 8 || protocol_version < 33) + goto invalid_msg; + char b[8]; + raw_read_buf(b, 8); + stats.touched_blocks_4k = IVAL64(b, 0); + iobuf.in_multiplexed = 1; + if (am_server && am_generator) + send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); + break; + } case MSG_REDO: if (msg_bytes != 4 || !am_generator) goto invalid_msg; diff --git a/main.c b/main.c index 8381935cf..2e894df34 100644 --- a/main.c +++ b/main.c @@ -443,6 +443,9 @@ static void output_summary(void) human_num(stats.total_transferred_size)); rprintf(FINFO,"Literal data: %s bytes\n", human_num(stats.literal_data)); + if (protocol_version >= 33) + rprintf(FINFO,"Number of 4 KiB logical blocks touched: %s\n", + comma_num(stats.touched_blocks_4k)); rprintf(FINFO,"Matched data: %s bytes\n", human_num(stats.matched_data)); rprintf(FINFO,"File list size: %s\n", @@ -1107,6 +1110,11 @@ static int do_recv(int f_in, int f_out, char *local_name) write_int(f_out, NDX_DONE); send_msg(MSG_STATS, (char*)&stats.total_read, sizeof stats.total_read, 0); + if(protocol_version >= 33) { + char b[8]; + SIVAL64(b, 0, stats.touched_blocks_4k); + send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); + } io_flush(FULL_FLUSH); /* Handle any keep-alive packets from the post-processing work diff --git a/receiver.c b/receiver.c index 28d663acc..f30ff6a46 100644 --- a/receiver.c +++ b/receiver.c @@ -471,6 +471,12 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, char *data; int32 i; char *map = NULL; + + /* Reset the 4K block tracker's per-file lifetime state. + * Placed here to guarantee a clean state for every new file payload, + * preventing bugs caused by POSIX file descriptor + * recycling when sequential files are assigned the same FD. */ + reset_block_tracker(); #ifdef SUPPORT_PREALLOCATION if (preallocate_files && fd != -1 && total_size > 0 && (!inplace_sizing || total_size > size_r)) { diff --git a/rsync.1.md b/rsync.1.md index ad686b646..df63f519f 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -3537,6 +3537,12 @@ sign) if you want the local shell to expand it. Note that this line is only output if deletions are in effect, and only if the negotiated protocol is at least 31 (the default when both sides are 3.1.0, September 2013, or newer). + - `Number of 4 KiB logical blocks touched` is the number of unique 4 KiB + logical file regions covered by rsync's own write operations. This metric + decouples network payload from the scope of local file modifications. + For example, a small amount of *Literal data* can result in a massive + number of touched blocks if the modifications are highly scattered + across a file, especially when using `--inplace`. - `Number of regular files transferred` is the count of normal files that were updated via rsync's delta-transfer algorithm, which does not include directories, symlinks, etc. diff --git a/rsync.h b/rsync.h index b15aa1af6..b0507139c 100644 --- a/rsync.h +++ b/rsync.h @@ -111,7 +111,7 @@ /* Update this if you make incompatible changes and ALSO update the * SUBPROTOCOL_VERSION if it is not a final (official) release. */ -#define PROTOCOL_VERSION 32 +#define PROTOCOL_VERSION 33 /* This is used when working on a new protocol version or for any unofficial * protocol tweaks. It should be a non-zero value for each pre-release repo @@ -299,6 +299,7 @@ enum msgcode { MSG_LOG=FLOG, MSG_CLIENT=FCLIENT, /* sibling logging */ MSG_REDO=9, /* reprocess indicated flist index */ MSG_STATS=10, /* message has stats data for generator */ + MSG_BLOCK_STATS=11, /* message has block-level stats for sender */ MSG_IO_ERROR=22,/* the sending side had an I/O error */ MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */ MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */ @@ -1080,6 +1081,7 @@ struct stats { int64 total_read; int64 literal_data; int64 matched_data; + int64 touched_blocks_4k; int64 flist_buildtime; int64 flist_xfertime; int64 flist_size; diff --git a/testsuite/write-touched-blocks_test.py b/testsuite/write-touched-blocks_test.py new file mode 100644 index 000000000..dc20d6c07 --- /dev/null +++ b/testsuite/write-touched-blocks_test.py @@ -0,0 +1,171 @@ +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from rsyncfns import ( + FROMDIR, RSYNC, SCRATCHDIR, + makepath, test_fail, test_skipped +) + +rsync_args = shlex.split(str(RSYNC)) +for arg in rsync_args: + if arg.startswith('--protocol='): + prot_version = int(arg.split('=')[1]) + if prot_version < 33: + test_skipped(f"Skipping write-touched-blocks: feature requires protocol 33, but CI forced {prot_version}") + +src = FROMDIR +makepath(src) + +base_file = src / 'base.bin' + +# Generate 4 MiB of random data in memory +data = os.urandom(4 * 1024 * 1024) + +def setup_test(dest_name): + """Resets the base file and creates a clean destination file.""" + dest_path = SCRATCHDIR / dest_name + base_file.write_bytes(data) + dest_path.write_bytes(data) + return dest_path + +def run_client(src_path, dest_path): + rsync_cmd = shlex.split(str(RSYNC)) + argv = rsync_cmd + ['-a', '--stats', '--inplace', '-I', '--no-whole-file', + str(src_path), str(dest_path)] + return subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True) + +# TEST 1: Contiguous Write (1 Block) +dest_contig = setup_test('dest_contiguous.bin') +with open(base_file, 'r+b') as f: + f.write(b'\x00' * 3000) + +proc = run_client(base_file, dest_contig) +if proc.returncode != 0: + test_fail(f"rsync failed on contiguous test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 1\n" not in proc.stdout: + test_fail(f"Contiguous check failed! Expected 1 block. Output:\n{proc.stdout}") + +# TEST 2: Scattered Write (10 Blocks) +dest_scatter = setup_test('dest_scattered.bin') +with open(base_file, 'r+b') as f: + for i in range(1, 11): + f.seek(i * 4096) + old_byte = f.read(1)[0] + f.seek(i * 4096) + f.write(bytes([old_byte ^ 0xFF])) + +proc = run_client(base_file, dest_scatter) +if proc.returncode != 0: + test_fail(f"rsync failed on scattered test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 10\n" not in proc.stdout: + test_fail(f"Scattered check failed! Expected 10 blocks. Output:\n{proc.stdout}") + +# TEST 3: Identical Files (0 Blocks Edge Case) +dest_zero = setup_test('dest_zero.bin') + +proc = run_client(base_file, dest_zero) +if proc.returncode != 0: + test_fail(f"rsync failed on zero-block test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 0\n" not in proc.stdout: + test_fail(f"Zero check failed! Expected 0 blocks. Output:\n{proc.stdout}") + +# TEST 4: Full File Write (1,024 Blocks) +dest_full = setup_test('dest_full.bin') +# Overwrite the entire 4 MiB base file with brand new random data +# This forces the delta algorithm to find 0 matches and write all 1,024 blocks. +base_file.write_bytes(os.urandom(4 * 1024 * 1024)) + +proc = run_client(base_file, dest_full) +if proc.returncode != 0: + test_fail(f"rsync failed on full write test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 1,024\n" not in proc.stdout: + test_fail(f"Full write check failed! Expected 1,024 blocks. Output:\n{proc.stdout}") + +# TEST 5: Sparse File Write (Hole skipping) +sparse_src = src / 'sparse.bin' +sparse_dest = SCRATCHDIR / 'sparse_dest.bin' + +# Create a file with written data on the ends, but a massive 4 MiB hole in the middle. +# 1 block data + 1,024 blocks hole + 1 block data = 1,026 blocks total size. +with open(sparse_src, 'wb') as f: + f.write(os.urandom(4096)) # Block 1 (Data) + f.seek(4 * 1024 * 1024, os.SEEK_CUR) # The Hole (4 MiB of nothing) + f.write(os.urandom(4096)) # Block 1026 (Data) + +# We must run this WITH --sparse (-S) and WITHOUT --inplace to force +# the receiver to create a brand new sparse file from scratch using write_sparse(). +rsync_cmd = shlex.split(str(RSYNC)) +argv_sparse = rsync_cmd + ['-a', '--stats', '--sparse', str(sparse_src), str(sparse_dest)] +proc = subprocess.run(argv_sparse, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + +if proc.returncode != 0: + test_fail(f"rsync failed on sparse test:\n{proc.stdout}") + +# Even though the file is over 4 MiB in size, only 2 logical 4K blocks +# should be written because the rest was skipped via lseek. +if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: + test_fail(f"Sparse check failed! Expected 2 logical blocks written. Output:\n{proc.stdout}") + +# TEST 6: Multiple Files +# Creates two separate 4KB files. If the tracker fails to reset between +# files due to FD recycling, it will report 1 block instead of 2. +fd_src_dir = src / 'fd_test' +fd_dest_dir = SCRATCHDIR / 'fd_dest' +makepath(fd_src_dir) +makepath(fd_dest_dir) + +# Create two distinct 1-block files +(fd_src_dir / 'fileA.bin').write_bytes(os.urandom(4096)) +(fd_src_dir / 'fileB.bin').write_bytes(os.urandom(4096)) + +# Sync the whole directory so rsync processes both in one process lifespan +rsync_cmd = shlex.split(str(RSYNC)) +argv_fd = rsync_cmd + ['-a', '--stats', '--inplace', str(fd_src_dir) + '/', str(fd_dest_dir) + '/'] +proc = subprocess.run(argv_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + +if proc.returncode != 0: + test_fail(f"rsync failed on multiple files test:\n{proc.stdout}") + +if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: + test_fail(f"FD Reuse bug confirmed! Expected 2 blocks (1 per file). Output:\n{proc.stdout}") + +# TEST 7: Batch Mode State Reset (--read-batch) +batch_src_dir = src / 'batch_src' +batch_dest_dir = SCRATCHDIR / 'batch_dest' +batch_file = SCRATCHDIR / 'test_batch.rsync' +makepath(batch_src_dir) +makepath(batch_dest_dir) + +# Create two 4KB source files with random data +(batch_src_dir / 'fileA.bin').write_bytes(os.urandom(4096)) +(batch_src_dir / 'fileB.bin').write_bytes(os.urandom(4096)) + +# Create two 4KB zeroed destination files (forces the delta algorithm to write exactly 1 block per file) +(batch_dest_dir / 'fileA.bin').write_bytes(b'\x00' * 4096) +(batch_dest_dir / 'fileB.bin').write_bytes(b'\x00' * 4096) + +# Step 1: Generate the batch file (Sender side) +# We MUST use '-I' because the files have identical sizes and timestamps. +rsync_cmd = shlex.split(str(RSYNC)) +argv_write_batch = rsync_cmd + ['-a', '-I', '--only-write-batch=' + str(batch_file), + str(batch_src_dir) + '/', str(batch_dest_dir) + '/'] +subprocess.run(argv_write_batch, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + +# Step 2: Apply the batch file and collect stats (Receiver side) +argv_read_batch = rsync_cmd + ['-a', '-I', '--inplace', '--stats', + '--read-batch=' + str(batch_file), str(batch_dest_dir) + '/'] + +proc = subprocess.run(argv_read_batch, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + +if proc.returncode != 0: + test_fail(f"rsync failed on read-batch test:\n{proc.stdout}") + +# If the state leaked across the batch read, this would output 1 block. +if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: + test_fail(f"Batch Mode tracker check failed! Expected 2 blocks (1 per file). Output:\n{proc.stdout}") + +print("write-touched-blocks: cleanly distinguishes contiguous, scattered, zero, full, sparse, multi-file, and batch writes") From d547e55818458c8afccbb9e99341e1c84a7e2b9f Mon Sep 17 00:00:00 2001 From: Zen Dodd Date: Sat, 5 Sep 2026 09:11:50 +1000 Subject: [PATCH 2/2] io: fix block stats integration --- .github/workflows/ubuntu-22.04-build.yml | 2 +- .github/workflows/ubuntu-build.yml | 2 +- fileio.c | 58 ++++++++++++------------ io.c | 11 +++-- main.c | 14 +++--- receiver.c | 7 +-- rsync.1.md | 2 +- rsync.h | 4 +- testsuite/skiplist/README.md | 1 + testsuite/skiplist/proto29.txt | 1 + testsuite/skiplist/proto30.txt | 9 ++++ testsuite/write-touched-blocks_test.py | 8 ++-- 12 files changed, 64 insertions(+), 55 deletions(-) create mode 100644 testsuite/skiplist/proto30.txt diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index af4ae6d51..dcde1e0f3 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -45,7 +45,7 @@ jobs: - name: check run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto30.txt make check30 - name: check29 run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index 3531c518c..1a33b1dca 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -41,7 +41,7 @@ jobs: - name: check run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto30.txt make check30 - name: check29 run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) diff --git a/fileio.c b/fileio.c index 5925c9333..b8db29b85 100644 --- a/fileio.c +++ b/fileio.c @@ -34,6 +34,7 @@ #define ALIGNED_LENGTH(len) ((((len) - 1) | (ALIGN_BOUNDARY-1)) + 1) extern int sparse_files; +extern struct stats stats; OFF_T preallocated_len = 0; @@ -165,7 +166,7 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int } if (i > start) { if (!use_seek) - track_block_touches(f, offset + start, i - start); + track_block_touches(f, offset + start, i - start); if (emit_sparse_span(f, use_seek, buf + start, i - start) < 0) return -1; sparse_past_write = offset + i; @@ -210,34 +211,34 @@ int flush_write_file(int f) void reset_block_tracker(void) { - last_tracked_fd = -1; - last_touched_blk = -1; + last_tracked_fd = -1; + last_touched_blk = -1; } -void track_block_touches(int f, OFF_T offset, int32 len) +void track_block_touches(int f, OFF_T offset, int32 len) { - if (len <= 0) - return; - extern struct stats stats; - if (f != last_tracked_fd) { - last_tracked_fd = f; - last_touched_blk = -1; - } - int64 start_blk = offset / 4096; - int64 end_blk = start_blk + (((offset % 4096) + len - 1) / 4096); - int64 blocks_to_add = 0; - if (start_blk > last_touched_blk) - blocks_to_add = (end_blk - start_blk + 1); - else if (end_blk > last_touched_blk) - blocks_to_add = (end_blk - last_touched_blk); - if (blocks_to_add > 0) { - if (INT64_MAX - stats.touched_blocks_4k < blocks_to_add) - stats.touched_blocks_4k = INT64_MAX; - else - stats.touched_blocks_4k += blocks_to_add; - } - if (end_blk > last_touched_blk) - last_touched_blk = end_blk; + int64 start_blk, end_blk, blocks_to_add = 0; + + if (len <= 0) + return; + if (f != last_tracked_fd) { + last_tracked_fd = f; + last_touched_blk = -1; + } + start_blk = offset / 4096; + end_blk = start_blk + (((offset % 4096) + len - 1) / 4096); + if (start_blk > last_touched_blk) + blocks_to_add = end_blk - start_blk + 1; + else if (end_blk > last_touched_blk) + blocks_to_add = end_blk - last_touched_blk; + if (blocks_to_add > 0) { + if (INT64_MAX - stats.touched_blocks_4k < blocks_to_add) + stats.touched_blocks_4k = INT64_MAX; + else + stats.touched_blocks_4k += blocks_to_add; + } + if (end_blk > last_touched_blk) + last_touched_blk = end_blk; } /* write_file does not allow incomplete writes. It loops internally @@ -247,9 +248,8 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len) { int ret = 0; - if (!use_seek && sparse_files == 0) { - track_block_touches(f, offset, len); - } + if (!use_seek && sparse_files == 0) + track_block_touches(f, offset, len); while (len > 0) { int r1; if (sparse_files > 0) { diff --git a/io.c b/io.c index fd1a1c64a..44d81ccc3 100644 --- a/io.c +++ b/io.c @@ -1693,15 +1693,16 @@ static void read_a_msg(void) iobuf.in_multiplexed = 1; break; case MSG_BLOCK_STATS: { - if (msg_bytes != 8 || protocol_version < 33) - goto invalid_msg; char b[8]; + + if (msg_bytes != 8 || protocol_version < 33 || (!am_generator && !am_sender)) + goto invalid_msg; raw_read_buf(b, 8); stats.touched_blocks_4k = IVAL64(b, 0); iobuf.in_multiplexed = 1; - if (am_server && am_generator) - send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); - break; + if (am_server && am_generator) + send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); + break; } case MSG_REDO: if (msg_bytes != 4 || !am_generator) diff --git a/main.c b/main.c index 2e894df34..604294927 100644 --- a/main.c +++ b/main.c @@ -443,9 +443,9 @@ static void output_summary(void) human_num(stats.total_transferred_size)); rprintf(FINFO,"Literal data: %s bytes\n", human_num(stats.literal_data)); - if (protocol_version >= 33) - rprintf(FINFO,"Number of 4 KiB logical blocks touched: %s\n", - comma_num(stats.touched_blocks_4k)); + if (protocol_version >= 33) + rprintf(FINFO,"Number of 4 KiB logical blocks touched: %s\n", + comma_num(stats.touched_blocks_4k)); rprintf(FINFO,"Matched data: %s bytes\n", human_num(stats.matched_data)); rprintf(FINFO,"File list size: %s\n", @@ -1110,10 +1110,10 @@ static int do_recv(int f_in, int f_out, char *local_name) write_int(f_out, NDX_DONE); send_msg(MSG_STATS, (char*)&stats.total_read, sizeof stats.total_read, 0); - if(protocol_version >= 33) { - char b[8]; - SIVAL64(b, 0, stats.touched_blocks_4k); - send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); + if (protocol_version >= 33) { + char b[8]; + SIVAL64(b, 0, stats.touched_blocks_4k); + send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); } io_flush(FULL_FLUSH); diff --git a/receiver.c b/receiver.c index f30ff6a46..20d0b21f7 100644 --- a/receiver.c +++ b/receiver.c @@ -471,11 +471,8 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, char *data; int32 i; char *map = NULL; - - /* Reset the 4K block tracker's per-file lifetime state. - * Placed here to guarantee a clean state for every new file payload, - * preventing bugs caused by POSIX file descriptor - * recycling when sequential files are assigned the same FD. */ + + /* Reset the block tracker's per-file state in case the OS reuses an fd. */ reset_block_tracker(); #ifdef SUPPORT_PREALLOCATION diff --git a/rsync.1.md b/rsync.1.md index df63f519f..d1c46c58d 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -3540,7 +3540,7 @@ sign) if you want the local shell to expand it. - `Number of 4 KiB logical blocks touched` is the number of unique 4 KiB logical file regions covered by rsync's own write operations. This metric decouples network payload from the scope of local file modifications. - For example, a small amount of *Literal data* can result in a massive + For example, a small amount of *Literal data* can result in a massive number of touched blocks if the modifications are highly scattered across a file, especially when using `--inplace`. - `Number of regular files transferred` is the count of normal files that diff --git a/rsync.h b/rsync.h index b0507139c..e23b188cc 100644 --- a/rsync.h +++ b/rsync.h @@ -125,7 +125,7 @@ * All older protocol versions MUST be compatible with the final, official * release of the protocol, so don't tweak the code to change the protocol * behavior for an older protocol version. */ -#define SUBPROTOCOL_VERSION 0 +#define SUBPROTOCOL_VERSION 8392 /* For testing */ /* We refuse to interoperate with versions that are not in this range. * Note that we assume we'll work with later versions: the onus is on @@ -299,7 +299,7 @@ enum msgcode { MSG_LOG=FLOG, MSG_CLIENT=FCLIENT, /* sibling logging */ MSG_REDO=9, /* reprocess indicated flist index */ MSG_STATS=10, /* message has stats data for generator */ - MSG_BLOCK_STATS=11, /* message has block-level stats for sender */ + MSG_BLOCK_STATS=11,/* message has block-level stats for sender */ MSG_IO_ERROR=22,/* the sending side had an I/O error */ MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */ MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */ diff --git a/testsuite/skiplist/README.md b/testsuite/skiplist/README.md index 31dd457ed..b7bfbe72f 100644 --- a/testsuite/skiplist/README.md +++ b/testsuite/skiplist/README.md @@ -27,6 +27,7 @@ different tests merge cleanly. | `macos.txt` | macOS-only additions | | `cygwin.txt` | Cygwin-only additions | | `proto29.txt` | additions for a `--protocol=29` run, on any platform | +| `proto30.txt` | additions for a `--protocol=30` run, on any platform | Compose them with commas; the result is the union, so listing a test twice is harmless. Plain test names may be mixed in with `@FILE` entries. diff --git a/testsuite/skiplist/proto29.txt b/testsuite/skiplist/proto29.txt index 2ec82b850..9ce01019d 100644 --- a/testsuite/skiplist/proto29.txt +++ b/testsuite/skiplist/proto29.txt @@ -14,3 +14,4 @@ daemon-copylinks-parent-target-regression # the stdio_daemon client speaks prot partial-protected-regular-retry-linux # one-inplace partial staging needs protocol >= 30 (forced 29) scanner-batch-flag-mismatch # xattrs (-X) need protocol 30+ symlink-exclude-xattr # xattr (-X) transfer requires protocol 30+ (negotiated 29) +write-touched-blocks # logical-block statistics require protocol 33+ (forced 29) diff --git a/testsuite/skiplist/proto30.txt b/testsuite/skiplist/proto30.txt new file mode 100644 index 000000000..f3575cfbe --- /dev/null +++ b/testsuite/skiplist/proto30.txt @@ -0,0 +1,9 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Additions for a --protocol=30 run (make check30), on top of the platform +# files. + +write-touched-blocks # logical-block statistics require protocol 33+ (forced 30) diff --git a/testsuite/write-touched-blocks_test.py b/testsuite/write-touched-blocks_test.py index dc20d6c07..0e58571ee 100644 --- a/testsuite/write-touched-blocks_test.py +++ b/testsuite/write-touched-blocks_test.py @@ -15,7 +15,7 @@ prot_version = int(arg.split('=')[1]) if prot_version < 33: test_skipped(f"Skipping write-touched-blocks: feature requires protocol 33, but CI forced {prot_version}") - + src = FROMDIR makepath(src) @@ -96,7 +96,7 @@ def run_client(src_path, dest_path): f.seek(4 * 1024 * 1024, os.SEEK_CUR) # The Hole (4 MiB of nothing) f.write(os.urandom(4096)) # Block 1026 (Data) -# We must run this WITH --sparse (-S) and WITHOUT --inplace to force +# We must run this WITH --sparse (-S) and WITHOUT --inplace to force # the receiver to create a brand new sparse file from scratch using write_sparse(). rsync_cmd = shlex.split(str(RSYNC)) argv_sparse = rsync_cmd + ['-a', '--stats', '--sparse', str(sparse_src), str(sparse_dest)] @@ -104,14 +104,14 @@ def run_client(src_path, dest_path): if proc.returncode != 0: test_fail(f"rsync failed on sparse test:\n{proc.stdout}") - + # Even though the file is over 4 MiB in size, only 2 logical 4K blocks # should be written because the rest was skipped via lseek. if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: test_fail(f"Sparse check failed! Expected 2 logical blocks written. Output:\n{proc.stdout}") # TEST 6: Multiple Files -# Creates two separate 4KB files. If the tracker fails to reset between +# Creates two separate 4KB files. If the tracker fails to reset between # files due to FD recycling, it will report 1 block instead of 2. fd_src_dir = src / 'fd_test' fd_dest_dir = SCRATCHDIR / 'fd_dest'