diff --git a/NEWS.md b/NEWS.md index ab8f86750..7700dcd89 100644 --- a/NEWS.md +++ b/NEWS.md @@ -17,6 +17,17 @@ - Changed to `/` immediately after a module chroot, before selecting an inner module directory. A failed working-directory reset aborts the connection. +### ENHANCEMENTS: + +- Added SHA-3 (FIPS 202) as an opt-in `--checksum-choice` algorithm + family: `sha3-256`, `sha3-384`, and `sha3-512`. The implementation + uses OpenSSL EVP and so requires that rsync be built with OpenSSL + support and linked against libcrypto >= 1.1.1. Like `sha256` and + `sha512`, SHA-3 is never picked by `auto` negotiation; it must be + explicitly requested on both ends (e.g. + `--checksum-choice=sha3-256`) or made part of the + `RSYNC_CHECKSUM_LIST` environment override. + ### BUILD AND TESTS: - Reworked test discovery, platform profiles and JSON test results. The runner now lives under `testsuite/` and uses Python 3.7 or later with no third-party diff --git a/checksum.c b/checksum.c index 4c91c2b2b..264283d7a 100644 --- a/checksum.c +++ b/checksum.c @@ -54,6 +54,15 @@ struct name_num_item valid_checksums_items[] = { #ifdef SUPPORT_XXHASH { CSUM_XXH64, 0, "xxh64", NULL }, { CSUM_XXH64, 0, "xxhash", NULL }, +#endif +#ifdef USE_OPENSSL + /* SHA-3 is offered only via OpenSSL EVP (sha3-* digests, OpenSSL >= 1.1.1). + * verify_digest() probes each entry at startup and disables any that the + * linked libcrypto does not support, so listing them here is safe even + * when the build environment is older. */ + { CSUM_SHA3_512, NNI_EVP, "sha3-512", NULL }, + { CSUM_SHA3_384, NNI_EVP, "sha3-384", NULL }, + { CSUM_SHA3_256, NNI_EVP, "sha3-256", NULL }, #endif { CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL }, { CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL }, @@ -255,6 +264,14 @@ int csum_len_for_type(int cst, BOOL flist_csum) #ifdef SHA512_DIGEST_LENGTH case CSUM_SHA512: return SHA512_DIGEST_LENGTH; +#endif +#ifdef USE_OPENSSL + case CSUM_SHA3_256: + return SHA3_256_DIGEST_LEN; + case CSUM_SHA3_384: + return SHA3_384_DIGEST_LEN; + case CSUM_SHA3_512: + return SHA3_512_DIGEST_LEN; #endif case CSUM_XXH64: case CSUM_XXH3_64: @@ -284,6 +301,9 @@ int canonical_checksum(int csum_type) case CSUM_SHA1: case CSUM_SHA256: case CSUM_SHA512: + case CSUM_SHA3_256: + case CSUM_SHA3_384: + case CSUM_SHA3_512: return -1; case CSUM_XXH64: case CSUM_XXH3_64: diff --git a/lib/md-defines.h b/lib/md-defines.h index 6ef6a6897..0b58c575c 100644 --- a/lib/md-defines.h +++ b/lib/md-defines.h @@ -10,6 +10,15 @@ #define MD4_DIGEST_LEN 16 #define MD5_DIGEST_LEN 16 + +/* SHA-3 (FIPS 202). OpenSSL exposes these via EVP only, with no + * SHA3_*_DIGEST_LENGTH macros, so define our own. SHA3-512 shares + * its 64-byte digest length with SHA-512, so MAX_DIGEST_LEN is + * unaffected. */ +#define SHA3_256_DIGEST_LEN 32 +#define SHA3_384_DIGEST_LEN 48 +#define SHA3_512_DIGEST_LEN 64 + #if defined SHA512_DIGEST_LENGTH #define MAX_DIGEST_LEN SHA512_DIGEST_LENGTH #elif defined SHA256_DIGEST_LENGTH @@ -35,3 +44,6 @@ #define CSUM_SHA1 9 #define CSUM_SHA256 10 #define CSUM_SHA512 11 +#define CSUM_SHA3_256 12 +#define CSUM_SHA3_384 13 +#define CSUM_SHA3_512 14 diff --git a/rsync.1.md b/rsync.1.md index fd63cde8a..0e3bdff1a 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -2029,11 +2029,18 @@ sign) if you want the local shell to expand it. - `xxh128` - `xxh3` - `xxh64` (aka `xxhash`) + - `sha3-512` + - `sha3-384` + - `sha3-256` - `md5` - `md4` - `sha1` - `none` + The `sha3-*` choices require an rsync built with OpenSSL support and + a libcrypto that provides SHA-3 (OpenSSL 1.1.1 or newer). They are + never selected by `auto` negotiation and must be requested explicitly. + Run `rsync --version` to see the default checksum list compiled into your version (which may differ from the list above). diff --git a/testsuite/tests/transfer/checksum-sha3.py b/testsuite/tests/transfer/checksum-sha3.py new file mode 100755 index 000000000..2762f9e2c --- /dev/null +++ b/testsuite/tests/transfer/checksum-sha3.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# SHA-3 checksum support (sha3-256, sha3-384, sha3-512). +# +# Needs rsync built with OpenSSL against a libcrypto that provides the SHA-3 +# family (OpenSSL 1.1.1 or newer); without it the test reports the openssl +# capability as unsupported. +# +# Each algorithm is used for the whole-file transfer checksum, for the +# pre-transfer file checksum (-c), and as an explicit xfer,file pair; then a +# delta update of a modified destination is checked with sha3-256. + +import json +import os + +from harness.environment import FROMDIR, TODIR +from harness.filesystem import assert_same, makepath, rmtree, walk_files +from harness.results import test_fail, test_skipped +from harness.rsync import run_rsync +from harness import metadata + +metadata(features={'openssl'}, transports={'pipe'}) + +ALGOS = ('sha3-256', 'sha3-384', 'sha3-512') + +vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout) +missing = [a for a in ALGOS if a not in vv.get('checksum_list', [])] +if missing: + test_skipped(f"rsync was not built with OpenSSL SHA-3 support (missing: {', '.join(missing)})", + capability='openssl') + +rmtree(FROMDIR) +makepath(FROMDIR / 'sub') +# Empty, small and chunk-spanning sizes (CHUNK_SIZE is 32k internally). +(FROMDIR / 'empty').write_bytes(b'') +(FROMDIR / 'small').write_text('hello sha-3\n') +(FROMDIR / 'sub' / 'big').write_bytes(os.urandom(70 * 1024)) + +src, dst = f'{FROMDIR}/', f'{TODIR}/' + + +def check_tree(label): + names = sorted(p.relative_to(FROMDIR) for p in walk_files(FROMDIR)) + if names != sorted(p.relative_to(TODIR) for p in walk_files(TODIR)): + test_fail(f'{label}: file lists differ') + for rel in names: + assert_same(TODIR / rel, FROMDIR / rel, label=f'{label} {rel}') + + +for algo in ALGOS: + # Whole-file path (transfer checksum only). + rmtree(TODIR) + run_rsync('-a', f'--checksum-choice={algo}', src, dst) + check_tree(algo) + # Pre-transfer + transfer checksum path (-c forces file_checksum to run). + rmtree(TODIR) + run_rsync('-ac', f'--checksum-choice={algo}', src, dst) + check_tree(f'{algo} -c') + # --checksum-choice with an explicit xfer,file pair. + rmtree(TODIR) + run_rsync('-ac', f'--checksum-choice={algo},{algo}', src, dst) + check_tree(f'{algo},{algo} -c') + +# Delta-update path: change the destination, re-sync with SHA-3, confirm it matches again. +rmtree(TODIR) +run_rsync('-a', '--checksum-choice=sha3-256', src, dst) +with open(TODIR / 'small', 'a') as f: + f.write('modified locally\n') +run_rsync('-a', '--no-whole-file', '--checksum-choice=sha3-256', src, dst) +check_tree('sha3-256 delta') +if (TODIR / 'small').read_text() != 'hello sha-3\n': + test_fail('delta update with sha3-256 did not restore the source content') + +print('checksum-sha3: sha3-256/384/512 transfer and file checksums, and a sha3-256 delta update')