Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,17 @@
- Changed to `/` immediately after a module chroot, before selecting an inner
module directory. A failed working-directory reset aborts the connection.

### ENHANCEMENTS:

- Added SHA-3 (FIPS 202) as an opt-in `--checksum-choice` algorithm
family: `sha3-256`, `sha3-384`, and `sha3-512`. The implementation
uses OpenSSL EVP and so requires that rsync be built with OpenSSL
support and linked against libcrypto >= 1.1.1. Like `sha256` and
`sha512`, SHA-3 is never picked by `auto` negotiation; it must be
explicitly requested on both ends (e.g.
`--checksum-choice=sha3-256`) or made part of the
`RSYNC_CHECKSUM_LIST` environment override.

### BUILD AND TESTS:
- Reworked test discovery, platform profiles and JSON test results. The runner
now lives under `testsuite/` and uses Python 3.7 or later with no third-party
Expand Down
20 changes: 20 additions & 0 deletions checksum.c
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@ struct name_num_item valid_checksums_items[] = {
#ifdef SUPPORT_XXHASH
{ CSUM_XXH64, 0, "xxh64", NULL },
{ CSUM_XXH64, 0, "xxhash", NULL },
#endif
#ifdef USE_OPENSSL
/* SHA-3 is offered only via OpenSSL EVP (sha3-* digests, OpenSSL >= 1.1.1).
* verify_digest() probes each entry at startup and disables any that the
* linked libcrypto does not support, so listing them here is safe even
* when the build environment is older. */
{ CSUM_SHA3_512, NNI_EVP, "sha3-512", NULL },
{ CSUM_SHA3_384, NNI_EVP, "sha3-384", NULL },
{ CSUM_SHA3_256, NNI_EVP, "sha3-256", NULL },
#endif
{ CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL },
{ CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL },
Expand Down Expand Up @@ -255,6 +264,14 @@ int csum_len_for_type(int cst, BOOL flist_csum)
#ifdef SHA512_DIGEST_LENGTH
case CSUM_SHA512:
return SHA512_DIGEST_LENGTH;
#endif
#ifdef USE_OPENSSL
case CSUM_SHA3_256:
return SHA3_256_DIGEST_LEN;
case CSUM_SHA3_384:
return SHA3_384_DIGEST_LEN;
case CSUM_SHA3_512:
return SHA3_512_DIGEST_LEN;
#endif
case CSUM_XXH64:
case CSUM_XXH3_64:
Expand Down Expand Up @@ -284,6 +301,9 @@ int canonical_checksum(int csum_type)
case CSUM_SHA1:
case CSUM_SHA256:
case CSUM_SHA512:
case CSUM_SHA3_256:
case CSUM_SHA3_384:
case CSUM_SHA3_512:
return -1;
case CSUM_XXH64:
case CSUM_XXH3_64:
Expand Down
12 changes: 12 additions & 0 deletions lib/md-defines.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,15 @@

#define MD4_DIGEST_LEN 16
#define MD5_DIGEST_LEN 16

/* SHA-3 (FIPS 202). OpenSSL exposes these via EVP only, with no
* SHA3_*_DIGEST_LENGTH macros, so define our own. SHA3-512 shares
* its 64-byte digest length with SHA-512, so MAX_DIGEST_LEN is
* unaffected. */
#define SHA3_256_DIGEST_LEN 32
#define SHA3_384_DIGEST_LEN 48
#define SHA3_512_DIGEST_LEN 64

#if defined SHA512_DIGEST_LENGTH
#define MAX_DIGEST_LEN SHA512_DIGEST_LENGTH
#elif defined SHA256_DIGEST_LENGTH
Expand All @@ -35,3 +44,6 @@
#define CSUM_SHA1 9
#define CSUM_SHA256 10
#define CSUM_SHA512 11
#define CSUM_SHA3_256 12
#define CSUM_SHA3_384 13
#define CSUM_SHA3_512 14
7 changes: 7 additions & 0 deletions rsync.1.md
Original file line number Diff line number Diff line change
Expand Up @@ -2029,11 +2029,18 @@ sign) if you want the local shell to expand it.
- `xxh128`
- `xxh3`
- `xxh64` (aka `xxhash`)
- `sha3-512`
- `sha3-384`
- `sha3-256`
- `md5`
- `md4`
- `sha1`
- `none`

The `sha3-*` choices require an rsync built with OpenSSL support and
a libcrypto that provides SHA-3 (OpenSSL 1.1.1 or newer). They are
never selected by `auto` negotiation and must be requested explicitly.

Run `rsync --version` to see the default checksum list compiled into your
version (which may differ from the list above).

Expand Down
73 changes: 73 additions & 0 deletions testsuite/tests/transfer/checksum-sha3.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
# SHA-3 checksum support (sha3-256, sha3-384, sha3-512).
#
# Needs rsync built with OpenSSL against a libcrypto that provides the SHA-3
# family (OpenSSL 1.1.1 or newer); without it the test reports the openssl
# capability as unsupported.
#
# Each algorithm is used for the whole-file transfer checksum, for the
# pre-transfer file checksum (-c), and as an explicit xfer,file pair; then a
# delta update of a modified destination is checked with sha3-256.

import json
import os

from harness.environment import FROMDIR, TODIR
from harness.filesystem import assert_same, makepath, rmtree, walk_files
from harness.results import test_fail, test_skipped
from harness.rsync import run_rsync
from harness import metadata

metadata(features={'openssl'}, transports={'pipe'})

ALGOS = ('sha3-256', 'sha3-384', 'sha3-512')

vv = json.loads(run_rsync('-VV', check=True, capture_output=True).stdout)
missing = [a for a in ALGOS if a not in vv.get('checksum_list', [])]
if missing:
test_skipped(f"rsync was not built with OpenSSL SHA-3 support (missing: {', '.join(missing)})",
capability='openssl')

rmtree(FROMDIR)
makepath(FROMDIR / 'sub')
# Empty, small and chunk-spanning sizes (CHUNK_SIZE is 32k internally).
(FROMDIR / 'empty').write_bytes(b'')
(FROMDIR / 'small').write_text('hello sha-3\n')
(FROMDIR / 'sub' / 'big').write_bytes(os.urandom(70 * 1024))

src, dst = f'{FROMDIR}/', f'{TODIR}/'


def check_tree(label):
names = sorted(p.relative_to(FROMDIR) for p in walk_files(FROMDIR))
if names != sorted(p.relative_to(TODIR) for p in walk_files(TODIR)):
test_fail(f'{label}: file lists differ')
for rel in names:
assert_same(TODIR / rel, FROMDIR / rel, label=f'{label} {rel}')


for algo in ALGOS:
# Whole-file path (transfer checksum only).
rmtree(TODIR)
run_rsync('-a', f'--checksum-choice={algo}', src, dst)
check_tree(algo)
# Pre-transfer + transfer checksum path (-c forces file_checksum to run).
rmtree(TODIR)
run_rsync('-ac', f'--checksum-choice={algo}', src, dst)
check_tree(f'{algo} -c')
# --checksum-choice with an explicit xfer,file pair.
rmtree(TODIR)
run_rsync('-ac', f'--checksum-choice={algo},{algo}', src, dst)
check_tree(f'{algo},{algo} -c')

# Delta-update path: change the destination, re-sync with SHA-3, confirm it matches again.
rmtree(TODIR)
run_rsync('-a', '--checksum-choice=sha3-256', src, dst)
with open(TODIR / 'small', 'a') as f:
f.write('modified locally\n')
run_rsync('-a', '--no-whole-file', '--checksum-choice=sha3-256', src, dst)
check_tree('sha3-256 delta')
if (TODIR / 'small').read_text() != 'hello sha-3\n':
test_fail('delta update with sha3-256 did not restore the source content')

print('checksum-sha3: sha3-256/384/512 transfer and file checksums, and a sha3-256 delta update')