diff --git a/src/borg/archive.py b/src/borg/archive.py index 5c7f2f2ddc..2b661dab79 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1877,12 +1877,14 @@ def __init__( log_json, file_status_printer=None, digest_algos=DIGEST_ALGOS_DEFAULT, + strip_components=0, ): self.cache = cache self.key = key self.add_item = add_item self.process_file_chunks = process_file_chunks self.show_progress = show_progress + self.strip_components = strip_components self.print_file_status = file_status_printer or (lambda *args: None) self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) @@ -1942,6 +1944,9 @@ def s_to_ns(s): item.acl_default = value.encode("utf-8", errors="surrogateescape") if xattrs: item.xattrs = xattrs + if self.strip_components: + # the caller already skips members with too few path components, so this never yields an empty path. + item.path = "/".join(item.path.split("/")[self.strip_components :]) yield item, status # if we get here, "with"-block worked ok without error/exception, the item was processed ok... self.add_item(item, stats=self.stats) @@ -1969,6 +1974,9 @@ def process_hardlink(self, *, tarinfo, status, type): # create a not hardlinked borg item, reusing the chunks, see HardLinkManager.__doc__ normalized_path = posixpath.normpath(tarinfo.linkname) safe_path = make_path_safe(normalized_path) + if self.strip_components: + # strip the link target like the member paths, so it matches the stripped path remembered in hlm. + safe_path = "/".join(safe_path.split("/")[self.strip_components :]) info = self.hlm.retrieve(safe_path) if info is not None: chunks, digests = info diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 64e91f5132..29ae232a31 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -359,16 +359,20 @@ def define_exclude_and_patterns(add_option, *, tag_files=False, strip_components ) if strip_components: - add_option( - "--strip-components", - metavar="NUMBER", - dest="strip_components", - type=int, - default=0, - action=Highlander, - help="Remove the specified number of leading path elements. " - "Paths with fewer elements will be silently skipped.", - ) + define_strip_components(add_option) + + +def define_strip_components(add_option): + add_option( + "--strip-components", + metavar="NUMBER", + dest="strip_components", + type=int, + default=0, + action=Highlander, + help="Remove the specified number of leading path elements. " + "Paths with fewer elements will be silently skipped.", + ) def define_exclusion_group(subparser, **kwargs): diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 02b5969dc8..cedabdf691 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -2,6 +2,7 @@ import contextlib import logging import os +import posixpath import stat import sys import tarfile @@ -24,6 +25,7 @@ from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, CompressionSpec from ..helpers import DigestAlgos from ..helpers import FilesystemPathSpec +from ..helpers import make_path_safe from ..helpers import remove_surrogates from ..helpers import timestamp, archive_ts_now from ..helpers import basic_json_data, json_print @@ -31,7 +33,7 @@ from ..helpers.argparsing import ArgumentParser from ..manifest import Manifest -from ._common import with_repository, with_archive, Highlander, define_exclusion_group +from ._common import with_repository, with_archive, Highlander, define_exclusion_group, define_strip_components from ._common import build_matcher, build_filter from ..logger import create_logger @@ -553,6 +555,7 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream): start=t0, log_json=args.log_json, ) + strip_components = args.strip_components cp = ChunksProcessor(cache=cache, key=key, add_item=archive.add_item, rechunkify=False) tfo = TarfileObjectProcessors( cache=cache, @@ -564,11 +567,22 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream): show_progress=args.progress, log_json=args.log_json, file_status_printer=self.print_file_status, + strip_components=strip_components, ) + def path_components(name): + # count the path components the same way TarfileObjectProcessors computes the stored path. + path = make_path_safe(posixpath.normpath(name)) + return [] if path == "." else path.split("/") + tar = tarfile.open(fileobj=tarstream, mode="r|", ignore_zeros=args.ignore_zeros) while tarinfo := tar.next(): + if strip_components: + if len(path_components(tarinfo.name)) <= strip_components: + continue # too few path elements: silently skip this member + if tarinfo.islnk() and len(path_components(tarinfo.linkname)) <= strip_components: + continue # hard link pointing to a skipped member: skip it, too if tarinfo.isreg(): status = tfo.process_file(tarinfo=tarinfo, status="A", type=stat.S_IFREG, tar=tar) elif tarinfo.isdir(): @@ -735,6 +749,11 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): Most documentation of borg create applies. Note that this command does not support excluding files. + ``--strip-components`` removes the specified number of leading path elements + from the tar member names when creating the archive items. Members whose path + has fewer or equally many elements are silently skipped. Hard link targets are + stripped accordingly, symbolic link targets are left unchanged. + A ``--sparse`` option (as found in borg create) is not needed: sparse members in input tarballs (old GNU and PAX sparse formats) are read correctly and their holes are stored as deduplicated all-zero chunks. @@ -795,6 +814,7 @@ def build_parser_tar(self, subparsers, common_parser, mid_common_parser): action="store_true", help="ignore zero-filled blocks in the input tarball", ) + define_strip_components(subparser.add_argument) archive_group = subparser.add_argument_group("Archive options") archive_group.add_argument( diff --git a/src/borg/testsuite/archiver/tar_cmds_test.py b/src/borg/testsuite/archiver/tar_cmds_test.py index 8f763891d9..d613dd4d6a 100644 --- a/src/borg/testsuite/archiver/tar_cmds_test.py +++ b/src/borg/testsuite/archiver/tar_cmds_test.py @@ -199,6 +199,72 @@ def test_import_tar_digests(archivers, request): assert tar_item_digests(archiver, "dst-default")["dir/file1"] is None +def test_import_tar_strip_components(archivers, request): + archiver = request.getfixturevalue(archivers) + # use "./"-prefixed member names, as e.g. GNU tar creates them for ./-relative archives. + with tarfile.open("input.tar", "w") as tar: + for name in ("./toplevel", "./toplevel/dir"): + tarinfo = tarfile.TarInfo(name) + tarinfo.type = tarfile.DIRTYPE + tar.addfile(tarinfo) + for name in ("./toplevel/dir/file", "./toplevel/file2"): + data = name.encode() + tarinfo = tarfile.TarInfo(name) + tarinfo.size = len(data) + tar.addfile(tarinfo, io.BytesIO(data)) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "import-tar", "--strip-components=1", "dst", "input.tar") + files = cmd(archiver, "list", "dst", "--format", "{path}{NL}").splitlines() + # the toplevel directory member itself has too few path elements and is skipped + assert set(files) == {"dir", "dir/file", "file2"} + # stripping more components than any member has imports an empty archive + cmd(archiver, "import-tar", "--strip-components=10", "empty", "input.tar") + files = cmd(archiver, "list", "empty", "--format", "{path}{NL}").splitlines() + assert files == [] + + +def test_import_tar_strip_components_links(archivers, request): + archiver = request.getfixturevalue(archivers) + data = b"file content" + with tarfile.open("input.tar", "w") as tar: + tarinfo = tarfile.TarInfo("toplevel/file1") + tarinfo.size = len(data) + tar.addfile(tarinfo, io.BytesIO(data)) + tarinfo = tarfile.TarInfo("toplevel/hardlink1") + tarinfo.type = tarfile.LNKTYPE + tarinfo.linkname = "toplevel/file1" + tar.addfile(tarinfo) + tarinfo = tarfile.TarInfo("toplevel/symlink1") + tarinfo.type = tarfile.SYMTYPE + tarinfo.linkname = "file1" + tar.addfile(tarinfo) + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "import-tar", "--strip-components=1", "dst", "input.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + with open("output/file1", "rb") as f: + assert f.read() == data + # the hard link references the stripped path of file1, so it reuses file1's content chunks + with open("output/hardlink1", "rb") as f: + assert f.read() == data + # symbolic link targets are not stripped + assert os.readlink("output/symlink1") == "file1" + + +def test_import_tar_strip_components_borg_format(archivers, request): + # the BORG tar format restores the items from pax headers, stripping must work for that path, too. + archiver = request.getfixturevalue(archivers) + create_test_files(archiver.input_path, create_hardlinks=False) # hard links become separate files + os.unlink("input/flagfile") + cmd(archiver, "repo-create", "--encryption=none-sha256") + cmd(archiver, "create", "src", "input") + cmd(archiver, "export-tar", "src", "simple.tar", "--tar-format=BORG") + cmd(archiver, "import-tar", "--strip-components=1", "dst", "simple.tar") + with changedir(archiver.output_path): + cmd(archiver, "extract", "dst") + assert_dirs_equal("input", "output", ignore_ns=True, ignore_xattrs=True) + + def test_import_unusual_tar(archivers, request): archiver = request.getfixturevalue(archivers)