Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 41 additions & 14 deletions src/borg/archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -1742,8 +1742,16 @@ def __next__(self):
class ArchiveChecker:
def __init__(self):
self.error_found = False
self.problems_found = 0
self.repairs_done = 0
self.key = None

def _note_problem(self, *, repaired=False):
self.error_found = True
self.problems_found += 1
if repaired and self.repair:
self.repairs_done += 1

def check(
self,
repository,
Expand Down Expand Up @@ -1793,16 +1801,18 @@ def check(
repository.get_manifest()
except NoManifestError:
logger.error("Repository manifest is missing.")
self.error_found = True
rebuild_manifest = True
else:
try:
self.manifest = Manifest.load(repository, (Manifest.Operation.CHECK,), key=self.key)
except IntegrityErrorBase as exc:
logger.error("Repository manifest is corrupted: %s", exc)
self.error_found = True
rebuild_manifest = True
if rebuild_manifest:
if self.repair:
self._note_problem(repaired=True)
else:
self._note_problem()
self.manifest = self.rebuild_manifest()
if find_lost_archives:
self.rebuild_archives_directory()
Expand All @@ -1811,7 +1821,14 @@ def check(
)
self.finish()
if self.error_found:
logger.error("Archive consistency check complete, problems found.")
if self.repair and self.repairs_done:
logger.error(
"Archive consistency check complete, %d problem(s) found, %d repaired.",
self.problems_found,
self.repairs_done,
)
else:
logger.error("Archive consistency check complete, %d problem(s) found.", self.problems_found)
else:
logger.info("Archive consistency check complete, no problems found.")
return self.repair or not self.error_found
Expand Down Expand Up @@ -1866,20 +1883,24 @@ def verify_data(self):
try:
encrypted_data = self.repository.get(chunk_id)
except (Repository.ObjectNotFound, IntegrityErrorBase) as err:
self.error_found = True
errors += 1
logger.error("chunk %s: %s", bin_to_hex(chunk_id), err)
if isinstance(err, IntegrityErrorBase):
defect_chunks.append(chunk_id)
if not self.repair:
self._note_problem()
else:
self._note_problem()
else:
try:
# we must decompress, so it'll call assert_id() in there:
self.repo_objs.parse(chunk_id, encrypted_data, decompress=True, ro_type=ROBJ_DONTCARE)
except IntegrityErrorBase as integrity_error:
self.error_found = True
errors += 1
logger.error("chunk %s, integrity error: %s", bin_to_hex(chunk_id), integrity_error)
defect_chunks.append(chunk_id)
if not self.repair:
self._note_problem()
pi.finish()
if defect_chunks:
if self.repair:
Expand All @@ -1901,6 +1922,7 @@ def verify_data(self):
self.repository.delete(defect_chunk, update_index=False)
# drop it from our own index too, so rebuild_archives reports the file it belongs to.
del self.chunks[defect_chunk]
self._note_problem(repaired=True)
else:
logger.warning("chunk %s not deleted, did not consistently fail.", bin_to_hex(defect_chunk))
else:
Expand Down Expand Up @@ -1954,7 +1976,7 @@ def valid_archive(obj):
meta = self.repo_objs.parse_meta(chunk_id, cdata, ro_type=ROBJ_DONTCARE)
except IntegrityErrorBase as exc:
logger.error("Skipping corrupted chunk: %s", exc)
self.error_found = True
self._note_problem()
continue
if meta["type"] != ROBJ_ARCHIVE_META:
continue
Expand All @@ -1964,7 +1986,7 @@ def valid_archive(obj):
meta, data = self.repo_objs.parse(chunk_id, cdata, ro_type=ROBJ_DONTCARE)
except IntegrityErrorBase as exc:
logger.error("Skipping corrupted chunk: %s", exc)
self.error_found = True
self._note_problem()
continue
if meta["type"] != ROBJ_ARCHIVE_META:
continue # should never happen
Expand All @@ -1985,12 +2007,13 @@ def valid_archive(obj):
f"We already have a soft-deleted archives directory entry for {name} {archive_id_hex}."
)
else:
self.error_found = True
if self.repair:
logger.warning(f"Creating archives directory entry for {name} {archive_id_hex}.")
self.manifest.archives.create(name, archive_id, archive.time)
self._note_problem(repaired=True)
else:
logger.warning(f"Would create archives directory entry for {name} {archive_id_hex}.")
self._note_problem()

pi.finish()
logger.info("Rebuilding missing archives directory entries completed.")
Expand Down Expand Up @@ -2026,7 +2049,7 @@ def verify_file_chunks(archive_name, item):
archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id)
)
)
self.error_found = True
self._note_problem()
offset += size
if "size" in item:
item_size = item.size
Expand Down Expand Up @@ -2060,7 +2083,7 @@ def missing_chunk_detector(chunk_id):
def report(msg, chunk_id, chunk_no):
cid = bin_to_hex(chunk_id)
msg += " [chunk: %06d_%s]" % (chunk_no, cid) # see "debug dump-archive-items"
self.error_found = True
self._note_problem()
logger.error(msg)

def list_keys_safe(keys):
Expand Down Expand Up @@ -2146,25 +2169,29 @@ def valid_item(obj):
f"Analyzing archive {info.name} {info.ts.astimezone()} {archive_id_hex} ({i + 1}/{num_archives})"
)
if archive_id not in self.chunks:
logger.error(f"Archive metadata block {archive_id_hex} is missing!")
self.error_found = True
if self.repair:
logger.error(f"Archive metadata block {archive_id_hex} is missing!")
logger.error(f"Deleting broken archive {info.name} {archive_id_hex}.")
self.manifest.archives.delete_by_id(archive_id)
self._note_problem(repaired=True)
else:
logger.error(f"Archive metadata block {archive_id_hex} is missing!")
logger.error(f"Would delete broken archive {info.name} {archive_id_hex}.")
self._note_problem()
continue
cdata = self.repository.get(archive_id)
try:
_, data = self.repo_objs.parse(archive_id, cdata, ro_type=ROBJ_ARCHIVE_META)
except IntegrityErrorBase as integrity_error:
logger.error(f"Archive metadata block {archive_id_hex} is corrupted: {integrity_error}")
self.error_found = True
if self.repair:
logger.error(f"Archive metadata block {archive_id_hex} is corrupted: {integrity_error}")
logger.error(f"Deleting broken archive {info.name} {archive_id_hex}.")
self.manifest.archives.delete_by_id(archive_id)
self._note_problem(repaired=True)
else:
logger.error(f"Archive metadata block {archive_id_hex} is corrupted: {integrity_error}")
logger.error(f"Would delete broken archive {info.name} {archive_id_hex}.")
self._note_problem()
continue
archive = self.key.unpack_archive(data)
archive = ArchiveItem(internal_dict=archive)
Expand Down
7 changes: 5 additions & 2 deletions src/borg/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -745,9 +745,12 @@ def store_list(namespace):
if objs_errors == 0:
logger.info(f"Finished {mode} repository check, no problems found.")
elif repair:
logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
logger.error(
f"Finished {mode} repository check, {objs_errors} error(s) found "
f"(repository repair not implemented)."
)
else:
logger.error(f"Finished {mode} repository check, errors found.")
logger.error(f"Finished {mode} repository check, {objs_errors} error(s) found.")
return objs_errors == 0 or repair

def list(self, limit=None, marker=None):
Expand Down
6 changes: 6 additions & 0 deletions src/borg/testsuite/archiver/check_cmd_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ def test_corrupted_manifest(archivers, request):
output = cmd(archiver, "check", "-v", "--repair", exit_code=0)
assert "archive1" in output
assert "archive2" in output
assert "problem(s) found" in output
assert "repaired" in output
cmd(archiver, "check", exit_code=0)


Expand Down Expand Up @@ -401,6 +403,8 @@ def test_verify_data(archivers, request, init_args):
output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0)
assert f"{bin_to_hex(chunk.id)}, integrity error" in output
assert f"{src_file}: Missing file chunk detected" in output
assert "problem(s) found" in output
assert "repaired" in output

# run with --verify-data again, it will notice the missing chunk.
output = cmd(archiver, "check", "--archives-only", "--verify-data", exit_code=1)
Expand Down Expand Up @@ -433,6 +437,8 @@ def test_corrupted_file_chunk(archivers, request, init_args):
output = cmd(archiver, "check", "--repair", "--verify-data", exit_code=0)
assert f"{bin_to_hex(chunk.id)}, integrity error" in output
assert f"{src_file}: Missing file chunk detected" in output
assert "problem(s) found" in output
assert "repaired" in output

# run normal check again
cmd(archiver, "check", "--repository-only", exit_code=0)
Expand Down
Loading