From 7ba70b1e495f4c5f890820205dd0efbe0770a324 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Mon, 6 Jul 2026 12:23:26 +0200 Subject: [PATCH 1/2] Serialise publishes to avoid concurrent-rsync failures --- .gitignore | 4 +- build_docs.py | 115 +++++++++++++++++++++++++++++++------------------- 2 files changed, 75 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index c257236..a0db69e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ /build_root/ /logs/ /www/ -# temporary lock file created while building the docs +# temporary lock files created while building the docs build_docs.lock build_docs_archives.lock build_docs_html.lock +build_docs_html_en.lock +publish-*.lock # Created by https://www.gitignore.io/api/python diff --git a/build_docs.py b/build_docs.py index b526d7b..6b935d8 100755 --- a/build_docs.py +++ b/build_docs.py @@ -553,6 +553,27 @@ def edit(file: Path): temporary.rename(file) +@contextmanager +def wait_for_lock( + path: Path, timeout: float = 600, poll_interval: float = 10 +) -> Iterator[None]: + """Context manager to hold *path* as a lock file, waiting up to *timeout* seconds for it.""" + deadline = perf_counter() + timeout + while True: + try: + lock = zc.lockfile.LockFile(path) + break + except zc.lockfile.LockError: + if perf_counter() >= deadline: + raise + logging.info("Waiting for lock %s...", path.name) + sleep(poll_interval) + try: + yield + finally: + lock.close() + + def setup_switchers(script_content: bytes, html_root: Path) -> None: """Setup cross-links between CPython versions: - Cross-link various languages in a language switcher @@ -826,50 +847,58 @@ def copy_build_to_webroot(self, http: urllib3.PoolManager) -> None: language_dir.chmod(0o775) target = language_dir / self.build_meta.version - target.mkdir(parents=True, exist_ok=True) - try: - target.chmod(0o775) - except PermissionError as err: - logging.warning("Can't change mod of %s: %s", target, str(err)) - chgrp(target, group=self.group, recursive=True) - - changed = 0 - if self.includes_html: - # Copy built HTML files to webroot (default /srv/docs.python.org) - changed += changed_files(self.checkout / "Doc" / "build" / "html", target) - logging.info("Copying HTML files to %s", target) - chgrp( - self.checkout / "Doc" / "build" / "html/", - group=self.group, - recursive=True, - ) - chmod_make_readable(self.checkout / "Doc" / "build" / "html") - run(( - "rsync", - "-a", - "--delete-delay", - "--filter", - "P archives/", - str(self.checkout / "Doc" / "build" / "html") + "/", - target, - )) - - dist_dir = self.checkout / "Doc" / "dist" - if dist_dir.is_dir(): - # Copy archive files to /archives/ - logging.debug("Copying dist files.") - chgrp(dist_dir, group=self.group, recursive=True) - chmod_make_readable(dist_dir) - archives_dir = target / "archives" - archives_dir.mkdir(parents=True, exist_ok=True) - archives_dir.chmod( - archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH - ) - chgrp(archives_dir, group=self.group) - changed += 1 - for dist_file in dist_dir.iterdir(): - shutil.copy2(dist_file, archives_dir / dist_file.name) + # Builds run concurrently but may publish the same language/version to + # the same directory, so serialise publishes per target. + # Contention is expected, but brief, so wait instead of dying. + with wait_for_lock( + HERE / f"publish-{self.build_meta.language}-{self.build_meta.version}.lock" + ): + target.mkdir(parents=True, exist_ok=True) + try: + target.chmod(0o775) + except PermissionError as err: + logging.warning("Can't change mod of %s: %s", target, str(err)) + chgrp(target, group=self.group, recursive=True) + + changed = 0 + if self.includes_html: + # Copy built HTML files to webroot (default /srv/docs.python.org) + changed += changed_files( + self.checkout / "Doc" / "build" / "html", target + ) + logging.info("Copying HTML files to %s", target) + chgrp( + self.checkout / "Doc" / "build" / "html/", + group=self.group, + recursive=True, + ) + chmod_make_readable(self.checkout / "Doc" / "build" / "html") + run(( + "rsync", + "-a", + "--delete-delay", + "--filter", + "P archives/", + str(self.checkout / "Doc" / "build" / "html") + "/", + target, + )) + + dist_dir = self.checkout / "Doc" / "dist" + if dist_dir.is_dir(): + # Copy archive files to /archives/ + logging.debug("Copying dist files.") + chgrp(dist_dir, group=self.group, recursive=True) + chmod_make_readable(dist_dir) + archives_dir = target / "archives" + archives_dir.mkdir(parents=True, exist_ok=True) + archives_dir.chmod( + archives_dir.stat().st_mode | stat.S_IROTH | stat.S_IXOTH + ) + chgrp(archives_dir, group=self.group) changed += 1 + for dist_file in dist_dir.iterdir(): + shutil.copy2(dist_file, archives_dir / dist_file.name) + changed += 1 logging.info("%s files changed", changed) if changed and not self.skip_cache_invalidation: From 4c311c141e4271fa115e76f53075b32f31e95637 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Wed, 8 Jul 2026 12:53:51 +0200 Subject: [PATCH 2/2] Apply @claude's (Hugo's?) suggestion --- build_docs.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/build_docs.py b/build_docs.py index 6b935d8..f76cf1a 100755 --- a/build_docs.py +++ b/build_docs.py @@ -563,9 +563,11 @@ def wait_for_lock( try: lock = zc.lockfile.LockFile(path) break - except zc.lockfile.LockError: + except zc.lockfile.LockError as err: if perf_counter() >= deadline: - raise + raise TimeoutError( + f"Gave up waiting for lock {path.name} after {timeout} seconds" + ) from err logging.info("Waiting for lock %s...", path.name) sleep(poll_interval) try: @@ -684,6 +686,13 @@ def run(self, http: urllib3.PoolManager, force_build: bool) -> bool | None: ) else: return None # skipped + except TimeoutError as err: + # Another builder held the publish lock for too long; the docs + # were built fine and will be published on the next run. + logging.error("%s", err) + if sentry_sdk: + sentry_sdk.capture_exception(err) + return False except Exception as err: logging.exception("Badly handled exception, human, please help.") if sentry_sdk: