From f7401311e2d4a85c58c1d775abf83f721903692b Mon Sep 17 00:00:00 2001 From: Evgeny Pavlov Date: Wed, 18 Nov 2026 15:51:28 -0700 Subject: [PATCH] Renew weave's W&B access token on each WIF identity token refresh --- docs/hackbot/security.md | 3 ++ .../hackbot_runtime/wandb_wif.py | 30 +++++++++++++++++-- libs/hackbot-runtime/tests/test_wandb_wif.py | 28 +++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/docs/hackbot/security.md b/docs/hackbot/security.md index 6fe2175e998..3fa7b6f62c9 100644 --- a/docs/hackbot/security.md +++ b/docs/hackbot/security.md @@ -49,6 +49,9 @@ The runtime fetches that token from the GCP metadata server, writes it to a priv SDK reads, and keeps refreshing it in the background — Google tokens outlive neither a long run nor a single exchange, so the file has to stay fresh for the whole run ([hackbot_runtime/anthropic_wif.py](../../libs/hackbot-runtime/hackbot_runtime/anthropic_wif.py)). +Weave also caches the W&B access token past the point W&B accepts it, so each refresh drops +that cache too +([hackbot_runtime/wandb_wif.py](../../libs/hackbot-runtime/hackbot_runtime/wandb_wif.py)). Federation is enabled by the presence of `ANTHROPIC_FEDERATION_RULE_ID`. An `ANTHROPIC_API_KEY` set _alongside_ it is refused with an error, because the key would take diff --git a/libs/hackbot-runtime/hackbot_runtime/wandb_wif.py b/libs/hackbot-runtime/hackbot_runtime/wandb_wif.py index 379b9e2f228..4b6f1f3e08c 100644 --- a/libs/hackbot-runtime/hackbot_runtime/wandb_wif.py +++ b/libs/hackbot-runtime/hackbot_runtime/wandb_wif.py @@ -14,9 +14,9 @@ The deploy provisions the audience (which must match the W&B org's federation trust config); this module owns the token file. It fetches the identity token, writes it to a private file, points ``WANDB_IDENTITY_TOKEN_FILE`` at it, and keeps -it fresh in a background thread — Google identity tokens live ~1h and the SDK -re-reads the file when it re-exchanges, so a periodically-rewritten file keeps -auth alive for runs longer than a token's lifetime. +it fresh in a background thread — Google identity tokens live ~1h. After each +rewrite it drops the SDK's cached W&B access token, so the next request exchanges +the fresh identity token and runs outlive any single token. When ``WANDB_FEDERATION_AUDIENCE`` is absent the runtime is in API-key mode and this module is inert, so local/compose runs keep using ``WANDB_API_KEY``. @@ -108,6 +108,28 @@ def _refresh_interval() -> float: return value +def _drop_cached_access_token() -> None: + """Make weave exchange the freshly written identity token on its next request. + + weave keeps the W&B access token it exchanged, in memory and in its + credentials file, until its stated expiry, but W&B stops accepting it sooner. + """ + try: + from weave.wandb_interface.auth import ( + IdentityTokenCredentials, + get_wandb_credentials, + ) + + credentials = get_wandb_credentials() + if not isinstance(credentials, IdentityTokenCredentials): + return + with credentials._lock: + credentials._cached_access_token = None + credentials.credentials_file.unlink(missing_ok=True) + except Exception: + log.exception("Failed to drop weave's cached W&B access token") + + def _default_token_path() -> Path: """A private, process-owned path for the identity token file. @@ -157,6 +179,8 @@ def _loop(self) -> None: "serving the previously written token: %s", exc, ) + continue + _drop_cached_access_token() def start(self) -> None: self._write_token() diff --git a/libs/hackbot-runtime/tests/test_wandb_wif.py b/libs/hackbot-runtime/tests/test_wandb_wif.py index 9d9dab5b931..b7396007977 100644 --- a/libs/hackbot-runtime/tests/test_wandb_wif.py +++ b/libs/hackbot-runtime/tests/test_wandb_wif.py @@ -216,3 +216,31 @@ def _boom(*_a, **_k): refresher._loop() assert calls == [1] + + +def test_refresh_loop_forces_a_new_exchange(monkeypatch, tmp_path): + from weave.wandb_interface import auth + + token_file = tmp_path / "identity-token" + credentials_file = tmp_path / "credentials.json" + monkeypatch.setenv(wandb_wif.TOKEN_FILE_ENV, str(token_file)) + monkeypatch.setenv(auth.CREDENTIALS_FILE_ENV, str(credentials_file)) + monkeypatch.delenv("WANDB_API_KEY", raising=False) + credentials = auth.get_wandb_credentials() + credentials._cached_access_token = ("stale", None) + credentials_file.write_text("{}") + + refresher = wandb_wif._TokenFileRefresher( + token_file, interval=0, audience=_AUDIENCE + ) + + def _fetch(*_a, **_k): + refresher.stop() + return "fresh.jwt" + + monkeypatch.setattr(wandb_wif, "fetch_gcp_identity_token", _fetch) + + refresher._loop() + + assert credentials._cached_access_token is None + assert not credentials_file.exists()