Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/hackbot/security.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ The runtime fetches that token from the GCP metadata server, writes it to a priv
SDK reads, and keeps refreshing it in the background — Google tokens outlive neither a long
run nor a single exchange, so the file has to stay fresh for the whole run
([hackbot_runtime/anthropic_wif.py](../../libs/hackbot-runtime/hackbot_runtime/anthropic_wif.py)).
Weave also caches the W&B access token past the point W&B accepts it, so each refresh drops
that cache too
([hackbot_runtime/wandb_wif.py](../../libs/hackbot-runtime/hackbot_runtime/wandb_wif.py)).

Federation is enabled by the presence of `ANTHROPIC_FEDERATION_RULE_ID`. An
`ANTHROPIC_API_KEY` set _alongside_ it is refused with an error, because the key would take
Expand Down
30 changes: 27 additions & 3 deletions libs/hackbot-runtime/hackbot_runtime/wandb_wif.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
The deploy provisions the audience (which must match the W&B org's federation
trust config); this module owns the token file. It fetches the identity token,
writes it to a private file, points ``WANDB_IDENTITY_TOKEN_FILE`` at it, and keeps
it fresh in a background thread — Google identity tokens live ~1h and the SDK
re-reads the file when it re-exchanges, so a periodically-rewritten file keeps
auth alive for runs longer than a token's lifetime.
it fresh in a background thread — Google identity tokens live ~1h. After each
rewrite it drops the SDK's cached W&B access token, so the next request exchanges
the fresh identity token and runs outlive any single token.

When ``WANDB_FEDERATION_AUDIENCE`` is absent the runtime is in API-key mode and
this module is inert, so local/compose runs keep using ``WANDB_API_KEY``.
Expand Down Expand Up @@ -108,6 +108,28 @@ def _refresh_interval() -> float:
return value


def _drop_cached_access_token() -> None:
"""Make weave exchange the freshly written identity token on its next request.

weave keeps the W&B access token it exchanged, in memory and in its
credentials file, until its stated expiry, but W&B stops accepting it sooner.
"""
try:
from weave.wandb_interface.auth import (
IdentityTokenCredentials,
get_wandb_credentials,
)

credentials = get_wandb_credentials()
if not isinstance(credentials, IdentityTokenCredentials):
return
with credentials._lock:
credentials._cached_access_token = None
credentials.credentials_file.unlink(missing_ok=True)
except Exception:
log.exception("Failed to drop weave's cached W&B access token")


def _default_token_path() -> Path:
"""A private, process-owned path for the identity token file.

Expand Down Expand Up @@ -157,6 +179,8 @@ def _loop(self) -> None:
"serving the previously written token: %s",
exc,
)
continue
_drop_cached_access_token()

def start(self) -> None:
self._write_token()
Expand Down
28 changes: 28 additions & 0 deletions libs/hackbot-runtime/tests/test_wandb_wif.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,31 @@ def _boom(*_a, **_k):
refresher._loop()

assert calls == [1]


def test_refresh_loop_forces_a_new_exchange(monkeypatch, tmp_path):
from weave.wandb_interface import auth

token_file = tmp_path / "identity-token"
credentials_file = tmp_path / "credentials.json"
monkeypatch.setenv(wandb_wif.TOKEN_FILE_ENV, str(token_file))
monkeypatch.setenv(auth.CREDENTIALS_FILE_ENV, str(credentials_file))
monkeypatch.delenv("WANDB_API_KEY", raising=False)
credentials = auth.get_wandb_credentials()
credentials._cached_access_token = ("stale", None)
credentials_file.write_text("{}")

refresher = wandb_wif._TokenFileRefresher(
token_file, interval=0, audience=_AUDIENCE
)

def _fetch(*_a, **_k):
refresher.stop()
return "fresh.jwt"

monkeypatch.setattr(wandb_wif, "fetch_gcp_identity_token", _fetch)

refresher._loop()

assert credentials._cached_access_token is None
assert not credentials_file.exists()