From 0c28844740b59271a36c33ced7d629021d61466c Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 11 Nov 2026 18:35:29 +0530 Subject: [PATCH 1/3] fix(harness): a settings template adds only SDK setting names (TH-9048) A template is free text, and each review of the shape rules found a new shape that put a fragment of a pasted value into env_names. _env_names now adds a template name only when it is one of 46 names SDKs read for themselves (_SDK_SETTINGS); the code's names, their order and the cap are unchanged. Whatever a template holds, it can add only names from that list. The left-out names are explained beside it. Ten tests that expected a template's other names are rewritten to the rule; new tests cover each route, every value shape in four encodings, 200 seeded values, an exact match, shared code, and the list itself, typed out rather than taken from the constant. TH-9048 Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_walk.py | 87 +++++++- tests/harness/test_scan_walk.py | 378 +++++++++++++++++++++++++++++--- 2 files changed, 429 insertions(+), 36 deletions(-) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 1c3741dad..b10cf2ee8 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -96,6 +96,81 @@ _PROJECT_FILES = ("pyproject.toml", "requirements.txt", "setup.py") _PACKAGE_FILE = "package.json" _TEMPLATES = (".env.example", ".env.sample", ".env.template") +# The setting names an SDK reads for itself when the code passes no value, so the customer's +# code never names them. These are the only names a template adds to a row (`_env_names`). +# The LiveKit names and the per-provider keys copy `credentials.py`'s `_CONNECTOR_REQUIREMENTS` +# and `_LIVEKIT_PROVIDER_CREDENTIALS`, which this module doesn't import. +# Left out on purpose: +# - `GOOGLE_APPLICATION_CREDENTIALS`: a key file, which the platform takes through its +# credential-file control, not a text field; +# - `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS`: on hosted runs the platform +# owns trace export, and a filled headers value would be merged into exports sent to its +# collector; +# - LiteLLM's provider names and the provider keys of LiveKit's OpenAI plugin (`with_cerebras`, +# `with_x_ai`, ...), too many to list; +# - the platform's own `ALK_*` names. +_SDK_SETTINGS: frozenset[str] = frozenset( + { + # OpenAI and Azure OpenAI (`openai`): installed, its reading grepped. + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "OPENAI_API_VERSION", + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_AD_TOKEN", + # Google GenAI and Google auth: installed, grepped. + "GOOGLE_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_GENAI_USE_VERTEXAI", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + # LiveKit and its Deepgram, Cartesia and ElevenLabs plugins: installed, grepped. The + # plugin reads `ELEVEN_API_KEY`; `credentials.py` names `ELEVENLABS_API_KEY` (below). + "LIVEKIT_URL", + "LIVEKIT_API_KEY", + "LIVEKIT_API_SECRET", + "DEEPGRAM_API_KEY", + "CARTESIA_API_KEY", + "ELEVEN_API_KEY", + # Future AGI's tracing (`fi_instrumentation`): installed, grepped. + "FI_API_KEY", + "FI_SECRET_KEY", + "FI_BASE_URL", + "FI_PROJECT_NAME", + # OpenTelemetry's SDK (`opentelemetry.sdk.environment_variables`): installed, grepped. + "OTEL_SERVICE_NAME", + # Hugging Face Hub (`huggingface_hub.utils._auth`): installed, grepped. + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + # Anthropic: from its SDK's documentation. + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + # Mistral, Cohere and Groq: from each SDK's documentation. + "MISTRAL_API_KEY", + "CO_API_KEY", + "GROQ_API_KEY", + # ElevenLabs' own SDK: from its documentation. + "ELEVENLABS_API_KEY", + # LangSmith, and LangChain's older names for the same settings: from the documentation. + "LANGSMITH_API_KEY", + "LANGSMITH_TRACING", + "LANGSMITH_PROJECT", + "LANGSMITH_ENDPOINT", + "LANGCHAIN_API_KEY", + "LANGCHAIN_TRACING_V2", + "LANGCHAIN_PROJECT", + "LANGCHAIN_ENDPOINT", + # AWS, through boto3, which `strands` uses for Bedrock: from boto3's documentation. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + "AWS_DEFAULT_REGION", + } +) _LANGGRAPH = "langgraph.json" _SCRIPT_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") _SOURCE_SUFFIXES = (".py", ".ipynb", *_SCRIPT_SUFFIXES) @@ -1411,12 +1486,16 @@ def _first_names(groups: Iterable[Iterable[str]]) -> list[str]: def _env_names(code: list[str], templates: set[str]) -> tuple[str, ...]: - """A row's setting names: the code's first, then a template's that the code doesn't read, - up to `ENV_NAMES_KEPT`, the most the answer file writes. ``code`` is the code's first - names (`_first_names`): when it is short, it is all of them.""" + """A row's setting names: the code's first, then a template's that are in `_SDK_SETTINGS` + and that the code doesn't read, up to `ENV_NAMES_KEPT`, the most the answer file writes. + ``code`` is the code's first names (`_first_names`): when it is short, it is all of them. + + A template adds only SDK names, because a template is free text that can hold pasted + values, and a name the code reads is listed from the code. Both the walk and + `describe_entry` build their rows' names here.""" if len(code) >= ENV_NAMES_KEPT: return tuple(code[:ENV_NAMES_KEPT]) - return (*code, *sorted(templates - set(code)))[:ENV_NAMES_KEPT] + return (*code, *sorted((templates & _SDK_SETTINGS) - set(code)))[:ENV_NAMES_KEPT] def _prompt_count(paths: list[str], subfolder: str) -> int: diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 0ebd273a3..d2ff5325a 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -7,6 +7,7 @@ import itertools import json import os +import random import tokenize import tracemalloc from collections.abc import Iterator @@ -298,11 +299,11 @@ def test_env_files_with_values_are_never_opened( _write( tmp_path, { - ".env": "REAL_SECRET=sk-fake-123\n", + ".env": "GROQ_API_KEY=sk-fake-123\n", ".env.local": "LOCAL_SECRET=sk-fake-456\n", "app/.env": "APP_SECRET=sk-fake-789\n", - ".env.example": "# a note\nEXAMPLE_NAME=sk-fake-000\n" - "export EXPORTED_NAME=1\n BAD-NAME=2\n", + ".env.example": "# a note\nOPENAI_API_KEY=sk-fake-000\n" + "export ANTHROPIC_API_KEY=1\n BAD-NAME=2\n", "bot.py": CLIENT, }, ) @@ -321,7 +322,8 @@ def spy(path: str, limit: int) -> bytes: assert ".env" not in opened assert ".env.local" not in opened [agent] = result.agents - assert agent.env_names == ("EXAMPLE_NAME", "EXPORTED_NAME") + # `GROQ_API_KEY` is an SDK name only `.env` holds: its absence shows `.env` stays unread. + assert agent.env_names == ("ANTHROPIC_API_KEY", "OPENAI_API_KEY") assert "sk-fake" not in repr(result) @@ -644,7 +646,7 @@ def test_an_upload_keeps_one_agent_at_the_root_counted_for_the_root( "svc/graph.py": GRAPH + "import os\nos.getenv('SVC_KEY')\n", "main.py": "import os\nos.getenv('ROOT_KEY')\n@tool\ndef search(): ...\n", "prompts/system.txt": "hello\n", - ".env.example": "TEMPLATE_KEY=x\n", + ".env.example": "OPENAI_API_KEY=x\n", "tests/test_x.py": "import os\nos.getenv('TEST_KEY')\n", "other/bot.py": CLIENT, }, @@ -655,14 +657,14 @@ def test_an_upload_keeps_one_agent_at_the_root_counted_for_the_root( [agent] = uploaded.agents assert (agent.entry, agent.subfolder) == ("svc/graph.py", "") - assert agent.env_names == ("ROOT_KEY", "SVC_KEY", "TEMPLATE_KEY") + assert agent.env_names == ("ROOT_KEY", "SVC_KEY", "OPENAI_API_KEY") assert agent.tools == 1 assert agent.prompt_files == 1 assert agent.build_may_stop == build_hazard(source, "") assert uploaded.upload_others == 1 in_folder = _agent(listed, "svc/graph.py") assert in_folder.subfolder == "svc" - assert in_folder.env_names == ("SVC_KEY", "TEMPLATE_KEY") + assert in_folder.env_names == ("SVC_KEY", "OPENAI_API_KEY") assert (in_folder.tools, in_folder.prompt_files) == (0, 0) side_only = tmp_path / "side" @@ -691,6 +693,7 @@ def test_env_names_are_checked_and_ordered(tmp_path: Path) -> None: "Y" * 128, "A_CODE", "naïve", + "OPENAI_API_KEY", ] code = "import os\n" + CLIENT + "".join(f"os.getenv('{n}')\n" for n in names) _write( @@ -698,22 +701,28 @@ def test_env_names_are_checked_and_ordered(tmp_path: Path) -> None: { "svc/pyproject.toml": "", "svc/bot.py": code, - "svc/.env.example": "B_TEMPLATE=1\nexport A_TEMPLATE=2\nA_CODE=3\n", - ".env.sample": "ROOT_TEMPLATE=1\n", - "elsewhere/.env.template": "ELSEWHERE=1\n", + "svc/.env.example": "GROQ_API_KEY=1\nexport ANTHROPIC_API_KEY=2\nOPENAI_API_KEY=3\n" + "A_CODE=4\nB_TEMPLATE=5\n", + "svc/.env.template": "CO_API_KEY=1\n", + ".env.sample": "MISTRAL_API_KEY=1\n", + "elsewhere/.env.template": "HF_TOKEN=1\n", }, ) [agent] = _walk(tmp_path).agents + # The code's names first, each once, then the SDK names of the agent's folder's templates + # and the root's, sorted; another folder's template adds nothing. assert agent.subfolder == "svc" assert agent.env_names == ( "A_CODE", + "OPENAI_API_KEY", "Y" * 128, "Z_CODE", - "A_TEMPLATE", - "B_TEMPLATE", - "ROOT_TEMPLATE", + "ANTHROPIC_API_KEY", + "CO_API_KEY", + "GROQ_API_KEY", + "MISTRAL_API_KEY", ) @@ -730,7 +739,8 @@ def test_describe_entry_reads_process_env(tmp_path: Path) -> None: "web/src/util.mjs": "export const u = process.env.UTIL_KEY;\n", "web/tests/agent.test.ts": "process.env.TEST_ONLY;\n", "web/node_modules/pkg/index.js": "process.env.DEPENDENCY_KEY;\n", - "web/.env.example": "TEMPLATE_KEY=1\n", + "web/.env.example": "GROQ_API_KEY=1\n", + ".env.sample": "MISTRAL_API_KEY=1\n", "api/server.js": "process.env.ELSEWHERE;\n", }, ) @@ -749,7 +759,8 @@ def test_describe_entry_reads_process_env(tmp_path: Path) -> None: "OTHER_KEY", "THIRD_KEY", "UTIL_KEY", - "TEMPLATE_KEY", + "GROQ_API_KEY", + "MISTRAL_API_KEY", ) @@ -816,6 +827,58 @@ def test_describe_entry_stops_at_its_deadline_with_partial( assert (capped.partial, capped.env_names) == (True, ("FIRST",)) +# The 46 SDK setting names, typed here rather than taken from `_SDK_SETTINGS`, so a change to +# either fails `test_the_sdk_names`: the only names a template adds to a row. +SDK_NAMES = frozenset( + { + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "OPENAI_API_VERSION", + "AZURE_OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT", + "AZURE_OPENAI_AD_TOKEN", + "GOOGLE_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_GENAI_USE_VERTEXAI", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + "LIVEKIT_URL", + "LIVEKIT_API_KEY", + "LIVEKIT_API_SECRET", + "DEEPGRAM_API_KEY", + "CARTESIA_API_KEY", + "ELEVEN_API_KEY", + "FI_API_KEY", + "FI_SECRET_KEY", + "FI_BASE_URL", + "FI_PROJECT_NAME", + "OTEL_SERVICE_NAME", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "MISTRAL_API_KEY", + "CO_API_KEY", + "GROQ_API_KEY", + "ELEVENLABS_API_KEY", + "LANGSMITH_API_KEY", + "LANGSMITH_TRACING", + "LANGSMITH_PROJECT", + "LANGSMITH_ENDPOINT", + "LANGCHAIN_API_KEY", + "LANGCHAIN_TRACING_V2", + "LANGCHAIN_PROJECT", + "LANGCHAIN_ENDPOINT", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + "AWS_DEFAULT_REGION", + } +) # Every value line below carries "FAKE", so any name holding it is a piece of a value. FAKE_KEY_LINE = "FAKEk3yAAAAAbbbbbbbbCCCCCCCCdddddddd0123456789EEEEEEEEffffffffGG" AROUND = ("A_BEFORE", "Z_AFTER") @@ -1000,17 +1063,19 @@ def test_a_template_in_utf16_or_latin1_gives_its_names_and_no_value( ) -> None: # Not UTF-8 either way: a byte-order mark of UTF-16, or a Latin-1 `ü`. text = ( - f"A_BEFORE=1\nCITY=Zürich\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\n" - "Z_AFTER=2\n" + f"OPENAI_API_KEY=1\nCITY=Zürich\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\n" + "FAKEt4il=\nGROQ_API_KEY=2\n" ) - expected = ("A_BEFORE", "CITY", "SIGNING_KEY", "Z_AFTER") + expected = ("CITY", "GROQ_API_KEY", "OPENAI_API_KEY", "SIGNING_KEY") data = encoded(text) # type: ignore[operator] (tmp_path / ".env.example").write_bytes(data) (tmp_path / "bot.py").write_text(CLIENT, encoding="utf-8") problems = _name_problems(scan_walk._template_names(data), expected) walked = _walk(tmp_path) - walked_problems = _name_problems(walked.agents[0].env_names, expected) + # The walk keeps only the template's SDK names. + kept = ("GROQ_API_KEY", "OPENAI_API_KEY") + walked_problems = _name_problems(walked.agents[0].env_names, kept) assert problems == ([], []) assert walked.unread_files == 0 @@ -1052,22 +1117,39 @@ def test_a_multi_line_value_in_a_template_is_never_a_name(tmp_path: Path) -> Non "WRAPPED_BLOB", "lowercasename", ) + direct = scan_walk._template_names(template.encode("utf-8")) + assert _name_problems(direct, expected) == ([], []) + # The names around each value become SDK names, so the routes, which keep only those, still + # show whether the reader found the lines before and after the value. + swap = { + "A_BEFORE": "OPENAI_API_KEY", + "Z_AFTER": "GROQ_API_KEY", + "BEFORE_NAME": "ANTHROPIC_API_KEY", + "AFTER_NAME": "MISTRAL_API_KEY", + } + + def swapped(text: str) -> str: + for old, new in swap.items(): + text = text.replace(old, new) + return text + # Each shape in a repository of its own, so one shape's value can't hide another's. repos = {"mixed": (template, expected), **TEMPLATE_SHAPES} for shape, (text, _) in repos.items(): (tmp_path / shape).mkdir() - (tmp_path / shape / ".env.example").write_bytes(text.encode("utf-8")) + (tmp_path / shape / ".env.example").write_bytes(swapped(text).encode("utf-8")) (tmp_path / shape / "bot.py").write_text(CLIENT, encoding="utf-8") for shape, (_, names) in repos.items(): + kept = tuple(sorted(name for name in map(swapped, names) if name in SDK_NAMES)) [agent] = _walk(tmp_path / shape).agents described = describe_entry(tmp_path / shape, "bot.py", "", deadline=FOREVER) assert described is not None for found in (agent.env_names, described.env_names): - problems = _name_problems(found, names) + problems = _name_problems(found, kept) assert (shape, problems) == (shape, ([], [])) - assert found == tuple(sorted(names)) + assert found == kept assert not any("FAKE" in name.upper() or "MIIE" in name for name in found), shape @@ -1450,8 +1532,8 @@ def test_the_step_after_the_walk_holds_a_bounded_number_of_edges( # A UTF-16 byte-order mark followed by an odd number of bytes: the decoder can't read it. The # same template with its last byte pair whole can. -ODD_UTF16_TEMPLATE = b"\xff\xfeA\x00=\x001\x00\n" -EVEN_UTF16_TEMPLATE = ODD_UTF16_TEMPLATE + b"\x00" +EVEN_UTF16_TEMPLATE = b"\xff\xfe" + "OPENAI_API_KEY=1\n".encode("utf-16-le") +ODD_UTF16_TEMPLATE = EVEN_UTF16_TEMPLATE[:-1] def test_a_template_the_decoder_rejects_marks_the_row_short( @@ -1469,7 +1551,7 @@ def test_a_template_the_decoder_rejects_marks_the_row_short( rejected = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) assert readable is not None and readable.partial is False - assert set(readable.env_names) == {"A", "TOOL_KEY"} + assert set(readable.env_names) == {"OPENAI_API_KEY", "TOOL_KEY"} assert rejected is not None and rejected.partial is True assert set(rejected.env_names) == {"TOOL_KEY"} # Short because the decoder rejected it, not because the kit's reader failed. @@ -1901,24 +1983,34 @@ def _all_names(walk: scan_walk._Walk, subfolder: str) -> list[str]: return sorted({name for path, facts in walk.facts.items() if scan_walk._under(path, subfolder) for name in facts.env}) -@pytest.mark.parametrize("names", [30, 300]) +@pytest.mark.parametrize("names", [0, 30, 49, 300]) def test_a_rows_setting_names_are_its_first_fifty(tmp_path: Path, names: int) -> None: _files(tmp_path, {"bot.py": "from openai import OpenAI\nOpenAI()\n"}) - _files( - tmp_path, - {f"lib/m{i:02}.py": "import os\n" + "".join(f"os.getenv('K{(i * 37 + n) % names:04}')\n" for n in range(40)) for i in range(20)}, - ) - _files(tmp_path, {".env.example": "".join(f"T{n:03}=\n" for n in range(40))}) + for i in range(20 if names else 0): + reads = "".join(f"os.getenv('K{(i * 37 + n) % names:04}')\n" for n in range(40)) + _files(tmp_path, {f"lib/m{i:02}.py": "import os\n" + reads}) + # Every SDK name, and 100 other names a template adds nothing for. + template = sorted(SDK_NAMES) + [f"T{n:03}" for n in range(100)] + _files(tmp_path, {".env.example": "".join(f"{name}=\n" for name in template)}) walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) walk.run() full = _all_names(walk, "") - expected = (*full, *sorted(walk.templates[""] - set(full)))[: scan_walk.ENV_NAMES_KEPT] + sdk = sorted(SDK_NAMES - set(full)) + expected = (*full, *sdk)[: scan_walk.ENV_NAMES_KEPT] row = _agent(walk.result(upload=False), "bot.py") described = describe_entry(tmp_path, "bot.py", "", deadline=FOREVER) + assert len(full) == names assert row.env_names == expected assert described is not None and described.env_names == expected + if names == 0: + # With no code names, a template fills at most the 46 SDK names, sorted. + assert row.env_names == tuple(sorted(SDK_NAMES)) + if names == 49: + # One SDK name fills the 50th place. + assert len(row.env_names) == scan_walk.ENV_NAMES_KEPT + assert row.env_names[-1] == sdk[0] assert scan_walk.ENV_NAMES_KEPT == 50 from fi.alk.harness import scan_document @@ -2297,3 +2389,225 @@ def charging(self: object, **kwargs: object) -> object: monkeypatch.setattr(scan_walk._Walk, "result", charging) assert _walk(tmp_path).stopped == "limit" + + +def _three_routes( + source: Path, entry: str, subfolder: str +) -> dict[str, tuple[str, ...]]: + """A row's setting names on the walk, the description and an upload.""" + described = describe_entry(source, entry, subfolder, deadline=FOREVER) + assert described is not None + return { + "walk": _agent(_walk(source), entry).env_names, + "description": described.env_names, + "upload": _agent(_walk(source, upload=True), entry).env_names, + } + + +def test_a_template_adds_only_sdk_names(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot.py": CLIENT + + "import os\nos.getenv('OPENAI_API_KEY')\nos.getenv('CODE_ONLY')\n", + ".env.example": "OPENAI_API_KEY=\nANTHROPIC_API_KEY=\nexport GROQ_API_KEY=1\n" + "TEMPLATE_ONLY=sk-fake-000\n", + }, + ) + + found = _three_routes(tmp_path, "bot.py", "") + + # The code's names first, `OPENAI_API_KEY` among them once; then the template's SDK names. + expected = ("CODE_ONLY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GROQ_API_KEY") + assert found == {"walk": expected, "description": expected, "upload": expected} + + +# Shapes in which a template's value spans lines or hides a line break. `{value}` is where the +# value's made-up lines go, each shaped like a setting, so a reader that takes one for a setting +# gives a name. +VALUE_SHAPES = { + "open_quote": 'KEY="first\n{value}\n', + "key_block": "PEM=-----BEGIN PRIVATE KEY-----\n{value}\n-----END PRIVATE KEY-----\n", + "wrapped_key": f"KEY={FAKE_KEY_LINE}\n{{value}}\n", + "heredoc": "KEY=< str: + # `CITY` holds a `ü`, so Latin-1 bytes aren't UTF-8; SDK names before and after the value. + body = VALUE_SHAPES[shape].replace("{value}", value) + return f"OPENAI_API_KEY=\nCITY=Zürich\n{body}GROQ_API_KEY=\n" + + +def _never_a_value(found: dict[str, tuple[str, ...]]) -> dict[str, list[int]]: + """Per route, the lengths of the names neither the code nor the SDK list gives. Lengths + only, so a failure never prints a piece of a value.""" + return { + route: sorted( + len(name) for name in names if name != "CODE_KEY" and name not in SDK_NAMES + ) + for route, names in found.items() + } + + +@pytest.mark.parametrize( + ("shape", "encoding"), + [ + (shape, encoding) + for shape in VALUE_SHAPES + for encoding in VALUE_ENCODINGS + if encoding != "latin1" or shape not in NOT_LATIN1 + ], +) +def test_a_template_value_never_becomes_a_name( + tmp_path: Path, shape: str, encoding: str +) -> None: + data = VALUE_ENCODINGS[encoding](_value_template(shape)) + _write(tmp_path, {"bot.py": CODE_READING}) + (tmp_path / ".env.example").write_bytes(data) + + found = _three_routes(tmp_path, "bot.py", "") + + assert _never_a_value(found) == NONE_ON_ANY_ROUTE + assert found["walk"][0] == "CODE_KEY" + # The template was read: the SDK name on the line before the value reaches every route. + assert all("OPENAI_API_KEY" in names for names in found.values()) + + +def test_a_template_value_never_becomes_a_name_in_random_values(tmp_path: Path) -> None: + # The cases above stay live: the reader alone takes some value line for a setting. + leaked = [ + shape + for shape in VALUE_SHAPES + if any( + name.startswith("LEAK_") + for name in scan_walk._template_names( + _value_template(shape).encode("utf-8") + ) + ) + ] + assert leaked + rng = random.Random(9048) + letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_" + for index in range(200): + shape = rng.choice(sorted(VALUE_SHAPES)) + encodings = sorted(VALUE_ENCODINGS) + encoding = rng.choice( + [e for e in encodings if e != "latin1" or shape not in NOT_LATIN1] + ) + # A made-up value wrapped over lines, each line shaped like a setting. + value = "\n".join( + "".join(rng.choice(letters) for _ in range(rng.randint(1, 40))) + + rng.choice(["=", "==", "=fake", ""]) + for _ in range(rng.randint(1, 6)) + ) + source = tmp_path / str(index) + _write(source, {"bot.py": CODE_READING}) + (source / ".env.example").write_bytes( + VALUE_ENCODINGS[encoding](_value_template(shape, value)) + ) + + found = _three_routes(source, "bot.py", "") + + assert _never_a_value(found) == NONE_ON_ANY_ROUTE, index + + +def test_a_value_line_that_spells_an_sdk_name_adds_it(tmp_path: Path) -> None: + template = "KEY=< None: + alone = tmp_path / "alone" + _write( + alone, {"bot.py": CLIENT, ".env.example": "openai_api_key=\nOpenAI_Api_Key=\n"} + ) + beside = tmp_path / "beside" + _write( + beside, + { + "bot.py": CLIENT + "import os\nos.getenv('OPENAI_API_KEY')\n", + ".env.example": "openai_api_key=\n", + }, + ) + + routes = ("walk", "description", "upload") + assert _three_routes(alone, "bot.py", "") == dict.fromkeys(routes, ()) + assert _three_routes(beside, "bot.py", "") == dict.fromkeys( + routes, ("OPENAI_API_KEY",) + ) + + +def test_a_shared_module_does_not_vouch_for_a_template_name(tmp_path: Path) -> None: + agent = CLIENT + "from common.config import settings\n" + _write( + tmp_path, + { + "agents/support/pyproject.toml": "", + "agents/support/agent.py": agent, + "agents/sales/pyproject.toml": "", + "agents/sales/agent.py": agent, + "common/__init__.py": "", + "common/config.py": "import os\nsettings = (\n os.getenv('DATABASE_URL'),\n" + " os.environ.get('SLACK_BOT_TOKEN'),\n os.environ['ZENDESK_API_TOKEN'],\n)\n", + ".env.example": "DATABASE_URL=\nSLACK_BOT_TOKEN=\nZENDESK_API_TOKEN=\n" + "OPENAI_API_KEY=\n", + }, + ) + + walked = _walk(tmp_path) + uploaded = _walk(tmp_path, upload=True) + + # Only the row's own files vouch for a template name; an upload's own files are the root's. + for entry in ("agents/sales/agent.py", "agents/support/agent.py"): + row = _agent(walked, entry) + described = describe_entry(tmp_path, entry, row.subfolder, deadline=FOREVER) + assert row.subfolder == entry.rsplit("/", 1)[0] + assert described is not None + assert (row.env_names, described.env_names) == (("OPENAI_API_KEY",),) * 2 + [upload] = uploaded.agents + assert upload.env_names == ( + "DATABASE_URL", + "SLACK_BOT_TOKEN", + "ZENDESK_API_TOKEN", + "OPENAI_API_KEY", + ) + + +def test_the_sdk_names() -> None: + assert len(SDK_NAMES) == 46 + assert scan_walk._SDK_SETTINGS == SDK_NAMES From a740fbb20e9758a0274562585532f544b91f8a7f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 12 Nov 2026 12:48:35 +0530 Subject: [PATCH 2/3] fix(harness): no runner-reserved name in the template list, and LiveKit's names from credentials (TH-9048) FI_API_KEY, FI_SECRET_KEY and FI_BASE_URL are the platform's own tracing account, which the runner and the connect form refuse, so a template no longer adds them: the list is 43 names. A test checks it shares no name with the runner's reserved set. The LiveKit names and its plugins' keys now come from credentials.py, so the two lists can't drift. Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_walk.py | 24 +++++++++--------- tests/harness/test_scan_walk.py | 44 ++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 184248cde..aaf2f3907 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -44,6 +44,7 @@ _Work, ) from .bundle_author_v2 import _IGNORED_ARTIFACT_PARTS as _BUILD_IGNORED +from .credentials import _CONNECTOR_REQUIREMENTS, _LIVEKIT_PROVIDER_CREDENTIALS from .reading_entry import MaskSecrets from .source_folder import _FOLDER_FORM, _FOLDER_MAX @@ -98,8 +99,8 @@ _TEMPLATES = (".env.example", ".env.sample", ".env.template") # The setting names an SDK reads for itself when the code passes no value, so the customer's # code never names them. These are the only names a template adds to a row (`_env_names`). -# The LiveKit names and the per-provider keys copy `credentials.py`'s `_CONNECTOR_REQUIREMENTS` -# and `_LIVEKIT_PROVIDER_CREDENTIALS`, which this module doesn't import. +# The LiveKit names and the per-provider keys of its plugins come from `credentials.py`, which +# says what a LiveKit agent needs. # Left out on purpose: # - `GOOGLE_APPLICATION_CREDENTIALS`: a key file, which the platform takes through its # credential-file control, not a text field; @@ -108,6 +109,8 @@ # collector; # - LiteLLM's provider names and the provider keys of LiveKit's OpenAI plugin (`with_cerebras`, # `with_x_ai`, ...), too many to list; +# - `FI_API_KEY`, `FI_SECRET_KEY` and `FI_BASE_URL`: the platform's own tracing account, which +# the runner and the connect form refuse as reserved names; # - the platform's own `ALK_*` names. _SDK_SETTINGS: frozenset[str] = frozenset( { @@ -126,18 +129,11 @@ "GOOGLE_GENAI_USE_VERTEXAI", "GOOGLE_CLOUD_PROJECT", "GOOGLE_CLOUD_LOCATION", - # LiveKit and its Deepgram, Cartesia and ElevenLabs plugins: installed, grepped. The - # plugin reads `ELEVEN_API_KEY`; `credentials.py` names `ELEVENLABS_API_KEY` (below). - "LIVEKIT_URL", - "LIVEKIT_API_KEY", - "LIVEKIT_API_SECRET", - "DEEPGRAM_API_KEY", - "CARTESIA_API_KEY", + # LiveKit's ElevenLabs plugin reads `ELEVEN_API_KEY`, where `credentials.py` names + # `ELEVENLABS_API_KEY`, the name ElevenLabs' own SDK reads (below): installed, grepped. "ELEVEN_API_KEY", - # Future AGI's tracing (`fi_instrumentation`): installed, grepped. - "FI_API_KEY", - "FI_SECRET_KEY", - "FI_BASE_URL", + # Future AGI's tracing (`fi_instrumentation`): installed, grepped. The account names are + # left out (above). "FI_PROJECT_NAME", # OpenTelemetry's SDK (`opentelemetry.sdk.environment_variables`): installed, grepped. "OTEL_SERVICE_NAME", @@ -170,6 +166,8 @@ "AWS_REGION", "AWS_DEFAULT_REGION", } + | frozenset(_CONNECTOR_REQUIREMENTS["livekit"]) + | frozenset(name for names in _LIVEKIT_PROVIDER_CREDENTIALS.values() for name in names) ) _LANGGRAPH = "langgraph.json" _SCRIPT_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 8b17aa536..db83b7669 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -827,7 +827,7 @@ def test_describe_entry_stops_at_its_deadline_with_partial( assert (capped.partial, capped.env_names) == (True, ("FIRST",)) -# The 46 SDK setting names, typed here rather than taken from `_SDK_SETTINGS`, so a change to +# The 43 SDK setting names, typed here rather than taken from `_SDK_SETTINGS`, so a change to # either fails `test_the_sdk_names`: the only names a template adds to a row. SDK_NAMES = frozenset( { @@ -850,9 +850,6 @@ def test_describe_entry_stops_at_its_deadline_with_partial( "DEEPGRAM_API_KEY", "CARTESIA_API_KEY", "ELEVEN_API_KEY", - "FI_API_KEY", - "FI_SECRET_KEY", - "FI_BASE_URL", "FI_PROJECT_NAME", "OTEL_SERVICE_NAME", "HF_TOKEN", @@ -2005,7 +2002,7 @@ def test_a_rows_setting_names_are_its_first_fifty(tmp_path: Path, names: int) -> assert row.env_names == expected assert described is not None and described.env_names == expected if names == 0: - # With no code names, a template fills at most the 46 SDK names, sorted. + # With no code names, a template fills at most the 43 SDK names, sorted. assert row.env_names == tuple(sorted(SDK_NAMES)) if names == 49: # One SDK name fills the 50th place. @@ -2609,10 +2606,45 @@ def test_a_shared_module_does_not_vouch_for_a_template_name(tmp_path: Path) -> N def test_the_sdk_names() -> None: - assert len(SDK_NAMES) == 46 + assert len(SDK_NAMES) == 43 assert scan_walk._SDK_SETTINGS == SDK_NAMES +def test_no_sdk_name_is_one_the_runner_refuses() -> None: + # The connect form shows these names to fill in; one the runner refuses as reserved would be + # a field whose value fails every run. + from fi.alk.harness.sandbox_server import _RUNNER_RESERVED_ENVIRONMENT + + assert not scan_walk._SDK_SETTINGS & _RUNNER_RESERVED_ENVIRONMENT + + +def test_the_livekit_names_are_credentials_own() -> None: + from fi.alk.harness.credentials import ( + _CONNECTOR_REQUIREMENTS, + _LIVEKIT_PROVIDER_CREDENTIALS, + ) + + livekit = set(_CONNECTOR_REQUIREMENTS["livekit"]) + livekit.update(name for names in _LIVEKIT_PROVIDER_CREDENTIALS.values() for name in names) + assert livekit <= scan_walk._SDK_SETTINGS + + +def test_a_tracing_template_lists_no_platform_account_name(tmp_path: Path) -> None: + source = tmp_path / "repo" + _write( + source, + { + "bot.py": CLIENT, + ".env.example": ( + "FI_API_KEY=\nFI_SECRET_KEY=\nFI_BASE_URL=\nFI_PROJECT_NAME=\nOPENAI_API_KEY=\n" + ), + }, + ) + + [agent] = _walk(source).agents + assert agent.env_names == ("FI_PROJECT_NAME", "OPENAI_API_KEY") + + def test_each_folder_is_charged_once_across_the_two_passes(tmp_path: Path) -> None: """The side pass walks the customer's own folders again; only the folders the first pass never entered (side folders, and folders below one) are charged there.""" From b9da87ac681b66e87dd6d333bb9711397da8a1a7 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 12 Nov 2026 16:05:14 +0530 Subject: [PATCH 3/3] test(harness): pin that the LiveKit names are imported, and that no ALK_ name is listed (TH-9048) From the fix-only check of a740fbb2: a literal copy of credentials.py's lists passed every test, and the reserved-name test didn't cover the runner's ALK_ prefix. Signed-off-by: khushalsonawat --- tests/harness/test_scan_walk.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index db83b7669..9827a5351 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -2616,6 +2616,8 @@ def test_no_sdk_name_is_one_the_runner_refuses() -> None: from fi.alk.harness.sandbox_server import _RUNNER_RESERVED_ENVIRONMENT assert not scan_walk._SDK_SETTINGS & _RUNNER_RESERVED_ENVIRONMENT + # The runner refuses every ALK_ name as well, not only the exact set. + assert not [name for name in scan_walk._SDK_SETTINGS if name.startswith("ALK_")] def test_the_livekit_names_are_credentials_own() -> None: @@ -2624,6 +2626,9 @@ def test_the_livekit_names_are_credentials_own() -> None: _LIVEKIT_PROVIDER_CREDENTIALS, ) + # Imported, not copied: a copy could drift from what credentials.py says an agent needs. + assert scan_walk._CONNECTOR_REQUIREMENTS is _CONNECTOR_REQUIREMENTS + assert scan_walk._LIVEKIT_PROVIDER_CREDENTIALS is _LIVEKIT_PROVIDER_CREDENTIALS livekit = set(_CONNECTOR_REQUIREMENTS["livekit"]) livekit.update(name for names in _LIVEKIT_PROVIDER_CREDENTIALS.values() for name in names) assert livekit <= scan_walk._SDK_SETTINGS