From 802a4ad51b25f780d4b233423bb396c7fdfecc22 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 10:22:49 +0530 Subject: [PATCH 1/2] fix(harness): answer the framework comments on the subfolder PR - Count .js, .jsx, .mjs, .cjs, .mts and .cts files as other source, so a Python folder beside them reads unknown, not none. Test and example folders still don't count. - Answer unknown on any blocker before the chat-model rule, so a chat model names LangChain only when nothing unseen could build the agent. - Scan a repository-origin process in the source, the same base the runtime builds it from; bundle-origin processes stay in the bundle. TH-8805 Signed-off-by: khushalsonawat --- src/fi/alk/harness/framework_detection.py | 21 +++- tests/harness/test_framework_detection.py | 145 +++++++++++++++++++++- tests/harness/test_hosted_entrypoint.py | 26 ++++ 3 files changed, 186 insertions(+), 6 deletions(-) diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py index bec4b49fa..c7650733c 100644 --- a/src/fi/alk/harness/framework_detection.py +++ b/src/fi/alk/harness/framework_detection.py @@ -76,7 +76,17 @@ "archive", } ) -_OTHER_SOURCE_SUFFIXES = (".ts", ".tsx", ".ipynb") +_OTHER_SOURCE_SUFFIXES = ( + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".ipynb", +) _OTHER_SOURCE_NAME = "root_agent.yaml" _VERSION_PATTERN = re.compile( @@ -911,9 +921,12 @@ def _decide(found: FolderFindings) -> tuple[str, str | None]: return "other", next(iter(labels)) if labels: return "unknown", None + # A chat model names LangChain only when nothing unseen could be building the agent. + if found.blockers: + return "unknown", None if any(f.strength == "model" for f in found.findings): return "langchain", "langchain" - if any(f.strength == "import" for f in found.findings) or found.blockers: + if any(f.strength == "import" for f in found.findings): return "unknown", None return "none", None @@ -1004,7 +1017,9 @@ def framework_folders_for( elif control_service: for process in manifest.processes: if process.name == control_service: - folder = bundle_dir / process.working_directory + # The same base the runtime builds the process from. + base = bundle_dir if process.source_origin == "bundle" else source + folder = base / process.working_directory break package_dir = ( work_directory / "build" / control_service if control_service else None diff --git a/tests/harness/test_framework_detection.py b/tests/harness/test_framework_detection.py index d93724357..bac8f703c 100644 --- a/tests/harness/test_framework_detection.py +++ b/tests/harness/test_framework_detection.py @@ -927,6 +927,12 @@ def test_a_use_of_a_long_target_allocates_nothing_of_its_size() -> None: ), (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/ui.ts": ""}, "agent", ""), (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/ui.tsx": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.mts": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.cts": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.js": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.jsx": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.mjs": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/x.cjs": ""}, "agent", ""), (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/n.ipynb": ""}, "agent", ""), ( fd._OTHER_SOURCE, @@ -950,6 +956,12 @@ def test_a_use_of_a_long_target_allocates_nothing_of_its_size() -> None: "outside_import_from_and_comma", "other_source_ts", "other_source_tsx", + "other_source_mts", + "other_source_cts", + "other_source_js", + "other_source_jsx", + "other_source_mjs", + "other_source_cjs", "other_source_ipynb", "other_source_root_agent", "no_python", @@ -964,7 +976,12 @@ def test_blockers_stop_none( assert blocker in folder_findings(tmp_path / folder, repo_root=root).blockers assert detect_framework(tmp_path / folder, repo_root=root) == UNKNOWN - # A blocker stops `none` and nothing else. + # Anything that blocks may hide what builds the agent, so a chat model names nothing. + if blocker != fd._NO_PYTHON: + _write(tmp_path, {f"{folder}/model.py": CHAT_OPENAI}) + assert detect_framework(tmp_path / folder, repo_root=root) == UNKNOWN + + # What the scan saw being built still names the framework. _write(tmp_path, {f"{folder}/crew.py": CREW}) assert detect_framework(tmp_path / folder, repo_root=root) == _named("crewai") @@ -1011,6 +1028,88 @@ def test_blockers_stop_none_no_outside_import_when_folder_is_the_root( assert detect_framework(tmp_path, repo_root=tmp_path) == NONE +AGENT_JS = ( + "import { Agent, run } from '@openai/agents';\n" + "const agent = new Agent({ name: 'x' });\n" + "await run(agent, 'hi');\n" +) +HEALTHCHECK = "import urllib.request\n" + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + ({"index.js": AGENT_JS, "healthcheck.py": HEALTHCHECK}, UNKNOWN), + ({"agent.mjs": AGENT_JS, "healthcheck.py": HEALTHCHECK}, UNKNOWN), + ({"tests/index.js": AGENT_JS, "healthcheck.py": HEALTHCHECK}, NONE), + ({"tests/x.jsx": AGENT_JS, "healthcheck.py": HEALTHCHECK}, NONE), + ({"examples/node/x.mjs": AGENT_JS, "healthcheck.py": HEALTHCHECK}, NONE), + ], + ids=["index_js", "agent_mjs", "js_in_tests", "jsx_in_tests", "mjs_in_examples"], +) +def test_a_javascript_agent_beside_a_python_helper_is_unknown( + tmp_path: Path, files: dict[str, str], expected: dict[str, Any] +) -> None: + _write(tmp_path, files) + + blocked = fd._OTHER_SOURCE in folder_findings(tmp_path).blockers + assert blocked is (expected == UNKNOWN) + assert detect_framework(tmp_path) == expected + + +SHARED_GRAPH = ( + "from langgraph.graph import StateGraph\n\n\n" + "def build():\n return StateGraph(dict).compile()\n" +) +MODEL_AND_SHARED_GRAPH = ( + "from shared.graph import build\nfrom langchain_openai import ChatOpenAI\n" + "build()\nChatOpenAI()\n" +) + + +@pytest.mark.parametrize( + ("files", "blocker", "expected"), + [ + ( + { + "svc/main.py": MODEL_AND_SHARED_GRAPH, + "shared/__init__.py": "", + "shared/graph.py": SHARED_GRAPH, + }, + fd._OUTSIDE_IMPORT, + UNKNOWN, + ), + ( + {"svc/main.py": CHAT_OPENAI, "svc/long.py": LONG_LINE}, + fd._FILE_SKIPPED, + UNKNOWN, + ), + ({"svc/main.py": CHAT_OPENAI, "svc/ui.ts": ""}, fd._OTHER_SOURCE, UNKNOWN), + ({"svc/main.py": CHAT_OPENAI}, None, _named("langchain")), + ], + ids=["outside_import", "file_skipped", "other_source", "no_blocker"], +) +def test_a_chat_model_with_a_blocker_is_unknown( + tmp_path: Path, + files: dict[str, str], + blocker: str | None, + expected: dict[str, Any], +) -> None: + _write(tmp_path, files) + + found = folder_findings(tmp_path / "svc", repo_root=tmp_path) + assert {f.strength for f in found.findings} == {"model", "import"} + assert found.blockers == ({blocker} if blocker else set()) + assert detect_framework(tmp_path / "svc", repo_root=tmp_path) == expected + + +def test_a_label_beside_a_blocker_stays_other(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": AUTOGEN, "ui.js": ""}) + + assert fd._OTHER_SOURCE in folder_findings(tmp_path).blockers + assert detect_framework(tmp_path) == _named("other", "autogen") + + SIDE_WORDS = [ "tests", "test", @@ -1350,12 +1449,16 @@ def test_metadata_read_stops_at_64_kib(tmp_path: Path) -> None: assert detect_framework(folder, package_dir=package_dir) == _named("crewai") -def _process_manifest() -> SimpleNamespace: +def _process_manifest(source_origin: str = "bundle") -> SimpleNamespace: return SimpleNamespace( runtime=SimpleNamespace(kind="process", control_service="agent"), processes=[ SimpleNamespace(name="postgres", working_directory=None), - SimpleNamespace(name="agent", working_directory="generated/source/agent"), + SimpleNamespace( + name="agent", + working_directory="generated/source/agent", + source_origin=source_origin, + ), ], ) @@ -1405,6 +1508,12 @@ def _compose_manifest() -> SimpleNamespace: (None, None, _process_manifest(), "bundle/generated/source/agent"), (None, "{not json", _process_manifest(), "bundle/generated/source/agent"), (None, ["a", "list"], _process_manifest(), "bundle/generated/source/agent"), + ( + None, + None, + _process_manifest("repository"), + "source/generated/source/agent", + ), (None, None, _compose_manifest(), "source"), ], ids=[ @@ -1419,6 +1528,7 @@ def _compose_manifest() -> SimpleNamespace: "missing_contract", "corrupt_contract", "contract_not_an_object", + "repository_origin_process", "compose_falls_to_source", ], ) @@ -1460,6 +1570,35 @@ def test_folder_order( ) +def test_a_repository_process_is_scanned_in_the_source(tmp_path: Path) -> None: + _write( + tmp_path, + {"source/main.py": LANGGRAPH, "bundle/scenarios/s1/check.py": PLAIN}, + ) + manifest = SimpleNamespace( + runtime=SimpleNamespace(kind="process", control_service="agent"), + processes=[ + SimpleNamespace( + name="agent", working_directory=".", source_origin="repository" + ) + ], + ) + job = SimpleNamespace(source=SimpleNamespace(subfolder=None)) + + folders = framework_folders_for( + job, + manifest, + source=tmp_path / "source", + bundle_dir=tmp_path / "bundle", + work_directory=tmp_path / "work", + ) + + assert folders.folder == tmp_path / "source" / "." + assert detect_framework( + folders.folder, repo_root=folders.repo_root, package_dir=folders.package_dir + ) == _named("langgraph") + + def _github_job(subfolder: str | None) -> SimpleNamespace: return SimpleNamespace( source=RepositorySource( diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 76140cc39..fade99631 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -5093,12 +5093,14 @@ def test_bad_subfolder_fails_terminal_before_compose(monkeypatch): def _harness_with_a_packaged_agent( packaged_main: bytes = b"from crewai import Crew\nCrew()\n", + source_origin: str = "bundle", ) -> Harness: real_body = _base_manifest_body def packaged_body() -> dict[str, Any]: body = real_body() body["processes"][1]["working_directory"] = "generated/source/agent" + body["processes"][1]["source_origin"] = source_origin return body files = { @@ -5178,6 +5180,30 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_run_job_gives_repository_source_its_folders(caplog) -> None: + async def scenario() -> None: + # The crew only in `source`: scanning the bundle's copy would find none. + harness = _harness_with_a_packaged_agent( + packaged_main=b"print('plain')\n", source_origin="repository" + ) + agent = harness.source / "generated" / "source" / "agent" + agent.mkdir(parents=True) + (agent / "main.py").write_text( + "from crewai import Crew\nCrew()\n", encoding="utf-8" + ) + _install_crewai(harness.work / "build" / "agent") + + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + + assert code == he.EXIT_OK + assert _posted_framework(harness) == {"name": "crewai", "version": "0.86.0"} + + with caplog.at_level("INFO", logger="fi.alk.harness"): + asyncio.run(scenario()) + + def test_folder_step_failure_keeps_the_run_going(caplog) -> None: secret = "/customer/private/path" From 634565c2230b5c8baa898894f20670cfddd900c7 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 11:51:33 +0530 Subject: [PATCH 2/2] test(harness): drop the framework tests that repeat other cases - The chat-model-with-a-blocker test repeated test_blockers_stop_none, which puts a chat model beside every blocker; its check that the chat model is read as a model moves there. - The JavaScript test keeps only the test and example folder rows; the .js and .mjs rows repeat BLOCKER_CASES. - The standalone repository-process test goes; the folder-order row and the end-to-end hosted test cover the rule. TH-8805 Signed-off-by: khushalsonawat --- tests/harness/test_framework_detection.py | 98 +++-------------------- 1 file changed, 11 insertions(+), 87 deletions(-) diff --git a/tests/harness/test_framework_detection.py b/tests/harness/test_framework_detection.py index bac8f703c..2b61ddf41 100644 --- a/tests/harness/test_framework_detection.py +++ b/tests/harness/test_framework_detection.py @@ -979,6 +979,8 @@ def test_blockers_stop_none( # Anything that blocks may hide what builds the agent, so a chat model names nothing. if blocker != fd._NO_PYTHON: _write(tmp_path, {f"{folder}/model.py": CHAT_OPENAI}) + found = folder_findings(tmp_path / folder, repo_root=root) + assert any(f.strength == "model" for f in found.findings) assert detect_framework(tmp_path / folder, repo_root=root) == UNKNOWN # What the scan saw being built still names the framework. @@ -1037,70 +1039,21 @@ def test_blockers_stop_none_no_outside_import_when_folder_is_the_root( @pytest.mark.parametrize( - ("files", "expected"), - [ - ({"index.js": AGENT_JS, "healthcheck.py": HEALTHCHECK}, UNKNOWN), - ({"agent.mjs": AGENT_JS, "healthcheck.py": HEALTHCHECK}, UNKNOWN), - ({"tests/index.js": AGENT_JS, "healthcheck.py": HEALTHCHECK}, NONE), - ({"tests/x.jsx": AGENT_JS, "healthcheck.py": HEALTHCHECK}, NONE), - ({"examples/node/x.mjs": AGENT_JS, "healthcheck.py": HEALTHCHECK}, NONE), - ], - ids=["index_js", "agent_mjs", "js_in_tests", "jsx_in_tests", "mjs_in_examples"], -) -def test_a_javascript_agent_beside_a_python_helper_is_unknown( - tmp_path: Path, files: dict[str, str], expected: dict[str, Any] -) -> None: - _write(tmp_path, files) - - blocked = fd._OTHER_SOURCE in folder_findings(tmp_path).blockers - assert blocked is (expected == UNKNOWN) - assert detect_framework(tmp_path) == expected - - -SHARED_GRAPH = ( - "from langgraph.graph import StateGraph\n\n\n" - "def build():\n return StateGraph(dict).compile()\n" -) -MODEL_AND_SHARED_GRAPH = ( - "from shared.graph import build\nfrom langchain_openai import ChatOpenAI\n" - "build()\nChatOpenAI()\n" -) - - -@pytest.mark.parametrize( - ("files", "blocker", "expected"), + "files", [ - ( - { - "svc/main.py": MODEL_AND_SHARED_GRAPH, - "shared/__init__.py": "", - "shared/graph.py": SHARED_GRAPH, - }, - fd._OUTSIDE_IMPORT, - UNKNOWN, - ), - ( - {"svc/main.py": CHAT_OPENAI, "svc/long.py": LONG_LINE}, - fd._FILE_SKIPPED, - UNKNOWN, - ), - ({"svc/main.py": CHAT_OPENAI, "svc/ui.ts": ""}, fd._OTHER_SOURCE, UNKNOWN), - ({"svc/main.py": CHAT_OPENAI}, None, _named("langchain")), + {"tests/index.js": AGENT_JS, "healthcheck.py": HEALTHCHECK}, + {"tests/x.jsx": AGENT_JS, "healthcheck.py": HEALTHCHECK}, + {"examples/node/x.mjs": AGENT_JS, "healthcheck.py": HEALTHCHECK}, ], - ids=["outside_import", "file_skipped", "other_source", "no_blocker"], + ids=["js_in_tests", "jsx_in_tests", "mjs_in_examples"], ) -def test_a_chat_model_with_a_blocker_is_unknown( - tmp_path: Path, - files: dict[str, str], - blocker: str | None, - expected: dict[str, Any], +def test_javascript_in_test_and_example_folders_does_not_block( + tmp_path: Path, files: dict[str, str] ) -> None: _write(tmp_path, files) - found = folder_findings(tmp_path / "svc", repo_root=tmp_path) - assert {f.strength for f in found.findings} == {"model", "import"} - assert found.blockers == ({blocker} if blocker else set()) - assert detect_framework(tmp_path / "svc", repo_root=tmp_path) == expected + assert fd._OTHER_SOURCE not in folder_findings(tmp_path).blockers + assert detect_framework(tmp_path) == NONE def test_a_label_beside_a_blocker_stays_other(tmp_path: Path) -> None: @@ -1570,35 +1523,6 @@ def test_folder_order( ) -def test_a_repository_process_is_scanned_in_the_source(tmp_path: Path) -> None: - _write( - tmp_path, - {"source/main.py": LANGGRAPH, "bundle/scenarios/s1/check.py": PLAIN}, - ) - manifest = SimpleNamespace( - runtime=SimpleNamespace(kind="process", control_service="agent"), - processes=[ - SimpleNamespace( - name="agent", working_directory=".", source_origin="repository" - ) - ], - ) - job = SimpleNamespace(source=SimpleNamespace(subfolder=None)) - - folders = framework_folders_for( - job, - manifest, - source=tmp_path / "source", - bundle_dir=tmp_path / "bundle", - work_directory=tmp_path / "work", - ) - - assert folders.folder == tmp_path / "source" / "." - assert detect_framework( - folders.folder, repo_root=folders.repo_root, package_dir=folders.package_dir - ) == _named("langgraph") - - def _github_job(subfolder: str | None) -> SimpleNamespace: return SimpleNamespace( source=RepositorySource(