Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions tests/unittests/cli/utils/test_cli_tools_click.py
Original file line number Diff line number Diff line change
Expand Up @@ -2857,6 +2857,111 @@ def test_cli_conformance_test_forwards_mode_and_report_options(
}]


def test_cli_conformance_test_accepts_multiple_directories(
tmp_path: Path, fake_conformance_test
) -> None:
"""Every PATHS argument is forwarded to the conformance runner."""
first_dir = tmp_path / "cases_one"
second_dir = tmp_path / "cases_two"
first_dir.mkdir()
second_dir.mkdir()

result = CliRunner().invoke(
cli_tools_click.main,
["conformance", "test", str(first_dir), str(second_dir)],
)

assert result.exit_code == 0, (result.output, repr(result.exception))
assert fake_conformance_test == [{
"test_paths": [
Path(os.path.realpath(first_dir)),
Path(os.path.realpath(second_dir)),
],
"mode": "replay",
"generate_report": False,
"report_dir": None,
"streaming_mode": None,
}]


def test_cli_conformance_test_forwards_live_mode(
tmp_path: Path, fake_conformance_test
) -> None:
"""--mode live is propagated to the conformance runner."""
case_dir = tmp_path / "cases"
case_dir.mkdir()

result = CliRunner().invoke(
cli_tools_click.main,
["conformance", "test", str(case_dir), "--mode", "live"],
)

assert result.exit_code == 0, (result.output, repr(result.exception))
assert fake_conformance_test == [{
"test_paths": [Path(os.path.realpath(case_dir))],
"mode": "live",
"generate_report": False,
"report_dir": None,
"streaming_mode": None,
}]


def test_cli_conformance_test_forwards_bidi_streaming_mode(
tmp_path: Path, fake_conformance_test
) -> None:
"""--streaming-mode bidi is parsed to the StreamingMode enum."""
case_dir = tmp_path / "cases"
case_dir.mkdir()

result = CliRunner().invoke(
cli_tools_click.main,
["conformance", "test", str(case_dir), "--streaming-mode", "bidi"],
)

assert result.exit_code == 0, (result.output, repr(result.exception))
assert fake_conformance_test == [{
"test_paths": [Path(os.path.realpath(case_dir))],
"mode": "replay",
"generate_report": False,
"report_dir": None,
"streaming_mode": StreamingMode.BIDI,
}]


def test_cli_conformance_test_rejects_invalid_mode(
tmp_path: Path,
) -> None:
"""An unknown --mode value is rejected by Click before any dispatch."""
case_dir = tmp_path / "cases"
case_dir.mkdir()

result = CliRunner().invoke(
cli_tools_click.main,
["conformance", "test", str(case_dir), "--mode", "fast"],
)

assert result.exit_code == 2
assert "Invalid value for '--mode'" in result.output
assert "'fast' is not one of 'replay', 'live'" in result.output


def test_cli_conformance_test_rejects_invalid_streaming_mode(
tmp_path: Path,
) -> None:
"""An unknown --streaming-mode value is rejected by Click before dispatch."""
case_dir = tmp_path / "cases"
case_dir.mkdir()

result = CliRunner().invoke(
cli_tools_click.main,
["conformance", "test", str(case_dir), "--streaming-mode", "eventstream"],
)

assert result.exit_code == 2
assert "Invalid value for '--streaming-mode'" in result.output
assert "'eventstream' is not one of 'none', 'sse', 'bidi'" in result.output


# adk eval_set create
def test_cli_create_eval_set_surfaces_duplicate_id_as_click_exception(
tmp_path: Path,
Expand Down