Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/python-tests-known-broken.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Python packages whose tests are known to fail today (see .github/workflows/python-tests.yml).
# A listed package still runs in CI and shows a warning, but its failures do not fail the required "Python tests" check.
# A listed package still runs in CI and shows a warning, but its test failures do not fail the required "Python tests" check.
# A failed install fails the check, listed or not.
# If a listed package's tests all pass, CI fails until its line is removed from this file.
# So a PR that makes a listed package pass must remove its line in the same PR, or CI fails
# (for example, a LangChain change that fixes its tests). A listed package with no tests at all also fails: remove its line.
Expand Down
19 changes: 11 additions & 8 deletions .github/scripts/known_broken_verdict.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,27 @@ case "$TESTS" in
exit 1
;;
esac
# The Tests step is skipped only when an earlier step (checkout or setup) failed outright, or
# A failed install fails the check, listed or not: the package's tests never ran against a working
# install, so neither a pass nor a failure of them says anything about the listed reason.
# The install steps have no continue-on-error, so a failed install skips the Tests step; this
# check comes before the "skipped" one so the error names the install, whatever TESTS says.
if [ -n "$install_failed" ]; then
echo "::error::$PKG_NAME: $install_failed failed; a failed install fails the check, listed or not"
exit 1
fi
# The Tests step is skipped when checkout or setup failed outright (a failed install is handled above), or
# the job was cancelled: that is not a pass, and the job fails with it.
if [ "$TESTS" = "skipped" ]; then
echo "::error::$PKG_NAME is known broken ($REASON), but its tests did not run (an earlier step failed, or the job was cancelled)"
exit 1
fi
# The tests passed (whatever an install step did): the line has to come off the list.
# The tests passed: the line has to come off the list.
if [ "$TESTS" = "success" ]; then
if [ "$OWN_TESTS" != "true" ] && [ -z "$CORE_TESTS" ]; then
echo "::error::$PKG_NAME is on the known-broken list but has no tests; remove its line from .github/python-tests-known-broken.txt"
else
echo "::error::$PKG_NAME is on the known-broken list but its tests pass; remove its line from .github/python-tests-known-broken.txt${install_failed:+ (failed in this run: $install_failed)}"
echo "::error::$PKG_NAME is on the known-broken list but its tests pass; remove its line from .github/python-tests-known-broken.txt"
fi
exit 1
fi
# The tests failed. After a failed install that may not be the listed reason: say so.
if [ -n "$install_failed" ]; then
echo "::warning::$PKG_NAME is known broken, but in this run these steps failed: $install_failed; its test results may not reflect the listed reason ($REASON)"
exit 0
fi
echo "::warning::$PKG_NAME is known broken ($REASON): its failures do not fail the check"
3 changes: 2 additions & 1 deletion .github/scripts/python_test_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@
``known_broken`` and ``known_broken_reason`` (from ``.github/python-tests-known-broken.txt``:
one ``<package dir name> | <reason>`` line per package; ``#`` comments and blank lines are
ignored; the reason is "" for a package that is not listed). The workflow lets a listed
package fail without failing the check, and fails when a listed package passes.
package's tests fail without failing the check, fails when a listed package passes, and fails
on any failed install, listed or not.

Paths from git are used exactly as written (stdin lines are stripped). Standard library only,
plus git for --diff and --all. It fails loudly (non-zero exit, message on stderr) on a
Expand Down
205 changes: 199 additions & 6 deletions .github/scripts/test_python_test_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,19 @@ class PlanTable(FixtureCase):
("a package's own test", ["python/frameworks/alpha/tests/test_x.py"], ["alpha"], False, False),
("the traceai package", ["python/traceai/traceai/__init__.py"], ["traceai"], False, False),
("a core file", ["python/fi_instrumentation/otel.py"], ALL_PACKAGES, False, True),
("a nested core file", ["python/fi_instrumentation/instrumentation/context_attributes.py"], ALL_PACKAGES, False, True),
(
"CI files and a core file together (core_install runs in every package job)",
[
".github/workflows/python-tests.yml",
".github/scripts/known_broken_verdict.sh",
".github/scripts/test_python_test_plan.py",
"python/fi_instrumentation/instrumentation/context_attributes.py",
],
ALL_PACKAGES,
True,
True,
),
("poetry.lock", ["python/poetry.lock"], ALL_PACKAGES, False, True),
("python/pyproject.toml", ["python/pyproject.toml"], ALL_PACKAGES, False, True),
("conftest.py", ["python/tests/conftest.py"], ["gamma"], False, False),
Expand Down Expand Up @@ -367,6 +380,9 @@ def test_a_diff_with_a_change_plans_it(self):
VERDICT = SCRIPTS_DIR / "known_broken_verdict.sh"


INSTALL_FAILS_THE_CHECK = "a failed install fails the check, listed or not"


class KnownBrokenVerdict(unittest.TestCase):
BASE = {
"PKG_NAME": "pkg",
Expand All @@ -385,20 +401,44 @@ class KnownBrokenVerdict(unittest.TestCase):
"tests pass after a failed install",
{"TESTS": "success", "INSTALL_POETRY": "failure"},
1,
"failed in this run: install (poetry)",
f"pkg: install (poetry) failed; {INSTALL_FAILS_THE_CHECK}",
),
("tests fail", {"TESTS": "failure"}, 0, "known broken (the listed reason)"),
(
"tests fail after failed poetry and core installs",
{"TESTS": "failure", "INSTALL_POETRY": "failure", "CORE_INSTALL": "failure"},
0,
"install (poetry), fi_instrumentation install",
1,
f"pkg: install (poetry), fi_instrumentation install failed; {INSTALL_FAILS_THE_CHECK}",
),
(
"tests fail after a failed hatch install",
{"TESTS": "failure", "INSTALL_POETRY": "skipped", "INSTALL_HATCH": "failure"},
0,
"install (hatch)",
1,
f"pkg: install (hatch) failed; {INSTALL_FAILS_THE_CHECK}",
),
(
"tests fail after a failed core install",
{"TESTS": "failure", "CORE_INSTALL": "failure"},
1,
f"pkg: fi_instrumentation install failed; {INSTALL_FAILS_THE_CHECK}",
),
(
"tests skipped after a failed install (the real case: a failed install skips the Tests step)",
{"TESTS": "skipped", "INSTALL_POETRY": "failure"},
1,
f"pkg: install (poetry) failed; {INSTALL_FAILS_THE_CHECK}",
),
(
"cancelled after a failed install",
{"TESTS": "cancelled", "INSTALL_HATCH": "failure"},
1,
f"pkg: install (hatch) failed; {INSTALL_FAILS_THE_CHECK}",
),
(
"no tests, the tests pass, and a failed install",
{"TESTS": "success", "OWN_TESTS": "false", "CORE_TESTS": "", "CORE_INSTALL": "failure"},
1,
INSTALL_FAILS_THE_CHECK,
),
("tests skipped", {"TESTS": "skipped"}, 1, "did not run"),
(
Expand All @@ -414,6 +454,7 @@ class KnownBrokenVerdict(unittest.TestCase):
"its tests pass",
),
("cancelled", {"TESTS": "cancelled"}, 0, "known broken"),
("tests fail, no install failed (skipped installs)", {"TESTS": "failure", "INSTALL_POETRY": "skipped", "CORE_INSTALL": "skipped"}, 0, "known broken (the listed reason)"),
("no Tests outcome", {"TESTS": ""}, 1, "unexpected Tests outcome"),
("a misspelled outcome", {"TESTS": "succes"}, 1, "unexpected Tests outcome"),
]
Expand All @@ -436,6 +477,9 @@ def test_cases(self):
output = done.stdout + done.stderr
self.assertEqual(done.returncode, code, output)
self.assertIn(phrase, output)
# The old "may not reflect the listed reason" warning and "(failed in this run: ...)" note are gone.
self.assertNotIn("may not reflect", output)
self.assertNotIn("failed in this run", output)

def test_the_syntax_is_valid(self):
subprocess.run(["bash", "-n", str(VERDICT)], check=True)
Expand Down Expand Up @@ -581,6 +625,155 @@ def test_the_planner_tests_step_cannot_be_skipped_or_ignored(self):
for line in step:
self.assertFalse(line.startswith(("continue-on-error", "if:")), line)

def test_every_run_step_uses_bash_with_pipefail(self):
# `shell: bash` under the workflow's `defaults: run:` makes GitHub run every run step as
# `bash --noprofile --norc -eo pipefail {0}` (a bare run step gets `bash -e {0}`, with no pipefail,
# so `cmd | tee log` would hide a failed cmd). The block is exactly that, and the file has
# exactly one `shell:` anywhere (flow style and comments included), so no job or step overrides it.
lines = self.text.splitlines()
self.assertEqual(lines.count("defaults:"), 1)
start = lines.index("defaults:")
self.assertEqual(lines[start : start + 3], ["defaults:", " run:", " shell: bash"])
self.assertFalse(lines[start + 3].startswith(" "), "another key under defaults")
self.assertEqual(len(re.findall(r"\bshell[\"']?\s*:", self.text)), 1)

INSTALL_STEPS = (
# (step id, step name, its if:, the install command that must stand alone on a line)
(
"install_poetry",
"Install the package and its dev dependencies (poetry)",
"if: matrix.package.backend == 'poetry'",
"poetry install --no-interaction",
),
(
"install_hatch",
"Install the package and its dev dependencies (hatch)",
"if: matrix.package.backend == 'hatch'",
'.venv/bin/pip install -e ".[dev]"',
),
(
"core_install",
"Test against this repo's fi_instrumentation",
"if: needs.plan.outputs.core_changed == 'true'",
'"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"',
),
)

# The commands of each install step, exactly as reviewed (comments and indentation aside). A line
# filter cannot see a multi-line wrapper that exempts the install from errexit (`{ ...; } && ...`,
# a function called from `if`, `trap ... ERR`, `bash -c '...'`), so the whole script is pinned:
# changing an install step means changing this table, in the same diff.
INSTALL_SCRIPTS = {
"install_poetry": [
'pip install "poetry==$POETRY_VERSION"',
"poetry install --no-interaction",
"for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do",
'module=$(echo "${pin%%==*}" | tr - _)',
'if poetry run python -c "import $module" 2>/dev/null; then',
'echo "$module: already present"',
"else",
'echo "$module: added $pin"',
'poetry run pip install "$pin"',
"fi",
"done",
],
"install_hatch": [
"python -m venv .venv",
'.venv/bin/pip install -e ".[dev]"',
"for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do",
'module=$(echo "${pin%%==*}" | tr - _)',
'if .venv/bin/python -c "import $module" 2>/dev/null; then',
'echo "$module: already present"',
"else",
'echo "$module: added $pin"',
'.venv/bin/pip install "$pin"',
"fi",
"done",
],
"core_install": [
'"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"',
],
}

def test_the_install_scripts_are_exactly_as_reviewed(self):
for step_id, name, _condition, _command in self.INSTALL_STEPS:
with self.subTest(step_id):
step = self.step(name, containing=f"id: {step_id}")
commands = [line for line in self.run_lines(step) if line and not line.startswith("#")]
self.assertEqual(commands, self.INSTALL_SCRIPTS[step_id])

@staticmethod
def run_lines(step):
"""The lines of the step's `run: |` block (the run block is the last key of the install steps)."""
index = step.index("run: |")
return step[index + 1 :]

def test_a_failed_install_fails_its_step_for_a_listed_package_too(self):
# With continue-on-error on an install step, a failed install of a listed package would not
# fail the job, and the Tests step would run on a broken install. So, for each install step:
# - no continue-on-error, no `shell:` and exactly the pinned `if:`;
# - over its run lines (comments included): no `||`, no `|` outside a `$( ... )` span (so no
# `cmd | tee log`; a `$(...)` is stripped before looking, nested ones are not handled), and
# no `set +` (so no `set +e` and no `set +o errexit`);
# - the install command is a line of its own, exactly: an `if ! ...; then` wrapper, a trailing
# comment or a pipe on that line is not the exact line, so it fails.
for step_id, name, condition, command in self.INSTALL_STEPS:
with self.subTest(step_id):
step = self.step(name, containing=f"id: {step_id}")
self.assertEqual([line for line in step if line.startswith("continue-on-error")], [], step)
self.assertEqual([line for line in step if re.search(r"\bshell[\"']?\s*:", line)], [], step)
self.assertEqual([line for line in step if line.startswith("if:")], [condition], step)
run = self.run_lines(step)
self.assertIn(command, run)
self.assertEqual(run.count(command), 1, run)
self.assertEqual([line for line in run if "||" in line], [], run)
outside = [re.sub(r"\$\([^)]*\)", "", line) for line in run]
self.assertEqual([line for line in outside if "|" in line], [], run)
self.assertEqual([line for line in run if "set +" in line], [], run)
# Only the Tests step may be allowed to fail (for a listed package).
tests = self.step("Tests", containing="id: tests")
self.assertEqual(tests.count("continue-on-error: ${{ matrix.package.known_broken }}"), 1)
self.assertEqual(sum(line.startswith("continue-on-error") for line in self.lines()), 1)

def test_the_tests_step_is_skipped_after_a_failed_install(self):
# The Tests step (id: tests) has no `if:`, so GitHub gives it the default success(): a failed
# install step skips it, and a broken install never gets its tests run.
tests = self.step("Tests", containing="id: tests")
self.assertEqual([line for line in tests if line.startswith("if:")], [], tests)

def test_every_job_has_a_time_limit(self):
# Every job under `jobs:` has exactly one job-level `timeout-minutes: N` (4 spaces, N a positive
# integer; a step's own timeout does not bound the job) and an entry in this table, so a new job
# must get both.
limits = {"plan": 10, "fi-instrumentation": 30, "package": 45, "result": 5}
lines = self.text.splitlines()

def is_comment(line):
return line.lstrip().startswith("#")

# A comment line (at any indent) neither ends the jobs block nor a job, and a job key may carry
# a trailing comment.
start = lines.index("jobs:")
end = next((i for i in range(start + 1, len(lines)) if re.match(r"^\S", lines[i]) and not is_comment(lines[i])), len(lines))
found_keys = {}
for i in range(start + 1, end):
match = re.match(r"^ ([A-Za-z0-9_-]+):\s*(#.*)?$", lines[i])
if match:
found_keys[match.group(1)] = i
keys = list(found_keys)
self.assertEqual(sorted(keys), sorted(limits), "a job without an entry in the table, or an entry without a job")
for key in keys:
with self.subTest(key):
first = found_keys[key]
last = next(
(i for i in range(first + 1, len(lines)) if re.match(r"^ \S|^\S", lines[i]) and not is_comment(lines[i])),
len(lines),
)
found = [line for line in lines[first:last] if re.match(r"^ timeout-minutes:", line)]
self.assertEqual(len(found), 1, found)
self.assertRegex(found[0], r"^ timeout-minutes: [1-9][0-9]*$")
self.assertEqual(found, [f" timeout-minutes: {limits[key]}"])

def test_the_verdict_step_runs_always_for_known_broken_packages(self):
step = self.step("Known-broken verdict")
self.assertIn("if: always() && matrix.package.known_broken", step)
Expand All @@ -598,7 +791,7 @@ def test_core_tests_reach_the_tests_and_verdict_steps(self):
def test_a_pr_base_missing_from_the_clone_is_fetched_then_everything_runs(self):
text = self.text
check = 'if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then'
fetch = 'git fetch --no-tags origin "$PR_BASE_SHA" || true'
fetch = 'timeout 120 git fetch --no-tags origin "$PR_BASE_SHA" || true'
message = 'echo "PR base $PR_BASE_SHA is not in the clone: treating every file as changed" >&2'
diff = 'python3 .github/scripts/python_test_plan.py --diff "$PR_BASE_SHA...$PR_HEAD_SHA"'
lines = self.lines()
Expand Down
Loading
Loading