diff --git a/.github/python-tests-known-broken.txt b/.github/python-tests-known-broken.txt index af9679267..72a19f835 100644 --- a/.github/python-tests-known-broken.txt +++ b/.github/python-tests-known-broken.txt @@ -1,5 +1,6 @@ # Python packages whose tests are known to fail today (see .github/workflows/python-tests.yml). -# A listed package still runs in CI and shows a warning, but its failures do not fail the required "Python tests" check. +# A listed package still runs in CI and shows a warning, but its test failures do not fail the required "Python tests" check. +# A failed install fails the check, listed or not. # If a listed package's tests all pass, CI fails until its line is removed from this file. # So a PR that makes a listed package pass must remove its line in the same PR, or CI fails # (for example, a LangChain change that fixes its tests). A listed package with no tests at all also fails: remove its line. diff --git a/.github/scripts/known_broken_verdict.sh b/.github/scripts/known_broken_verdict.sh index 7225e1f1e..13ed64c3d 100755 --- a/.github/scripts/known_broken_verdict.sh +++ b/.github/scripts/known_broken_verdict.sh @@ -16,24 +16,27 @@ case "$TESTS" in exit 1 ;; esac -# The Tests step is skipped only when an earlier step (checkout or setup) failed outright, or +# A failed install fails the check, listed or not: the package's tests never ran against a working +# install, so neither a pass nor a failure of them says anything about the listed reason. +# The install steps have no continue-on-error, so a failed install skips the Tests step; this +# check comes before the "skipped" one so the error names the install, whatever TESTS says. +if [ -n "$install_failed" ]; then + echo "::error::$PKG_NAME: $install_failed failed; a failed install fails the check, listed or not" + exit 1 +fi +# The Tests step is skipped when checkout or setup failed outright (a failed install is handled above), or # the job was cancelled: that is not a pass, and the job fails with it. if [ "$TESTS" = "skipped" ]; then echo "::error::$PKG_NAME is known broken ($REASON), but its tests did not run (an earlier step failed, or the job was cancelled)" exit 1 fi -# The tests passed (whatever an install step did): the line has to come off the list. +# The tests passed: the line has to come off the list. if [ "$TESTS" = "success" ]; then if [ "$OWN_TESTS" != "true" ] && [ -z "$CORE_TESTS" ]; then echo "::error::$PKG_NAME is on the known-broken list but has no tests; remove its line from .github/python-tests-known-broken.txt" else - echo "::error::$PKG_NAME is on the known-broken list but its tests pass; remove its line from .github/python-tests-known-broken.txt${install_failed:+ (failed in this run: $install_failed)}" + echo "::error::$PKG_NAME is on the known-broken list but its tests pass; remove its line from .github/python-tests-known-broken.txt" fi exit 1 fi -# The tests failed. After a failed install that may not be the listed reason: say so. -if [ -n "$install_failed" ]; then - echo "::warning::$PKG_NAME is known broken, but in this run these steps failed: $install_failed; its test results may not reflect the listed reason ($REASON)" - exit 0 -fi echo "::warning::$PKG_NAME is known broken ($REASON): its failures do not fail the check" diff --git a/.github/scripts/python_test_plan.py b/.github/scripts/python_test_plan.py index 3c3d1c692..4362c2aa4 100755 --- a/.github/scripts/python_test_plan.py +++ b/.github/scripts/python_test_plan.py @@ -36,7 +36,8 @@ ``known_broken`` and ``known_broken_reason`` (from ``.github/python-tests-known-broken.txt``: one `` | `` line per package; ``#`` comments and blank lines are ignored; the reason is "" for a package that is not listed). The workflow lets a listed -package fail without failing the check, and fails when a listed package passes. +package's tests fail without failing the check, fails when a listed package passes, and fails +on any failed install, listed or not. Paths from git are used exactly as written (stdin lines are stripped). Standard library only, plus git for --diff and --all. It fails loudly (non-zero exit, message on stderr) on a diff --git a/.github/scripts/test_python_test_plan.py b/.github/scripts/test_python_test_plan.py index 8d5ca741e..0f52ca91a 100644 --- a/.github/scripts/test_python_test_plan.py +++ b/.github/scripts/test_python_test_plan.py @@ -75,6 +75,19 @@ class PlanTable(FixtureCase): ("a package's own test", ["python/frameworks/alpha/tests/test_x.py"], ["alpha"], False, False), ("the traceai package", ["python/traceai/traceai/__init__.py"], ["traceai"], False, False), ("a core file", ["python/fi_instrumentation/otel.py"], ALL_PACKAGES, False, True), + ("a nested core file", ["python/fi_instrumentation/instrumentation/context_attributes.py"], ALL_PACKAGES, False, True), + ( + "CI files and a core file together (core_install runs in every package job)", + [ + ".github/workflows/python-tests.yml", + ".github/scripts/known_broken_verdict.sh", + ".github/scripts/test_python_test_plan.py", + "python/fi_instrumentation/instrumentation/context_attributes.py", + ], + ALL_PACKAGES, + True, + True, + ), ("poetry.lock", ["python/poetry.lock"], ALL_PACKAGES, False, True), ("python/pyproject.toml", ["python/pyproject.toml"], ALL_PACKAGES, False, True), ("conftest.py", ["python/tests/conftest.py"], ["gamma"], False, False), @@ -367,6 +380,9 @@ def test_a_diff_with_a_change_plans_it(self): VERDICT = SCRIPTS_DIR / "known_broken_verdict.sh" +INSTALL_FAILS_THE_CHECK = "a failed install fails the check, listed or not" + + class KnownBrokenVerdict(unittest.TestCase): BASE = { "PKG_NAME": "pkg", @@ -385,20 +401,44 @@ class KnownBrokenVerdict(unittest.TestCase): "tests pass after a failed install", {"TESTS": "success", "INSTALL_POETRY": "failure"}, 1, - "failed in this run: install (poetry)", + f"pkg: install (poetry) failed; {INSTALL_FAILS_THE_CHECK}", ), ("tests fail", {"TESTS": "failure"}, 0, "known broken (the listed reason)"), ( "tests fail after failed poetry and core installs", {"TESTS": "failure", "INSTALL_POETRY": "failure", "CORE_INSTALL": "failure"}, - 0, - "install (poetry), fi_instrumentation install", + 1, + f"pkg: install (poetry), fi_instrumentation install failed; {INSTALL_FAILS_THE_CHECK}", ), ( "tests fail after a failed hatch install", {"TESTS": "failure", "INSTALL_POETRY": "skipped", "INSTALL_HATCH": "failure"}, - 0, - "install (hatch)", + 1, + f"pkg: install (hatch) failed; {INSTALL_FAILS_THE_CHECK}", + ), + ( + "tests fail after a failed core install", + {"TESTS": "failure", "CORE_INSTALL": "failure"}, + 1, + f"pkg: fi_instrumentation install failed; {INSTALL_FAILS_THE_CHECK}", + ), + ( + "tests skipped after a failed install (the real case: a failed install skips the Tests step)", + {"TESTS": "skipped", "INSTALL_POETRY": "failure"}, + 1, + f"pkg: install (poetry) failed; {INSTALL_FAILS_THE_CHECK}", + ), + ( + "cancelled after a failed install", + {"TESTS": "cancelled", "INSTALL_HATCH": "failure"}, + 1, + f"pkg: install (hatch) failed; {INSTALL_FAILS_THE_CHECK}", + ), + ( + "no tests, the tests pass, and a failed install", + {"TESTS": "success", "OWN_TESTS": "false", "CORE_TESTS": "", "CORE_INSTALL": "failure"}, + 1, + INSTALL_FAILS_THE_CHECK, ), ("tests skipped", {"TESTS": "skipped"}, 1, "did not run"), ( @@ -414,6 +454,7 @@ class KnownBrokenVerdict(unittest.TestCase): "its tests pass", ), ("cancelled", {"TESTS": "cancelled"}, 0, "known broken"), + ("tests fail, no install failed (skipped installs)", {"TESTS": "failure", "INSTALL_POETRY": "skipped", "CORE_INSTALL": "skipped"}, 0, "known broken (the listed reason)"), ("no Tests outcome", {"TESTS": ""}, 1, "unexpected Tests outcome"), ("a misspelled outcome", {"TESTS": "succes"}, 1, "unexpected Tests outcome"), ] @@ -436,6 +477,9 @@ def test_cases(self): output = done.stdout + done.stderr self.assertEqual(done.returncode, code, output) self.assertIn(phrase, output) + # The old "may not reflect the listed reason" warning and "(failed in this run: ...)" note are gone. + self.assertNotIn("may not reflect", output) + self.assertNotIn("failed in this run", output) def test_the_syntax_is_valid(self): subprocess.run(["bash", "-n", str(VERDICT)], check=True) @@ -581,6 +625,155 @@ def test_the_planner_tests_step_cannot_be_skipped_or_ignored(self): for line in step: self.assertFalse(line.startswith(("continue-on-error", "if:")), line) + def test_every_run_step_uses_bash_with_pipefail(self): + # `shell: bash` under the workflow's `defaults: run:` makes GitHub run every run step as + # `bash --noprofile --norc -eo pipefail {0}` (a bare run step gets `bash -e {0}`, with no pipefail, + # so `cmd | tee log` would hide a failed cmd). The block is exactly that, and the file has + # exactly one `shell:` anywhere (flow style and comments included), so no job or step overrides it. + lines = self.text.splitlines() + self.assertEqual(lines.count("defaults:"), 1) + start = lines.index("defaults:") + self.assertEqual(lines[start : start + 3], ["defaults:", " run:", " shell: bash"]) + self.assertFalse(lines[start + 3].startswith(" "), "another key under defaults") + self.assertEqual(len(re.findall(r"\bshell[\"']?\s*:", self.text)), 1) + + INSTALL_STEPS = ( + # (step id, step name, its if:, the install command that must stand alone on a line) + ( + "install_poetry", + "Install the package and its dev dependencies (poetry)", + "if: matrix.package.backend == 'poetry'", + "poetry install --no-interaction", + ), + ( + "install_hatch", + "Install the package and its dev dependencies (hatch)", + "if: matrix.package.backend == 'hatch'", + '.venv/bin/pip install -e ".[dev]"', + ), + ( + "core_install", + "Test against this repo's fi_instrumentation", + "if: needs.plan.outputs.core_changed == 'true'", + '"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"', + ), + ) + + # The commands of each install step, exactly as reviewed (comments and indentation aside). A line + # filter cannot see a multi-line wrapper that exempts the install from errexit (`{ ...; } && ...`, + # a function called from `if`, `trap ... ERR`, `bash -c '...'`), so the whole script is pinned: + # changing an install step means changing this table, in the same diff. + INSTALL_SCRIPTS = { + "install_poetry": [ + 'pip install "poetry==$POETRY_VERSION"', + "poetry install --no-interaction", + "for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do", + 'module=$(echo "${pin%%==*}" | tr - _)', + 'if poetry run python -c "import $module" 2>/dev/null; then', + 'echo "$module: already present"', + "else", + 'echo "$module: added $pin"', + 'poetry run pip install "$pin"', + "fi", + "done", + ], + "install_hatch": [ + "python -m venv .venv", + '.venv/bin/pip install -e ".[dev]"', + "for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do", + 'module=$(echo "${pin%%==*}" | tr - _)', + 'if .venv/bin/python -c "import $module" 2>/dev/null; then', + 'echo "$module: already present"', + "else", + 'echo "$module: added $pin"', + '.venv/bin/pip install "$pin"', + "fi", + "done", + ], + "core_install": [ + '"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"', + ], + } + + def test_the_install_scripts_are_exactly_as_reviewed(self): + for step_id, name, _condition, _command in self.INSTALL_STEPS: + with self.subTest(step_id): + step = self.step(name, containing=f"id: {step_id}") + commands = [line for line in self.run_lines(step) if line and not line.startswith("#")] + self.assertEqual(commands, self.INSTALL_SCRIPTS[step_id]) + + @staticmethod + def run_lines(step): + """The lines of the step's `run: |` block (the run block is the last key of the install steps).""" + index = step.index("run: |") + return step[index + 1 :] + + def test_a_failed_install_fails_its_step_for_a_listed_package_too(self): + # With continue-on-error on an install step, a failed install of a listed package would not + # fail the job, and the Tests step would run on a broken install. So, for each install step: + # - no continue-on-error, no `shell:` and exactly the pinned `if:`; + # - over its run lines (comments included): no `||`, no `|` outside a `$( ... )` span (so no + # `cmd | tee log`; a `$(...)` is stripped before looking, nested ones are not handled), and + # no `set +` (so no `set +e` and no `set +o errexit`); + # - the install command is a line of its own, exactly: an `if ! ...; then` wrapper, a trailing + # comment or a pipe on that line is not the exact line, so it fails. + for step_id, name, condition, command in self.INSTALL_STEPS: + with self.subTest(step_id): + step = self.step(name, containing=f"id: {step_id}") + self.assertEqual([line for line in step if line.startswith("continue-on-error")], [], step) + self.assertEqual([line for line in step if re.search(r"\bshell[\"']?\s*:", line)], [], step) + self.assertEqual([line for line in step if line.startswith("if:")], [condition], step) + run = self.run_lines(step) + self.assertIn(command, run) + self.assertEqual(run.count(command), 1, run) + self.assertEqual([line for line in run if "||" in line], [], run) + outside = [re.sub(r"\$\([^)]*\)", "", line) for line in run] + self.assertEqual([line for line in outside if "|" in line], [], run) + self.assertEqual([line for line in run if "set +" in line], [], run) + # Only the Tests step may be allowed to fail (for a listed package). + tests = self.step("Tests", containing="id: tests") + self.assertEqual(tests.count("continue-on-error: ${{ matrix.package.known_broken }}"), 1) + self.assertEqual(sum(line.startswith("continue-on-error") for line in self.lines()), 1) + + def test_the_tests_step_is_skipped_after_a_failed_install(self): + # The Tests step (id: tests) has no `if:`, so GitHub gives it the default success(): a failed + # install step skips it, and a broken install never gets its tests run. + tests = self.step("Tests", containing="id: tests") + self.assertEqual([line for line in tests if line.startswith("if:")], [], tests) + + def test_every_job_has_a_time_limit(self): + # Every job under `jobs:` has exactly one job-level `timeout-minutes: N` (4 spaces, N a positive + # integer; a step's own timeout does not bound the job) and an entry in this table, so a new job + # must get both. + limits = {"plan": 10, "fi-instrumentation": 30, "package": 45, "result": 5} + lines = self.text.splitlines() + + def is_comment(line): + return line.lstrip().startswith("#") + + # A comment line (at any indent) neither ends the jobs block nor a job, and a job key may carry + # a trailing comment. + start = lines.index("jobs:") + end = next((i for i in range(start + 1, len(lines)) if re.match(r"^\S", lines[i]) and not is_comment(lines[i])), len(lines)) + found_keys = {} + for i in range(start + 1, end): + match = re.match(r"^ ([A-Za-z0-9_-]+):\s*(#.*)?$", lines[i]) + if match: + found_keys[match.group(1)] = i + keys = list(found_keys) + self.assertEqual(sorted(keys), sorted(limits), "a job without an entry in the table, or an entry without a job") + for key in keys: + with self.subTest(key): + first = found_keys[key] + last = next( + (i for i in range(first + 1, len(lines)) if re.match(r"^ \S|^\S", lines[i]) and not is_comment(lines[i])), + len(lines), + ) + found = [line for line in lines[first:last] if re.match(r"^ timeout-minutes:", line)] + self.assertEqual(len(found), 1, found) + self.assertRegex(found[0], r"^ timeout-minutes: [1-9][0-9]*$") + self.assertEqual(found, [f" timeout-minutes: {limits[key]}"]) + def test_the_verdict_step_runs_always_for_known_broken_packages(self): step = self.step("Known-broken verdict") self.assertIn("if: always() && matrix.package.known_broken", step) @@ -598,7 +791,7 @@ def test_core_tests_reach_the_tests_and_verdict_steps(self): def test_a_pr_base_missing_from_the_clone_is_fetched_then_everything_runs(self): text = self.text check = 'if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then' - fetch = 'git fetch --no-tags origin "$PR_BASE_SHA" || true' + fetch = 'timeout 120 git fetch --no-tags origin "$PR_BASE_SHA" || true' message = 'echo "PR base $PR_BASE_SHA is not in the clone: treating every file as changed" >&2' diff = 'python3 .github/scripts/python_test_plan.py --diff "$PR_BASE_SHA...$PR_HEAD_SHA"' lines = self.lines() diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 1a96ddc42..c708aa2de 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -12,8 +12,10 @@ name: Python tests # A change to python/tests/conftest.py or pytest.ini tests the packages that have core tests. # A moved file plans both the package it left and the one it entered. # A package on .github/python-tests-known-broken.txt still runs and shows a warning, but its -# failures do not fail the check; if its tests pass, the "Known-broken verdict" step fails the -# job until the package is taken off the list. +# test failures do not fail the check; if its tests pass, the "Known-broken verdict" step fails the +# job until the package is taken off the list. A failed install fails the check, listed or not: +# the install steps have no continue-on-error, so the package's tests never run on a broken install. +# Every job has a timeout-minutes, so nothing can hang for GitHub's 6-hour default. # The "Python tests" job at the end is the one check to mark required on main and dev (on dev only # once this file is on dev: GitHub runs the workflow found in the PR's merge commit or the pushed commit). @@ -30,6 +32,12 @@ on: permissions: contents: read +# With an explicit bash, GitHub runs every run step as `bash --noprofile --norc -eo pipefail {0}`: +# a failing command stops the step, and so does a failure on the left of a pipe. +defaults: + run: + shell: bash + concurrency: group: python-tests-${{ github.ref }} # A newer run of the same PR cancels the older one. A running push to main or dev is never @@ -52,6 +60,8 @@ jobs: plan: name: Plan runs-on: ubuntu-latest + # A hung step would otherwise hold the run for GitHub's 6-hour default. + timeout-minutes: 10 outputs: matrix: ${{ steps.plan.outputs.matrix }} has_packages: ${{ steps.plan.outputs.has_packages }} @@ -83,9 +93,9 @@ jobs: pull_request) # The checkout holds every branch's history (fetch-depth 0), so the base commit is # missing only when the base branch was force-pushed or deleted. Fetch it (the repo is - # public, no credentials needed); if that fails too, run every package. + # public, no credentials needed) with a 120 s limit; if that fails or is killed too, run every package. if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then - git fetch --no-tags origin "$PR_BASE_SHA" || true + timeout 120 git fetch --no-tags origin "$PR_BASE_SHA" || true fi if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then echo "PR base $PR_BASE_SHA is not in the clone: treating every file as changed" >&2 @@ -118,6 +128,7 @@ jobs: fi-instrumentation: name: fi_instrumentation runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: @@ -145,6 +156,7 @@ jobs: needs: plan if: needs.plan.outputs.has_packages == 'true' runs-on: ubuntu-latest + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -165,7 +177,6 @@ jobs: - name: Install the package and its dev dependencies (poetry) id: install_poetry if: matrix.package.backend == 'poetry' - continue-on-error: ${{ matrix.package.known_broken }} working-directory: ${{ matrix.package.dir }} run: | pip install "poetry==$POETRY_VERSION" @@ -185,7 +196,6 @@ jobs: - name: Install the package and its dev dependencies (hatch) id: install_hatch if: matrix.package.backend == 'hatch' - continue-on-error: ${{ matrix.package.known_broken }} working-directory: ${{ matrix.package.dir }} run: | python -m venv .venv @@ -205,7 +215,6 @@ jobs: - name: Test against this repo's fi_instrumentation id: core_install if: needs.plan.outputs.core_changed == 'true' - continue-on-error: ${{ matrix.package.known_broken }} run: | # This change touches python/fi_instrumentation/ (or its pyproject / lock): # replace the fi-instrumentation-otel the package resolved with the one in the repo. @@ -254,8 +263,9 @@ jobs: fi exit "$status" - # A package on the known-broken list may fail above without failing the job. This step - # says so with a warning, and fails the job when the listed package passes instead. + # A package on the known-broken list may fail its Tests step without failing the job. This step + # says so with a warning, and fails the job when the listed package passes instead, or when an + # install step failed (the install steps have no continue-on-error). - name: Known-broken verdict if: always() && matrix.package.known_broken env: @@ -274,6 +284,7 @@ jobs: needs: [plan, fi-instrumentation, package] if: always() runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Check the results env: diff --git a/python/fi_instrumentation/instrumentation/context_attributes.py b/python/fi_instrumentation/instrumentation/context_attributes.py index b338c6260..a4e25353d 100644 --- a/python/fi_instrumentation/instrumentation/context_attributes.py +++ b/python/fi_instrumentation/instrumentation/context_attributes.py @@ -192,7 +192,7 @@ def __init__(self, tags: List[str]) -> None: class using_prompt_template(_UsingAttributesContextManager): """ - Context manager to add prompt template, with its version and variables a to the + Context manager to add prompt template, with its version and variables, to the current OpenTelemetry Context. TraceAI instrumentations will read this Context and pass the prompt template as a span attribute, following the TraceAI semantic conventions.