diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a80d39a..7ffd8420 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,13 +1,14 @@ # Pull requests that touch these files need a maintainer's approval (branch # protection: "Require review from Code Owners"). They hold recorded results, -# the tests that pin formulas to their papers, and the CI that runs them; a -# pull request must not edit them to make itself pass. -/test/regression/ @Hananel-Hazan -/test/api_surface.json @Hananel-Hazan -/test/regen_api_surface.py @Hananel-Hazan -/test/test_api_surface.py @Hananel-Hazan -/test/network/test_all_classes.py @Hananel-Hazan -/test/network/test_learning_rule_specs.py @Hananel-Hazan -/test/network/test_mstdp_florian.py @Hananel-Hazan -/test/network/test_perf_equivalence.py @Hananel-Hazan -/.github/ @Hananel-Hazan +# the tests and the settings that decide which tests run, and the CI that runs +# them; a pull request must not edit them to make itself pass. +/test/ @Hananel-Hazan +/.github/ @Hananel-Hazan +/pyproject.toml @Hananel-Hazan +/poetry.lock @Hananel-Hazan +/pytest.ini @Hananel-Hazan +/.pytest.ini @Hananel-Hazan +/tox.ini @Hananel-Hazan +/setup.cfg @Hananel-Hazan +/.coveragerc @Hananel-Hazan +/conftest.py @Hananel-Hazan diff --git a/.github/scripts/bench_ab.py b/.github/scripts/bench_ab.py index 4a825391..0f59655d 100644 --- a/.github/scripts/bench_ab.py +++ b/.github/scripts/bench_ab.py @@ -30,8 +30,12 @@ The report (Markdown) is written to DIR/bench_ab.md, appended to --summary if given (GitHub's $GITHUB_STEP_SUMMARY), and the raw numbers to DIR/bench_ab.json. -The exit status is always 0: this is a non-required check and a human decides; -the verdict, or a TOOL ERROR line, is in the report. +Without --fail-above the exit status is always 0 and a human decides; the +verdict, or a TOOL ERROR line, is in the report. With --fail-above X the exit +status is 1 when any workload's median ratio is above X and its interval's lower +end is above 1.5 (a large, clear slowdown, far outside the no-change spread of +0.90 to 1.12 per round measured on GitHub runners), or when the tool failed; CI +uses X = 2. """ from __future__ import annotations @@ -318,6 +322,12 @@ def main() -> int: ap.add_argument("--time", type=int, default=100, help="time steps per run") ap.add_argument("--threads", type=int, default=2) ap.add_argument("--threshold", type=float, default=1.25) + ap.add_argument( + "--fail-above", + type=float, + default=None, + help="exit 1 when a median ratio is above this and the interval's lower end above 1.5", + ) ap.add_argument("--min-seconds", type=float, default=1.0) ap.add_argument("--only", default=",".join(WORKLOADS)) ap.add_argument("--timeout", type=int, default=900, help="seconds per process") @@ -383,10 +393,25 @@ def log(msg): indent=1, ) ) + failing = [] + if args.fail_above is not None: + failing = [ + r["workload"] + for r in rows + if r["ratio_median"] > args.fail_above and r["ratio_ci95"][0] > 1.5 + ] + if error: + failing.append("tool error") + report += ( + f"\n**Failed:** {', '.join(failing)} (median above {args.fail_above}x " + "the base, or the tool failed).\n" + if failing + else f"\nRequired limit: no workload above {args.fail_above}x the base.\n" + ) if args.summary: with open(args.summary, "a") as f: f.write(report) - return 0 + return 1 if failing else 0 if __name__ == "__main__": diff --git a/.github/scripts/coverage_floor.py b/.github/scripts/coverage_floor.py index 4e178f84..9a773768 100755 --- a/.github/scripts/coverage_floor.py +++ b/.github/scripts/coverage_floor.py @@ -13,12 +13,19 @@ import sys import xml.etree.ElementTree as ET -# Measured on CI? No: measured locally on 2026-10-24 with torch 2.14.1 CPU after -# Measured on GitHub CI (build job, Python 3.13, CPU torch), run 40798879593, -# 2026-10-28. +# One point under the coverage measured on GitHub CI (build job, Python 3.13, CPU +# torch): network and learning in run 40798879593, the others in run 40804291700 +# (master 33913c7), both 2026-10-28. FLOORS = { "network": 89.6, # 90.6% measured "learning": 94.2, # 95.2% measured + "models": 98.3, # 99.3% measured + "encoding": 73.6, # 74.6% measured + "conversion": 58.8, # 59.8% measured + "analysis": 32.8, # 33.8% measured + "environment": 26.3, # 27.3% measured + "pipeline": 19.8, # 20.8% measured + "datasets": 20.6, # 21.6% measured } # Cobertura filenames are relative to the element, which pytest-cov sets to # the bindsnet package directory, so "network/nodes.py", not "bindsnet/network/...". diff --git a/.github/scripts/fails_before_fix.py b/.github/scripts/fails_before_fix.py index 8063b348..9f6948d5 100644 --- a/.github/scripts/fails_before_fix.py +++ b/.github/scripts/fails_before_fix.py @@ -9,8 +9,13 @@ - on the head commit as checked out. It passes when at least one test fails (or cannot be collected) on the base and -passes on the head. A test file that cannot be imported on the base (for -example, it imports a function the pull request adds) counts as failing there. +passes on the head. Only a test that ran on the base and failed there counts +(any exception in its body, including an assertion or "DID NOT RAISE"). A test +that cannot be imported or set up on the base (for example, it imports a +function the pull request adds) shows no bug and does not count; a pull request +that adds a feature needs the maintainer label. The summary shows each counted +test's failure on the base, because a test can also fail for reasons unrelated +to the bug, and a maintainer must read it. It also passes, with a note, when the pull request does not touch any ``bindsnet/*.py`` file (documentation, dependencies, workflows), or when the @@ -53,8 +58,11 @@ def changed(base, head, path, filters="ACMR"): def run_pytest(tree, files, xml): """Run pytest on ``files`` inside ``tree`` with ``tree`` first on the path. - Returns {test id: "pass" | "fail"} and {module: "error"} for files that - could not be collected. Skipped and xfailed tests are left out. + Returns {test id: ("pass" | "fail" | "error", message)} and the modules that + could not be collected. "fail" means the test ran and failed (any exception + in its body); "error" means set-up or collection failed. Skipped and xfailed + tests are left out. The settings come from the tree's pyproject.toml only, so + an added pytest.ini or setup.cfg cannot change what runs. """ env = dict(os.environ, PYTHONPATH=str(tree)) where = subprocess.run( @@ -69,7 +77,8 @@ def run_pytest(tree, files, xml): raise RuntimeError(f"bindsnet imported from {where}, not from {tree}") subprocess.run( [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider"] - + ["--continue-on-collection-errors", f"--junitxml={xml}", *files], + + ["-c", "pyproject.toml", "--continue-on-collection-errors"] + + [f"--junitxml={xml}", *files], cwd=tree, env=env, ) @@ -82,7 +91,12 @@ def run_pytest(tree, files, xml): test_id = f"{case.get('classname')}::{case.get('name')}" if "skipped" in tags: continue - results[test_id] = "fail" if tags & {"failure", "error"} else "pass" + for kind, tag in (("fail", "failure"), ("error", "error"), ("pass", None)): + child = case.find(tag) if tag else None + if tag is None or child is not None: + message = (child.get("message") or "") if child is not None else "" + results[test_id] = (kind, message.splitlines()[0][:200] if message else "") + break return results, broken @@ -106,6 +120,10 @@ def finish(code, message): f.write(text) return code + # Commit ids, so "master" or "HEAD" mean the same inside the base worktree. + args.base = git("rev-parse", args.base).strip() + args.head = git("rev-parse", args.head).strip() + labels = {label.strip() for label in args.labels.split(",") if label.strip()} if args.skip_label in labels: return finish(0, f"Skipped: a maintainer added the label `{args.skip_label}`.") @@ -137,26 +155,42 @@ def finish(code, message): subprocess.run(["git", "worktree", "remove", "--force", str(base_tree)]) shutil.rmtree(work, ignore_errors=True) - def failed_on_base(test_id): + def collection_broken(test_id): module = test_id.split("::")[0] - if any(module == m or module.startswith(m + ".") for m in broken): - return True - return on_base.get(test_id) == "fail" - - shown = [t for t, r in on_head.items() if r == "pass" and failed_on_base(t)] + return any(module == m or module.startswith(m + ".") for m in broken) + + passing = [t for t, (kind, _) in on_head.items() if kind == "pass"] + # Evidence: the test ran on the base and failed there. A test that could not + # be imported or set up on the base (for example, it imports a name the PR + # adds) shows nothing about a bug; it is listed for the maintainer instead. + shown = [t for t in passing if on_base.get(t, ("", ""))[0] == "fail"] + unshown = [ + t + for t in passing + if t not in shown and (collection_broken(t) or on_base.get(t, ("",))[0] == "error") + ] + if unshown: + lines.append( + "Not counted (could not be collected or set up on the base, so they " + "show no bug): " + ", ".join(f"`{t}`" for t in sorted(unshown)[:20]) + ) + lines.append("") if shown: - lines.append("Tests that fail on the base and pass with this change:") - lines += [f"- `{t}`" for t in sorted(shown)[:50]] + lines.append( + "Tests that fail on the base and pass with this change, with the " + "base failure (a maintainer checks that it is the bug):" + ) + lines += [f"- `{t}`: {on_base[t][1]}" for t in sorted(shown)[:50]] if len(shown) > 50: lines.append(f"- ... and {len(shown) - 50} more") return finish(0, "\n**Passed.**") return finish( 1, - "**Failed:** every test in the changed test files gives the same result " - "with and without the library change, so none of them shows the bug " - "this pull request fixes. Add a test that fails on `master` and passes " - "with this change. If this is not a bug fix (a refactor or a speed-up), " - f"a maintainer can add the label `{args.skip_label}`.", + "**Failed:** no test in the changed test files ran and failed on the " + "base while passing with this change, so none of them shows the bug this " + "pull request fixes. Add a test that fails on `master` and passes with " + "this change. If this is not a bug fix (a refactor, a speed-up, or a new " + f"feature), a maintainer can add the label `{args.skip_label}`.", ) diff --git a/.github/scripts/weakened_tests.py b/.github/scripts/weakened_tests.py index c2f5e1ab..aef35a6a 100644 --- a/.github/scripts/weakened_tests.py +++ b/.github/scripts/weakened_tests.py @@ -4,19 +4,24 @@ This script compares the base and head commits and lists every place where that may have happened: -- a test file or a test function was removed; -- a function in a test file (helpers included) has fewer checks (``assert`` - statements and calls whose name contains ``assert``, ``allclose``, - ``equal`` or ``raises``); -- a test file has more skip or xfail markers (``skip``, ``skipif``, ``xfail``, - ``importorskip``); -- a tolerance in a test file now allows a larger value than before: a keyword - argument ``atol``, ``rtol``, ``abs``, ``rel``, ``tol`` or ``delta``, or an - assigned name containing ``tol``; +- a test file, test function or collected test id disappeared (``--collect`` + compares what pytest collects on both commits, which also catches + ``del test_x``, ``__test__ = False``, shorter parameter lists and tests + deselected by a conftest); renamed files count as removed plus added; +- a function in a ``test/`` file (test helpers and ``networks.py`` included) + has fewer checks or fewer exact comparisons (``torch.equal`` and the like), + more checks that cannot fail (``assert x or True``, an ``except + AssertionError``, a ``return`` before the first check), or more skip/xfail + markers (counted per function and for the module, e.g. ``pytestmark``); +- a tolerance is new or larger: keyword ``atol``/``rtol``/``abs``/``rel``/ + ``tol``/``delta``, the positional tolerance of ``allclose``/``isclose``/ + ``approx``, a parameter default, or an assigned name containing ``tol`` + (constant arithmetic is evaluated); - a recorded-result file changed: golden outputs, operation counts, the API snapshot, or the coverage floor; -- a ``conftest.py`` or the pytest settings in ``pyproject.toml`` changed (they - decide which tests run). +- a file that steers the tests changed: any ``conftest.py``, ``pyproject.toml``, + ``pytest.ini``, ``tox.ini``, ``setup.cfg``, ``.coveragerc``, or the torch + versions in ``poetry.lock`` (goldens are tied to them). Each finding needs a maintainer's look, not a fix: removing a wrong test or regenerating goldens after a real fix is fine when the pull request says why. @@ -32,19 +37,41 @@ import argparse import ast +import os +import shutil import subprocess import sys +import tempfile ACCEPT_LABEL = "tests-changed-ok" -CHECK_WORDS = ("assert", "allclose", "equal", "raises") +CHECK_WORDS = ("assert", "allclose", "isclose", "equal", "raises", "approx") +EXACT_WORDS = ("equal",) # torch.equal, array_equal, assert_equal SKIP_WORDS = {"skip", "skipif", "xfail", "importorskip"} TOLERANCE_KEYWORDS = {"atol", "rtol", "abs", "rel", "tol", "delta"} +# Positional tolerance arguments: function name -> {position: keyword}. +POSITIONAL_TOLERANCE = { + "allclose": {2: "rtol", 3: "atol"}, + "isclose": {2: "rtol", 3: "atol"}, + "approx": {1: "rel", 2: "abs"}, +} RECORDED = ( "test/regression/golden/", "test/regression/op_counts.json", "test/api_surface.json", ".github/scripts/coverage_floor.py", + # How goldens are compared and computed (FLOAT_ATOL, the CPU pins). + "test/regression/networks.py", + "test/regression/cpu_pins.py", ) +# Files that decide which tests run, how, or how coverage is measured. +STEERING = { + "pyproject.toml", + "pytest.ini", + ".pytest.ini", + "tox.ini", + "setup.cfg", + ".coveragerc", +} def git(*args): @@ -70,16 +97,33 @@ def call_name(node): def number(node): - try: - value = ast.literal_eval(node) - except (ValueError, SyntaxError, TypeError): - return None - return value if isinstance(value, (int, float)) else None + """Value of a numeric constant expression (``1e-6``, ``10 * 1e-6``), else None.""" + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): + return None if isinstance(node.value, bool) else node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + v = number(node.operand) + return None if v is None else (-v if isinstance(node.op, ast.USub) else v) + if isinstance(node, ast.BinOp): + left, right = number(node.left), number(node.right) + if left is None or right is None: + return None + ops = { + ast.Add: lambda x, y: x + y, + ast.Sub: lambda x, y: x - y, + ast.Mult: lambda x, y: x * y, + ast.Div: lambda x, y: x / y if y else None, + ast.Pow: lambda x, y: x**y if abs(y) < 100 else None, + } + op = ops.get(type(node.op)) + return op(left, right) if op else None + return None def tolerances_in(nodes): - """Largest value of each tolerance keyword argument, and of each assigned - name containing "tol" (``ATOL = 1e-6``), among ``nodes``.""" + """Largest value of each tolerance among ``nodes``: keyword arguments, + positional tolerance arguments of allclose/isclose/approx, names containing + "tol" that are assigned (also with a type annotation), and parameter + defaults named like a tolerance.""" found = {} for node in nodes: pairs = [] @@ -87,12 +131,29 @@ def tolerances_in(nodes): pairs = [ (k.arg, k.value) for k in node.keywords if k.arg in TOLERANCE_KEYWORDS ] + for pos, key in POSITIONAL_TOLERANCE.get(call_name(node), {}).items(): + if len(node.args) > pos: + pairs.append((key, node.args[pos])) elif isinstance(node, ast.Assign): pairs = [ (t.id, node.value) for t in node.targets if isinstance(t, ast.Name) and "tol" in t.id.lower() ] + elif isinstance(node, ast.AnnAssign) and node.value is not None: + t = node.target + if isinstance(t, ast.Name) and "tol" in t.id.lower(): + pairs = [(t.id, node.value)] + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args.args + node.args.kwonlyargs + defaults = [None] * (len(node.args.args) - len(node.args.defaults)) + defaults += node.args.defaults + node.args.kw_defaults + pairs = [ + (arg.arg, d) + for arg, d in zip(args, defaults) + if d is not None + and (arg.arg in TOLERANCE_KEYWORDS or "tol" in arg.arg.lower()) + ] for key, value_node in pairs: value = number(value_node) if value is not None: @@ -100,97 +161,198 @@ def tolerances_in(nodes): return found +def _always_true(test): + if isinstance(test, ast.Constant): + return bool(test.value) + if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.Or): + return any(_always_true(v) for v in test.values) + return False + + +def _swallows_assertions(node): + if not isinstance(node, ast.Try): + return False + for handler in node.handlers: + caught = handler.type + names = [] + if caught is None: + return True + for c in caught.elts if isinstance(caught, ast.Tuple) else [caught]: + names.append(c.id if isinstance(c, ast.Name) else getattr(c, "attr", "")) + if any(n in ("AssertionError", "Exception", "BaseException") for n in names): + return True + return False + + +def _early_return(node): + body = node.body + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant): + body = body[1:] # docstring + return bool(body) and isinstance(body[0], ast.Return) + + def describe(source): """Per function (helpers included, so a check moved into a helper still - counts): number of checks and the largest value of each tolerance. The - module level is the entry "". Plus the file's count of skip and - xfail markers.""" + counts) and for the module level (""): number of checks, of exact + comparisons, of checks that cannot fail, of skip/xfail markers, and the + largest value of each tolerance.""" tree = ast.parse(source) - skips = 0 - for node in ast.walk(tree): - if isinstance(node, (ast.Name, ast.Attribute)): - name = node.id if isinstance(node, ast.Name) else node.attr - skips += name in SKIP_WORDS - functions = {"": (0, tolerances_in(tree.body))} + + def summary(nodes, own=None): + nodes = list(nodes) + calls = [n for n in nodes if isinstance(n, ast.Call)] + names = [ + n.id if isinstance(n, ast.Name) else n.attr + for n in nodes + if isinstance(n, (ast.Name, ast.Attribute)) + ] + return { + "checks": sum(isinstance(n, ast.Assert) for n in nodes) + + sum(any(w in call_name(c).lower() for w in CHECK_WORDS) for c in calls), + "exact": sum(any(w in call_name(c).lower() for w in EXACT_WORDS) for c in calls), + "cannot fail": sum( + isinstance(n, ast.Assert) and _always_true(n.test) for n in nodes + ) + + sum(_swallows_assertions(n) for n in nodes) + + int(own is not None and _early_return(own)), + "skip/xfail markers": sum(n in SKIP_WORDS for n in names), + "tolerances": tolerances_in(nodes), + } + + module_nodes = [ + n + for top in tree.body + if not isinstance(top, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + for n in ast.walk(top) + ] + functions = {"": summary(module_nodes)} stack = [(node, "") for node in tree.body] while stack: node, prefix = stack.pop() if isinstance(node, ast.ClassDef): stack += [(child, f"{prefix}{node.name}.") for child in node.body] - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - inner = list(ast.walk(node)) - checks = sum(isinstance(n, ast.Assert) for n in inner) + sum( - any(word in call_name(n).lower() for word in CHECK_WORDS) - for n in inner - if isinstance(n, ast.Call) + functions[prefix + node.name] = summary( + [n for d in node.decorator_list for n in ast.walk(d)] ) - functions[prefix + node.name] = (checks, tolerances_in(inner)) - return functions, skips + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + functions[prefix + node.name] = summary(ast.walk(node), own=node) + return functions -def compare_test_file(path, before, after): +def compare_source(path, before, after): + """Weakenings in one test or test-helper file.""" if after is None: - return [f"`{path}`: test file removed"] + return [f"`{path}`: file removed"] if before is None: return [] - old_functions, old_skips = describe(before) - new_functions, new_skips = describe(after) + old, new = describe(before), describe(after) found = [] - for name, (old_checks, old_tols) in old_functions.items(): - if name not in new_functions: + for name, o in old.items(): + where = f"`{path}::{name}`" + if name not in new: if name.rsplit(".", 1)[-1].startswith("test"): - found.append(f"`{path}::{name}`: test removed") + found.append(f"{where}: test removed") continue - new_checks, new_tols = new_functions[name] - if new_checks < old_checks: - found.append( - f"`{path}::{name}`: checks went from {old_checks} to {new_checks}" - ) - for keyword, value in new_tols.items(): - if keyword in old_tols and value > old_tols[keyword]: + n = new[name] + for key in ("checks", "exact"): + if n[key] < o[key]: + found.append(f"{where}: {key} went from {o[key]} to {n[key]}") + for key in ("cannot fail", "skip/xfail markers"): + if n[key] > o[key]: + found.append(f"{where}: {key} went from {o[key]} to {n[key]}") + for keyword, value in n["tolerances"].items(): + if keyword not in o["tolerances"]: + found.append(f"{where}: tolerance `{keyword}` added ({value:g})") + elif value > o["tolerances"][keyword]: found.append( - f"`{path}::{name}`: `{keyword}` widened from " - f"{old_tols[keyword]:g} to {value:g}" + f"{where}: `{keyword}` widened from " + f"{o['tolerances'][keyword]:g} to {value:g}" ) - if new_skips > old_skips: - found.append( - f"`{path}`: skip/xfail markers went from {old_skips} to {new_skips}" - ) + for name, n in new.items(): # new functions or classes that start skipped + if name not in old and n["skip/xfail markers"]: + found.append(f"`{path}::{name}`: new, with skip/xfail markers") return found +def torch_pins(lock): + """Locked versions of torch and torchvision in a poetry.lock text.""" + pins, name = {}, None + for line in (lock or "").splitlines(): + if line.startswith("name = "): + name = line.split('"')[1] + elif line.startswith("version = ") and name in ("torch", "torchvision"): + pins[name] = line.split('"')[1] + return pins + + +def collected_ids(commit): + """Test ids pytest collects at ``commit`` (its own code and pyproject.toml).""" + work = tempfile.mkdtemp(prefix="weakened_tests_") + tree = os.path.join(work, "tree") + try: + git("worktree", "add", "--detach", tree, commit) + out = subprocess.run( + [sys.executable, "-m", "pytest", "--collect-only", "-q"] + + ["-p", "no:cacheprovider", "-c", "pyproject.toml"], + cwd=tree, + env=dict(os.environ, PYTHONPATH=tree), + capture_output=True, + text=True, + ).stdout + finally: + subprocess.run(["git", "worktree", "remove", "--force", tree]) + shutil.rmtree(work, ignore_errors=True) + return {line.strip() for line in out.splitlines() if "::" in line} + + def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--base", required=True) parser.add_argument("--head", required=True) parser.add_argument("--labels", default="") parser.add_argument("--accept-label", default=ACCEPT_LABEL) + parser.add_argument( + "--collect", + action="store_true", + help="also compare the test ids pytest collects (needs the test environment)", + ) parser.add_argument("--summary", default=None) args = parser.parse_args() + args.base = git("rev-parse", args.base).strip() + args.head = git("rev-parse", args.head).strip() - paths = git("diff", "--name-only", args.base, args.head).splitlines() + # --no-renames: a renamed file is a removal plus an addition, so changes + # inside a renamed test file are compared with the old file. + paths = git("diff", "--name-only", "--no-renames", args.base, args.head).splitlines() found = [] for path in paths: name = path.rsplit("/", 1)[-1] if path.startswith(RECORDED): - found.append(f"`{path}`: recorded results changed") + found.append(f"`{path}`: recorded results, or how they are compared, changed") elif name == "conftest.py": found.append(f"`{path}`: changed (it decides how tests run)") - elif path == "pyproject.toml": - before = source_at(args.base, path) or "" - after = source_at(args.head, path) or "" - section = "[tool.pytest" - if before.partition(section)[2] != after.partition(section)[2]: - found.append("`pyproject.toml`: pytest settings changed") - elif ( - path.startswith("test/") - and name.startswith("test_") - and name.endswith(".py") - ): - found += compare_test_file( + elif path in STEERING: + found.append(f"`{path}`: changed (it can decide which tests run and how)") + elif path == "poetry.lock": + old, new = torch_pins(source_at(args.base, path)), torch_pins( + source_at(args.head, path) + ) + if old != new: + found.append( + f"`poetry.lock`: torch versions changed {old} -> {new}; the golden " + "and operation-count files must be regenerated in this PR" + ) + elif path.startswith("test/") and name.endswith(".py"): + found += compare_source( path, source_at(args.base, path), source_at(args.head, path) ) + if args.collect: + gone = sorted(collected_ids(args.base) - collected_ids(args.head)) + found += [f"`{t}`: no longer collected" for t in gone[:50]] + if len(gone) > 50: + found.append(f"... and {len(gone) - 50} more tests no longer collected") + labels = {label.strip() for label in args.labels.split(",") if label.strip()} lines = ["## Tests weakened or recorded results changed", ""] if not found: diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b9a25760..77f33fd4 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,9 +1,11 @@ -# Speed comparison of a pull request against its base. Informational only: -# this is NOT a required check, and it always exits 0. The verdict (SLOWER or -# not) is in the job summary and in the uploaded "bench-ab" artifact; a -# maintainer decides. It does not post PR comments, because pull requests from -# forks get a read-only token. Script and statistics: .github/scripts/bench_ab.py. -name: Speed comparison (informational) +# Speed comparison of a pull request against its base, required on master: it +# fails only on a large, clear slowdown (a workload more than 2x slower than the +# base, --fail-above 2), which the operation-count tests cannot see (for +# example slow pure-Python code). Smaller changes are flagged SLOWER in the job +# summary and the "bench-ab" artifact for a maintainer. It does not post PR +# comments, because pull requests from forks get a read-only token. Script and +# statistics: .github/scripts/bench_ab.py. +name: Speed comparison on: pull_request: @@ -41,11 +43,11 @@ jobs: # difference from the base commit is exactly the pull request's change. - name: Compare base and head (alternating rounds) env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.sha }} + # HEAD^1 of the test merge is the master tip it merges into. run: > poetry run python .github/scripts/bench_ab.py - --base "$BASE_SHA" --head "$HEAD_SHA" + --base "$(git rev-parse HEAD^1)" --head "$HEAD_SHA" --fail-above 2 --out bench-out --summary "$GITHUB_STEP_SUMMARY" - uses: actions/upload-artifact@v7 if: always() diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 7dc3c7d5..a1132a37 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -41,7 +41,11 @@ jobs: - name: Lint with flake8 (syntax errors and undefined names) run: pipx run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - name: Test with pytest (with coverage) - run: poetry run pytest --cov=bindsnet --cov-report=term --cov-report=xml + # -c and --cov-config pin the settings to pyproject.toml, so an added + # pytest.ini, setup.cfg, tox.ini or .coveragerc cannot change what runs. + run: > + poetry run pytest -c pyproject.toml --cov-config=pyproject.toml + --cov=bindsnet --cov-report=term --cov-report=xml # A floor, not a target: fails only when coverage of the core packages drops # below the value recorded in the script (see its docstring). - name: Check coverage floor @@ -75,4 +79,4 @@ jobs: grep -m1 "model name" /proc/cpuinfo if grep -qw avx512f /proc/cpuinfo; then echo "AVX-512 yes"; else echo "AVX-512 no"; fi - name: Test with pytest - run: poetry run pytest + run: poetry run pytest -c pyproject.toml diff --git a/.github/workflows/test-guard.yml b/.github/workflows/test-guard.yml index 3340ba8e..c37da6a9 100644 --- a/.github/workflows/test-guard.yml +++ b/.github/workflows/test-guard.yml @@ -31,17 +31,24 @@ jobs: weakened-tests: name: weakened-tests runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 15 steps: - uses: actions/checkout@v7 with: fetch-depth: 2 # HEAD is the test merge; HEAD^1 is master it merges into + - name: Install Poetry + run: pipx install "poetry==${POETRY_VERSION}" - uses: actions/setup-python@v7 with: python-version: "3.13" + cache: poetry + - name: Install dependencies (to collect tests on both commits) + run: | + poetry install + .github/scripts/use_cpu_torch.sh "poetry run python" - name: Compare tests with the base run: > - python .github/scripts/weakened_tests.py + poetry run python .github/scripts/weakened_tests.py --collect --base "$(git rev-parse HEAD^1)" --head "$(git rev-parse HEAD)" --labels "$LABELS" --summary "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ad14ac2..8379da55 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,20 +35,29 @@ If your pull request changes any file under `bindsnet/`, it must add or change a test that **fails on `master` and passes with your change**. The `bug-test` check (`.github/scripts/fails_before_fix.py`) runs the test files you added or changed twice, once on `master`'s code and once on yours, and fails if none of them goes -from failing to passing. This shows the bug is real and that your change is what -fixes it. You can run the same check locally: +from failing to passing. Only a test that runs on `master` and fails there counts; +a test that cannot even be imported on `master` (because it imports something your +change adds) shows no bug. The job summary shows how each counted test failed on +`master`, and a maintainer checks that it is the bug. You can run the same check +locally: ```shell poetry run python .github/scripts/fails_before_fix.py --base master --head HEAD ``` -A change that is not a bug fix (a refactor or a speed-up) has no such test; a -maintainer then adds the label `no-bug-test`, and every existing test must still pass. +A change that is not a bug fix (a refactor, a speed-up, or a new feature) has no +such test; a maintainer then adds the label `no-bug-test`, and every existing test +must still pass. The `weakened-tests` check (`.github/scripts/weakened_tests.py`) lists every -place where a pull request may make tests pass by changing the tests: a removed -test, fewer checks in a test, a new skip or xfail, a wider tolerance, or changed -recorded results (golden files, operation counts, API snapshot, coverage floor). +place where a pull request may make tests pass by changing the tests: a removed or +no longer collected test, fewer checks or exact comparisons, a check that cannot +fail, a new skip or xfail, a new or wider tolerance, changed recorded results +(golden files, operation counts, API snapshot, coverage floor), or a changed file +that decides which tests run (`conftest.py`, `pyproject.toml`, `pytest.ini`, +`setup.cfg`, `tox.ini`, `.coveragerc`, the torch version in `poetry.lock`). CI runs +pytest with `-c pyproject.toml`, so other configuration files are ignored there. +The `bench` check fails when a workload runs more than twice as slow as on `master`. Some of these are right, for example regenerating goldens after a real fix. Explain each one in the pull request; a maintainer reviews them and adds the label `tests-changed-ok`. Files listed in `.github/CODEOWNERS` also need a maintainer's diff --git a/test/regression/golden/conn_postpre_mean.pt.xz b/test/regression/golden/conn_postpre_mean.pt.xz new file mode 100644 index 00000000..53d6262b Binary files /dev/null and b/test/regression/golden/conn_postpre_mean.pt.xz differ diff --git a/test/regression/networks.py b/test/regression/networks.py index 587d18f3..eb520d14 100644 --- a/test/regression/networks.py +++ b/test/regression/networks.py @@ -18,6 +18,7 @@ import io import lzma +import os from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Dict, List, Tuple @@ -78,7 +79,7 @@ def dc2015_b16(dt): return _dc2015(16, dt) -def _dense_postpre(batch, dt): +def _dense_postpre(batch, dt, **rule_kwargs): from bindsnet.learning import PostPre from bindsnet.network import Network from bindsnet.network.nodes import Input, LIFNodes @@ -96,6 +97,7 @@ def _dense_postpre(batch, dt): wmin=0.0, wmax=1.0, norm=20.0, + **rule_kwargs, ), "in", "out", @@ -118,6 +120,15 @@ def conn_postpre_dt05(dt): return _dense_postpre(1, dt) +@network("conn_postpre_mean", trials=[(4, 60, 1), (4, 40, 2)], norm=True, dt=0.5) +def conn_postpre_mean(dt): + # A custom reduction takes PostPre's general (bmm) path instead of the fused + # addmm_ one; dt = 0.5 makes a stray dt factor visible there. + import torch + + return _dense_postpre(4, dt, reduction=torch.mean) + + @network("local2d_postpre", trials=[(1, 60, 1), (1, 40, 2)]) def local2d_postpre(dt): from bindsnet.learning import PostPre @@ -389,7 +400,17 @@ def rules_mix_dt05(dt): def deterministic(): """CPU determinism settings required for exact comparisons (see - ``cpu_pins.py``). Fails when the capability pin was applied too late.""" + ``cpu_pins.py``). Fails when a pin is missing or was applied too late.""" + import cpu_pins + + unpinned = { + k: os.environ.get(k) for k, v in cpu_pins.PINS.items() if os.environ.get(k) != v + } + if unpinned: + raise RuntimeError( + f"CPU maths libraries not pinned ({unpinned}): run through pytest " + "(test/conftest.py) or regen.py, which apply cpu_pins.PINS" + ) capability = torch.backends.cpu.get_cpu_capability() if capability != "AVX2": raise RuntimeError( diff --git a/test/regression/op_counts.json b/test/regression/op_counts.json index 395fc6ff..de4b9653 100644 --- a/test/regression/op_counts.json +++ b/test/regression/op_counts.json @@ -3,8 +3,8 @@ "torch": "2.14.1+cu130", "python": "3.11.9", "date": "2026-10-28", - "commit": "926e60dcbe593715424780b403878a9d45d18729+uncommitted bindsnet code changes", - "reason": "MCC_learning.MSTDPET and Rmax keep per-sample state ([batch, ...]) so batch > 1 works: bmm instead of outer and a reduction over the batch; at batch 1 mcc.MSTDPET 24 -> 27 and rule.Rmax 24 -> 26 operations; mcc.MSTDPET.b4 and rule.Rmax.b4 counted again" + "commit": "fa0847d0091f654bec0256e6e8827c781284627a", + "reason": "Workloads added: one forward() of every Nodes class (only LIFNodes was counted), so an extra operation in any neuron step is caught" }, "excluded": {}, "totals": { @@ -20,8 +20,30 @@ "mcc.NoOp.b4": 0, "mcc.PostPre.b1": 19, "mcc.PostPre.b4": 19, + "nodes.AdaptiveLIFNodes_forward.b1": 20, + "nodes.AdaptiveLIFNodes_forward.b4": 20, + "nodes.BoostedLIFNodes_forward.b1": 11, + "nodes.BoostedLIFNodes_forward.b4": 11, + "nodes.CSRMNodes_forward.b1": 41, + "nodes.CSRMNodes_forward.b4": 43, + "nodes.CurrentLIFNodes_forward.b1": 16, + "nodes.CurrentLIFNodes_forward.b4": 16, + "nodes.DiehlAndCookNodes_forward.b1": 22, + "nodes.DiehlAndCookNodes_forward.b4": 22, + "nodes.IFNodes_forward.b1": 11, + "nodes.IFNodes_forward.b4": 11, + "nodes.Input_forward.b1": 4, + "nodes.Input_forward.b4": 4, + "nodes.IzhikevichNodes_forward.b1": 42, + "nodes.IzhikevichNodes_forward.b4": 54, "nodes.LIFNodes_forward.b1": 12, "nodes.LIFNodes_forward.b4": 12, + "nodes.McCullochPitts_forward.b1": 4, + "nodes.McCullochPitts_forward.b4": 4, + "nodes.Nodes_forward.b1": 3, + "nodes.Nodes_forward.b4": 3, + "nodes.SRM0Nodes_forward.b1": 24, + "nodes.SRM0Nodes_forward.b4": 24, "rule.DiehlAndCook.b1": 18, "rule.DiehlAndCook.b4": 18, "rule.Hebbian.b1": 18, @@ -162,6 +184,213 @@ "aten::t": 2, "aten::view": 4 }, + "nodes.AdaptiveLIFNodes_forward.b1": { + "aten::_to_copy": 2, + "aten::add.Tensor": 1, + "aten::add_.Tensor": 3, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 2, + "aten::mul_.Tensor": 3, + "aten::sub_.Tensor": 2, + "aten::sum.dim_IntList": 1 + }, + "nodes.AdaptiveLIFNodes_forward.b4": { + "aten::_to_copy": 2, + "aten::add.Tensor": 1, + "aten::add_.Tensor": 3, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 2, + "aten::mul_.Tensor": 3, + "aten::sub_.Tensor": 2, + "aten::sum.dim_IntList": 1 + }, + "nodes.BoostedLIFNodes_forward.b1": { + "aten::add_.Tensor": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::gt.Scalar": 1, + "aten::masked_fill_.Scalar": 2, + "aten::masked_fill_.Tensor": 2, + "aten::mul_.Tensor": 2, + "aten::sub_.Tensor": 1 + }, + "nodes.BoostedLIFNodes_forward.b4": { + "aten::add_.Tensor": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::gt.Scalar": 1, + "aten::masked_fill_.Scalar": 2, + "aten::masked_fill_.Tensor": 2, + "aten::mul_.Tensor": 2, + "aten::sub_.Tensor": 1 + }, + "nodes.CSRMNodes_forward.b1": { + "aten::_to_copy": 1, + "aten::add.Tensor": 1, + "aten::add_.Tensor": 3, + "aten::bmm": 2, + "aten::cat": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul.Tensor": 1, + "aten::mul_.Tensor": 3, + "aten::permute": 10, + "aten::slice.Tensor": 1, + "aten::sum.dim_IntList": 1, + "aten::unsqueeze": 5, + "aten::view": 9 + }, + "nodes.CSRMNodes_forward.b4": { + "aten::_to_copy": 1, + "aten::_unsafe_view": 2, + "aten::add.Tensor": 1, + "aten::add_.Tensor": 3, + "aten::bmm": 2, + "aten::cat": 1, + "aten::clone": 3, + "aten::ge.Tensor": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul.Tensor": 1, + "aten::mul_.Tensor": 3, + "aten::permute": 10, + "aten::slice.Tensor": 1, + "aten::sum.dim_IntList": 1, + "aten::unsqueeze": 5, + "aten::view": 7 + }, + "nodes.CurrentLIFNodes_forward.b1": { + "aten::_to_copy": 1, + "aten::add_.Tensor": 3, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 1, + "aten::mul_.Tensor": 3, + "aten::sub_.Tensor": 2 + }, + "nodes.CurrentLIFNodes_forward.b4": { + "aten::_to_copy": 1, + "aten::add_.Tensor": 3, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 1, + "aten::mul_.Tensor": 3, + "aten::sub_.Tensor": 2 + }, + "nodes.DiehlAndCookNodes_forward.b1": { + "aten::_local_scalar_dense": 1, + "aten::_to_copy": 2, + "aten::add.Tensor": 1, + "aten::add_.Tensor": 3, + "aten::any": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 2, + "aten::mul_.Tensor": 3, + "aten::sub_.Tensor": 2, + "aten::sum.dim_IntList": 1 + }, + "nodes.DiehlAndCookNodes_forward.b4": { + "aten::_local_scalar_dense": 1, + "aten::_to_copy": 2, + "aten::add.Tensor": 1, + "aten::add_.Tensor": 3, + "aten::any": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 2, + "aten::mul_.Tensor": 3, + "aten::sub_.Tensor": 2, + "aten::sum.dim_IntList": 1 + }, + "nodes.IFNodes_forward.b1": { + "aten::_to_copy": 1, + "aten::add_.Tensor": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 1, + "aten::mul_.Tensor": 1, + "aten::sub_.Tensor": 1 + }, + "nodes.IFNodes_forward.b4": { + "aten::_to_copy": 1, + "aten::add_.Tensor": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::le.Scalar": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 1, + "aten::mul_.Tensor": 1, + "aten::sub_.Tensor": 1 + }, + "nodes.Input_forward.b1": { + "aten::_to_copy": 1, + "aten::clone": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul_.Tensor": 1 + }, + "nodes.Input_forward.b4": { + "aten::_to_copy": 1, + "aten::clone": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul_.Tensor": 1 + }, + "nodes.IzhikevichNodes_forward.b1": { + "aten::_local_scalar_dense": 1, + "aten::add.Tensor": 7, + "aten::add_.Tensor": 4, + "aten::any": 1, + "aten::cat": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::index.Tensor": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul.Tensor": 11, + "aten::mul_.Tensor": 1, + "aten::pow.Tensor_Scalar": 2, + "aten::select.int": 1, + "aten::sub.Tensor": 3, + "aten::sum.dim_IntList": 1, + "aten::unsqueeze": 1, + "aten::view": 2, + "aten::where.self": 2 + }, + "nodes.IzhikevichNodes_forward.b4": { + "aten::_local_scalar_dense": 1, + "aten::add.Tensor": 7, + "aten::add_.Tensor": 4, + "aten::any": 1, + "aten::cat": 1, + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::index.Tensor": 4, + "aten::masked_fill_.Tensor": 1, + "aten::mul.Tensor": 11, + "aten::mul_.Tensor": 1, + "aten::pow.Tensor_Scalar": 2, + "aten::select.int": 4, + "aten::sub.Tensor": 3, + "aten::sum.dim_IntList": 4, + "aten::unsqueeze": 4, + "aten::view": 2, + "aten::where.self": 2 + }, "nodes.LIFNodes_forward.b1": { "aten::add_.Tensor": 2, "aten::ge.Tensor": 1, @@ -180,6 +409,62 @@ "aten::mul_.Tensor": 2, "aten::sub_.Tensor": 2 }, + "nodes.McCullochPitts_forward.b1": { + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul_.Tensor": 1 + }, + "nodes.McCullochPitts_forward.b4": { + "aten::clone": 1, + "aten::ge.Tensor": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul_.Tensor": 1 + }, + "nodes.Nodes_forward.b1": { + "aten::clone": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul_.Tensor": 1 + }, + "nodes.Nodes_forward.b4": { + "aten::clone": 1, + "aten::masked_fill_.Tensor": 1, + "aten::mul_.Tensor": 1 + }, + "nodes.SRM0Nodes_forward.b1": { + "aten::_to_copy": 1, + "aten::add_.Tensor": 2, + "aten::clone": 1, + "aten::div.Tensor": 1, + "aten::exp": 2, + "aten::le.Scalar": 1, + "aten::lt.Tensor": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 4, + "aten::mul_.Tensor": 2, + "aten::neg": 1, + "aten::rand_like": 1, + "aten::rsub.Scalar": 1, + "aten::sub.Tensor": 1, + "aten::sub_.Tensor": 2 + }, + "nodes.SRM0Nodes_forward.b4": { + "aten::_to_copy": 1, + "aten::add_.Tensor": 2, + "aten::clone": 1, + "aten::div.Tensor": 1, + "aten::exp": 2, + "aten::le.Scalar": 1, + "aten::lt.Tensor": 1, + "aten::masked_fill_.Tensor": 3, + "aten::mul.Tensor": 4, + "aten::mul_.Tensor": 2, + "aten::neg": 1, + "aten::rand_like": 1, + "aten::rsub.Scalar": 1, + "aten::sub.Tensor": 1, + "aten::sub_.Tensor": 2 + }, "rule.DiehlAndCook.b1": { "aten::_local_scalar_dense": 2, "aten::_to_copy": 1, @@ -473,6 +758,13 @@ "date": "2026-10-28", "commit": "926e60dcbe593715424780b403878a9d45d18729+uncommitted bindsnet code changes", "reason": "MCC_learning.MSTDPET and Rmax keep per-sample state ([batch, ...]) so batch > 1 works: bmm instead of outer and a reduction over the batch; at batch 1 mcc.MSTDPET 24 -> 27 and rule.Rmax 24 -> 26 operations; mcc.MSTDPET.b4 and rule.Rmax.b4 counted again" + }, + { + "torch": "2.14.1+cu130", + "python": "3.11.9", + "date": "2026-10-28", + "commit": "fa0847d0091f654bec0256e6e8827c781284627a", + "reason": "Workloads added: one forward() of every Nodes class (only LIFNodes was counted), so an extra operation in any neuron step is caught" } ] } diff --git a/test/regression/test_golden.py b/test/regression/test_golden.py index 5fd5e4e2..584420ee 100644 --- a/test/regression/test_golden.py +++ b/test/regression/test_golden.py @@ -15,6 +15,8 @@ ``python test/regression/regen.py --reason ""`` and commit the new files. """ +import os + import networks as N import pytest import torch @@ -33,7 +35,9 @@ def test_golden(name): golden = N.load_golden(name) made_with = golden["meta"]["torch"] if _torch_minor(made_with) != _torch_minor(torch.__version__): - pytest.skip( + # On CI a torch upgrade must regenerate the goldens in the same PR, + # instead of silently skipping every golden test after it merges. + (pytest.fail if os.environ.get("CI") else pytest.skip)( f"golden files were made with torch {made_with}; " "regenerate with regen --reason" ) diff --git a/test/regression/test_op_counts.py b/test/regression/test_op_counts.py index b0123b20..554a3691 100644 --- a/test/regression/test_op_counts.py +++ b/test/regression/test_op_counts.py @@ -23,6 +23,7 @@ import importlib import inspect import json +import os import warnings from collections import Counter from pathlib import Path @@ -147,6 +148,37 @@ def _lif_workload(batch): return lambda: layer.forward(step) +def _node_classes(): + """Every concrete Nodes class in bindsnet.network.nodes except LIFNodes (which + has its own workload, warmed up through Network.run).""" + import bindsnet.network.nodes as nodes_module + + return { + name: cls + for name, cls in inspect.getmembers(nodes_module, inspect.isclass) + if issubclass(cls, nodes_module.Nodes) + and not inspect.isabstract(cls) + and cls.__module__ == nodes_module.__name__ + and cls is not LIFNodes + } + + +def _node_workload(cls, batch): + """One forward() of ``cls`` after a seeded warm-up of forward() calls.""" + torch.manual_seed(0) + gen = torch.Generator().manual_seed(1) + net = Network(dt=1.0, batch_size=batch) + layer = cls(n=LIF_N, traces=True) + net.add_layer(layer, name="layer") + # CSRMNodes takes a window of inputs, [batch, window, n]. + window = (int(layer.res_window_size),) if hasattr(layer, "res_window_size") else () + x = torch.rand(WARMUP + 1, batch, *window, LIF_N, generator=gen) * 20.0 + for t in range(WARMUP): + layer.forward(x[t].clone()) + step = x[WARMUP] + return lambda: layer.forward(step.clone()) + + def workloads(): """Workload name -> function that builds it and returns the call to count.""" out = {} @@ -160,6 +192,8 @@ def workloads(): out[f"run.DiehlAndCook2015_784x100.b{b}"] = lambda b=b: _dc2015_workload(b) for b in LIF_BATCHES: out[f"nodes.LIFNodes_forward.b{b}"] = lambda b=b: _lif_workload(b) + for name, cls in sorted(_node_classes().items()): + out[f"nodes.{name}_forward.b{b}"] = lambda c=cls, b=b: _node_workload(c, b) return out @@ -187,7 +221,8 @@ def _major_minor(version): def stored(): data = load_counts() if _major_minor(data["meta"]["torch"]) != _major_minor(torch.__version__): - pytest.skip( + # On CI a torch upgrade must regenerate the counts in the same PR. + (pytest.fail if os.environ.get("CI") else pytest.skip)( f"op counts were made with torch {data['meta']['torch']}; regenerate " "with test/regression/regen_op_counts.py --reason" )