diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e84dc293e..7d89f24bf 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -745,6 +745,149 @@ jobs: echo "Fixture-dependent selection: $selection" python -m pytest -q -n auto --dist load $selection + # GitHub issue #981: two test modules + # (tests/protocols/application/test_http_unit.py and + # tests/test_base_class_contract.py) each pass alone, and pytest -- the + # runner every job above uses -- reports the combined run green, while + # plain unittest over the same two modules in one process fails four + # subTest cases. pytest-subtests is not why -- it is not even installed in + # this project (absent from the `test` extra in pyproject.toml), and plain + # pytest already reports a failed subTest as its own top-level SUBFAILED + # entry rather than folding it into a passing parent. The real cause is + # tests/conftest.py's autouse restore_module_table fixture, which + # reconciles the cross-module pcapkit reimport pollution #981 is about -- + # but only under pytest, since plain unittest never loads conftest.py at + # all. Every job above runs under pytest, so that reconciliation is why + # none of them can see this shape of defect -- invisible by construction + # once the fixture is doing its job, not by oversight. This job exists + # solely to make that visible, by using a different test *runner* -- + # unittest.TextTestRunner -- that never loads conftest.py and so never + # gets the reconciliation that hides the defect from pytest. + # + # A cheaper, narrower alternative: `pytest --noconftest` over just those two + # modules, in that order, disables that same autouse fixture directly and + # reproduced the identical four subTest failures in about 70s on the pre-fix + # tree, under the pytest this CI already installs. Whether that single + # invocation would have been sufficient instead of this dedicated job and its + # per-directory matrix was not evaluated when this job was written -- + # recorded here rather than justified after the fact; see + # util/run_unittest_leg.py's own docstring for the fuller version of both + # this and the paragraph above. + # + # util/run_unittest_leg.py (its own docstring has the full reasoning) runs + # one tests/ subdirectory per matrix cell, paired with every root-level + # tests/test_*.py module, directory first and root second -- the order + # #981's own reproduction needs, since the defect is an earlier module + # polluting a later one. Running the whole suite in one process is not an + # option (it OOMs at 29 GB on the machine this was diagnosed on); a matrix + # of one leg per top-level directory is the affordable approximation, and it + # is sized for wall time rather than for memory. Peak RSS was measured for + # the three cheapest legs only -- cli 152 MiB, dumpkit 237 MiB, interface + # 305 MiB (resource.getrusage on RUSAGE_CHILDREN, this venv) -- leaving the + # largest of the three some 90x short of 29 GB; no figure was taken for + # tests/protocols, the largest leg. Measured wall times (this venv, serial): + # cli 18s, const 268s, dumpkit 39s, foundation 272s, interface 38s, project + # 70s, protocols 847s, toolkit 97s, utilities 42s, one leg per matrix cell -- + # see `timeout-minutes` below for why the job-level budget is no longer 20. + # + # Three directories are deliberately absent from ``leg``, for reasons that + # are not this job's to fix: + # + # * tests/corekit -- already known and already accepted, not a new + # finding. tests/_support.py's own purge_modules() docstring measures + # and documents exactly this: "python -m unittest discover -s + # tests/corekit fails 5 of that class's identity subtests, because + # test_module sorts first and purges". pytest is "the supported + # runner" for precisely this reason -- tests/conftest.py's autouse + # restore_module_table fixture reconciles the module-identity drift + # that plain unittest has no way to see coming. Running this leg + # against tests/corekit would be permanently red for a characteristic + # the suite's own documentation already treats as expected, not a + # regression this job would be reporting. + # * tests/vendor -- while sizing this job, running it this way surfaced a + # previously-unknown instance of the *same* class of defect #981 is + # about, in tests/vendor/test_vendor_snapshot_restore_unit.py: several + # earlier-sorting files in that directory purge pcapkit the same + # asymmetric way, and + # test_a_failure_leaves_the_previous_file_byte_for_byte_intact's own + # assertWarnsRegex(VendorRuntimeWarning, ...) ends up checking a stale + # generation of that class against a warning raised under a fresher + # one. Genuine, but a different file than this change touches and + # outside what #981 asks this particular job to fix -- left out so + # this new job lands green against a known and fixed cause, with the + # finding tracked as #985 rather than silently hidden or fixed in + # passing here. Until #985 lands, excluding the directory also means + # the other 13 tests/vendor modules get no unittest-ordering coverage + # at all, not just the one file #985 is about -- that cost is accepted + # for now, not unnoticed. + # * tests/integration -- needs generated captures this script does not + # build, and is already run whole, under pytest, by the `integration` + # job above. The exclusion lives here, in this matrix, and nowhere else: + # util/run_unittest_leg.py does not refuse the directory, it collects + # nothing from it. tests._tiers.is_unit_tier() rejects all 12 of its + # test_*.py files, so leg_modules('integration') returns an empty tuple + # and the leg would run the root-level modules alone -- green, while + # covering none of tests/integration. Adding it back here would buy that + # nothing, not a failure. + # + # Not added to ``required-checks`` below: a brand-new job earns a place in + # the required set once its own track record justifies it, which is the + # maintainer's call to make separately from landing it. + unittest-ordering: + name: Plain unittest ordering (${{ matrix.leg }}) + if: ${{ inputs.gate-only != true }} + runs-on: ubuntu-latest + # 45, not 20: `timeout-minutes` is job-level, so it has to cover + # checkout/setup/install (~39s) as well as the leg itself, and this is + # the only test job in this file with no xdist underneath it, so its + # protocols leg (847s serial, measured above) carries the full cost + # alone. A review pass measured medium legs at 1.22-1.31x that figure on + # a contended machine, putting protocols at 17-18.5 minutes, and one + # contended run of it hit 1275s outright -- over the old 20-minute cap. + # 45 matches the `test`/`integration`/`pypcap-parity` jobs' own cap + # rather than inventing a new number (`engine-tests` is 30, but runs + # under xdist, which this job deliberately does not); splitting + # `protocols` into sub-legs is the alternative if 45 ever stops fitting. + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + leg: + - cli + - const + - dumpkit + - foundation + - interface + - project + - protocols + - toolkit + - utilities + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.14" + cache: pip + + # DPKT/crypto/NGAP join `test` for the same reason the `test` job above + # installs them: this leg's root-level modules and its own directory's + # modules both reach gates that would otherwise skip silently rather + # than run. + - name: Install package and test dependencies + run: | + python -m pip install -U pip setuptools wheel + python -m pip install -e '.[test,DPKT,crypto,NGAP]' + + - name: Report available parallelism + run: | + nproc + python -c "import os; print('cpu_count', os.cpu_count())" + + - name: Run tests/${{ matrix.leg }} and the root-level modules under plain unittest + run: python util/run_unittest_leg.py ${{ matrix.leg }} + # ``CHANGELOG.md`` is generated from the newest entry under # ``docs/source/changelog/`` by ``util/changelog_md.py``, so it falls out of step # the moment an entry is edited without regenerating it. That is worth its own diff --git a/tests/test_base_class_contract.py b/tests/test_base_class_contract.py index ef398484c..c644ae9d0 100644 --- a/tests/test_base_class_contract.py +++ b/tests/test_base_class_contract.py @@ -61,10 +61,41 @@ The invariant is asserted over the *library's own* classes instead, which is what it is actually about and is immune to collection order. + It fires, though, and GitHub issue #981 is the reproduction: + ``tests.protocols.application.test_http_unit`` run before this module, one + process, plain :mod:`unittest`, desyncs + :class:`RegistrationGateTests.test_user_style_subclass_registers_when_it_opts_in` + on all four suites. That test's own ``setUp`` calls + :func:`tests._support.purge_modules` on ``pcapkit`` and re-imports it from + source, which mints a *second generation* of every ``pcapkit`` class -- + deliberately asymmetric, see that function's own docstring, and ordinarily + reconciled straight back by :func:`tests.conftest.restore_module_table`'s + autouse fixture. Plain :mod:`unittest` never loads that fixture, so the + second generation stays live: this module's own top-level ``from + pcapkit... import Engine, EngineBase, ...`` is now bound to the *first* + generation, while ``Dumper.__init_subclass__`` and + ``Extractor.register_engine``/``register_reassembly``/``register_traceflow`` + each re-import their collaborators locally and so see the *second*. A + dynamically-created ``UserOptIn_engines(Engine, engine=...)`` is then a + first-generation class handed to a second-generation + ``issubclass(..., EngineBase)`` check -- which is false, two same-named but + distinct classes -- and the ``dumpers`` suite's registration lands in the + second generation's ``Extractor.__output__`` while + :meth:`RegistrationGateTests.registry` keeps reading the first generation's, + so the key the test just added looks absent. :class:`RegistrationGateTests` + closes this by never trusting its own module-level import for anything it + compares a *live* class against: :meth:`~RegistrationGateTests.setUp` + re-resolves every base/public pair and the ``Extractor`` singleton through + :func:`importlib.import_module` -- a no-op lookup in :data:`sys.modules` + when nothing has reimported, and the current generation when something has + -- so the suite always compares like generation to like, regardless of what + ran before it in the same process. + """ from __future__ import annotations import ast +import importlib import pkgutil import unittest from typing import TYPE_CHECKING @@ -239,21 +270,68 @@ class itself*, so identity is what is asserted, not absence: the name class RegistrationGateTests(unittest.TestCase): """Only a subclass of the public class registers -- pinned, per suite.""" - #: ``(label, base, public, keyword, registry accessor)`` per suite. The - #: protocols suite is absent on purpose: its name registry is - #: ``pcapkit.protocols.__proto__`` and its hook takes no registration - #: keyword of this shape, so it is covered by - #: :meth:`test_library_classes_are_not_descendants_of_the_public_class` and - #: by ``tests/protocols/`` instead. - SUITES = ( - ('engines', EngineBase, Engine, 'engine', 'ENGINE'), - ('reassembly', ReassemblyBase, Reassembly, 'protocol', 'REASSEMBLY'), - ('traceflow', TraceFlowBase, TraceFlow, 'protocol', 'TRACEFLOW'), - ('dumpers', DumperBase, Dumper, 'fmt', 'OUTPUT'), - ) - - @staticmethod - def registry(which: 'str') -> 'dict[str, Any]': + def setUp(self) -> None: + """Re-resolve every base/public pair and the ``Extractor`` singleton, fresh. + + GitHub issue #981: this module's own top-level ``from pcapkit... import + Engine, EngineBase, ...`` binds whatever generation of those classes was + live when *this module* was imported. A sibling test that purges + ``pcapkit`` from :data:`sys.modules` and re-imports it -- + :func:`tests._support.purge_modules`, deliberately asymmetric; see its own + docstring -- mints a new generation that :meth:`make_subclass` and + :meth:`registry` would otherwise silently disagree about, because + ``Dumper.__init_subclass__`` and ``Extractor.register_engine`` / + ``register_reassembly`` / ``register_traceflow`` each re-import their own + collaborators locally and so always see the *current* generation, not the + one this module's import statement captured. + + :func:`tests.conftest.restore_module_table`'s autouse fixture reconciles + the two generations back together after every test, but only under + :program:`pytest` -- plain :mod:`unittest` loads no ``conftest`` at all, so + the mismatch survives into this test. The fix is to never compare this + module's own stale import against something that might be live-generation: + :func:`importlib.import_module` here returns the module straight out of + :data:`sys.modules` when nothing has reimported it (a no-op lookup, no + reload) and the current generation when something has, so every + comparison below is generation-consistent regardless of what ran earlier + in this process. + + """ + engine_mod = importlib.import_module('pcapkit.foundation.engines.engine') + reassembly_mod = importlib.import_module('pcapkit.foundation.reassembly.reassembly') + traceflow_mod = importlib.import_module('pcapkit.foundation.traceflow.traceflow') + dumper_mod = importlib.import_module('pcapkit.dumpkit.common') + protocol_mod = importlib.import_module('pcapkit.protocols.protocol') + self._extractor = importlib.import_module('pcapkit.foundation.extraction').Extractor + + #: ``(label, base, public, keyword, registry accessor)`` per suite, + #: resolved fresh in :meth:`setUp` rather than carried as a class + #: attribute -- see this method's own docstring. The protocols suite is + #: absent on purpose: its name registry is ``pcapkit.protocols.__proto__`` + #: and its hook takes no registration keyword of this shape, so it is + #: covered by :meth:`test_library_classes_are_not_descendants_of_the_public_class` + #: and by ``tests/protocols/`` instead. + self.SUITES = ( + ('engines', engine_mod.EngineBase, engine_mod.Engine, 'engine', 'ENGINE'), + ('reassembly', reassembly_mod.ReassemblyBase, reassembly_mod.Reassembly, + 'protocol', 'REASSEMBLY'), + ('traceflow', traceflow_mod.TraceFlowBase, traceflow_mod.TraceFlow, + 'protocol', 'TRACEFLOW'), + ('dumpers', dumper_mod.DumperBase, dumper_mod.Dumper, 'fmt', 'OUTPUT'), + ) # type: tuple[tuple[str, type, type, str, str], ...] + + #: ``(label, base, public)`` per suite, including ``protocols`` -- + #: :meth:`test_library_classes_are_not_descendants_of_the_public_class`'s + #: own set, resolved the same fresh way for the same reason. + self._descendant_pairs = ( + ('protocols', protocol_mod.ProtocolBase, protocol_mod.Protocol), + ('engines', engine_mod.EngineBase, engine_mod.Engine), + ('reassembly', reassembly_mod.ReassemblyBase, reassembly_mod.Reassembly), + ('traceflow', traceflow_mod.TraceFlowBase, traceflow_mod.TraceFlow), + ('dumpers', dumper_mod.DumperBase, dumper_mod.Dumper), + ) # type: tuple[tuple[str, type, type], ...] + + def registry(self, which: 'str') -> 'dict[str, Any]': """The name-keyed registry for a suite. Args: @@ -264,10 +342,10 @@ def registry(which: 'str') -> 'dict[str, Any]': """ return { - 'ENGINE': Extractor.__engine__, - 'REASSEMBLY': Extractor.__reassembly__, - 'TRACEFLOW': Extractor.__traceflow__, - 'OUTPUT': Extractor.__output__, + 'ENGINE': self._extractor.__engine__, + 'REASSEMBLY': self._extractor.__reassembly__, + 'TRACEFLOW': self._extractor.__traceflow__, + 'OUTPUT': self._extractor.__output__, }[which] def make_subclass(self, name: 'str', base: 'type', **kwargs: 'Any') -> 'type': @@ -347,11 +425,7 @@ def test_library_classes_are_not_descendants_of_the_public_class(self) -> None: the thing to assert. """ - for label, base, public in (('protocols', ProtocolBase, Protocol), - ('engines', EngineBase, Engine), - ('reassembly', ReassemblyBase, Reassembly), - ('traceflow', TraceFlowBase, TraceFlow), - ('dumpers', DumperBase, Dumper)): + for label, base, public in self._descendant_pairs: with self.subTest(suite=label): found, pending = set(), [base] while pending: diff --git a/util/run_unittest_leg.py b/util/run_unittest_leg.py new file mode 100644 index 000000000..e31ff90bc --- /dev/null +++ b/util/run_unittest_leg.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +"""Run one unit-tier directory under plain :mod:`unittest`, in a single process. + +GitHub issue #981: two test modules -- +:mod:`tests.protocols.application.test_http_unit` and +:mod:`tests.test_base_class_contract` -- each pass alone and :program:`pytest` +reports the whole run green when they are collected together, but plain +:mod:`unittest` over the same two modules in one process fails -- four +``subTest`` cases. ``pytest-subtests`` is *not* why: that plugin is not +installed in this project at all (absent from the ``test`` extra in +``pyproject.toml``), and plain :program:`pytest` (9.1.1 in this checkout) +already reports a failed ``subTest`` as its own top-level ``SUBFAILED`` entry +rather than folding it into a passing parent -- confirmed here with a +synthetic two-``subTest`` probe that pytest reported as two separate +failures. The real cause (fixed separately, in +:mod:`tests.test_base_class_contract`) is cross-module ``pcapkit`` reimport +pollution that :mod:`tests.conftest`'s autouse ``restore_module_table`` +fixture reconciles after every test -- but only under :program:`pytest`, +because that fixture lives in a ``conftest.py`` that plain :mod:`unittest` +never loads. Without it, whatever a sibling module's +:func:`tests._support.purge_modules` leaves behind survives into the next +module run in the same process. + +That is this script's whole reason to exist: it is not a faster or stricter +pytest, it is a *different* test runner, chosen because plain +:mod:`unittest` never loads ``tests/conftest.py`` and so gets none of the +reconciliation :func:`tests.conftest.restore_module_table` performs -- the +same reconciliation that, under ordinary pytest, is what let #981's defect +through undetected. A ``subTest`` failure under +:class:`unittest.TextTestRunner` is a top-level ``FAIL``/``ERROR``, counted +and printed, with no autouse fixture smoothing the import table out from +under it. + +A cheaper alternative exists, and is recorded here rather than left +undocumented: running ``pytest --noconftest`` over just those same two +modules, in that order, disables the very same autouse fixture directly and +reproduces the identical four ``subTest`` failures in about 70s on the pre-fix +tree, under the exact :program:`pytest` version this CI already installs. +Whether that single invocation would have been sufficient instead of this +dedicated runner and its per-directory matrix was not evaluated when this +script was written; this paragraph records that gap rather than inventing a +reason for the choice after the fact. + +Scope, and why it stops where it does +-------------------------------------- + +The whole suite in one process is not an option -- it OOMs at 29 GB on the +machine this was diagnosed on. This script instead runs one :file:`tests/` +subdirectory per invocation (its ``directory`` argument), which is what +:file:`.github/workflows/unit-tests.yml`'s ``unittest-ordering`` job calls +once per entry of its own matrix -- the leg list lives in that workflow, not +as a module-level constant here. :mod:`tests.protocols` is the largest leg, +measured in that workflow's own comment at 847s serially. At the other end sit +two legs of two unit-tier modules each, with no clean ordering between them -- +:mod:`tests.dumpkit` (199 tests, ~39s) and :mod:`tests.interface` (201 tests, +~37s), dumpkit carrying two fewer tests but running two seconds slower. +Neither small leg is :mod:`tests.const` (478 tests, ~268s), which is larger on +both axes. + +Peak RSS was measured for the three cheapest legs only, via +:func:`resource.getrusage` on ``RUSAGE_CHILDREN`` in this venv: +:mod:`tests.cli` 152 MiB, :mod:`tests.dumpkit` 237 MiB, :mod:`tests.interface` +305 MiB. No figure was taken for the larger legs, and that workflow comment's +table carries wall times only, no RSS -- but the largest of the three is still +some 90x short of 29 GB, which is enough to place the OOM on running +*everything* in one process rather than on any one directory. + +Every module :func:`root_modules` returns runs in *every* invocation, after the +directory's own modules -- not before, and not interleaved. Issue #981's own +reproduction is a sibling module (``tests.protocols.application.test_http_unit``) +*polluting* a root-level module (:mod:`tests.test_base_class_contract`) that +runs after it in the same process; ``tests.test_base_class_contract`` first and +the directory second would never reproduce that shape, because the pollution +would land after the sensitive module had already made its assertions and +finished. Running the directory first and the root modules second is what gives +any purging module in that directory a chance to desync a root module that +assumes its own import is current -- matching the reproduction exactly for +``protocols`` and ``const``, and giving the same opportunity to every other +directory this script is pointed at, most of which have never been tried in +that configuration before. + +What this still does not catch: a defect running the *other* direction (a +root module polluting a directory module), interference between two +directories neither of which is bundled with the other in the same leg, and +anything that only manifests with the fixture-dependent tier +(:data:`tests._tiers.FIXTURE_TIER_DIRS`) alongside it -- ``tests/integration/`` +is deliberately never one of the ``unittest-ordering`` job's matrix legs +(defined in :file:`.github/workflows/unit-tests.yml`, not in this module), +both because it needs generated captures this script does not build and +because it is already run whole, under :program:`pytest`, by the +``integration`` job. + +""" +from __future__ import annotations + +import argparse +import pathlib +import sys +import time +import unittest + +#: Repository root -- resolved from this file's own location, not from the +#: working directory or ``PYTHONPATH``, so this script runs the same way +#: whether it is invoked as ``python util/run_unittest_leg.py ...`` from the +#: repository root (what CI does) or from anywhere else. +ROOT = pathlib.Path(__file__).resolve().parents[1] +TESTS_ROOT = ROOT / 'tests' + +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from tests._tiers import is_unit_tier # noqa: E402 pylint: disable=wrong-import-position + +#: Root-level modules bundled, in this order, into *every* leg -- see the +#: module docstring for why they run after the directory's own modules rather +#: than before or interleaved. Discovered rather than hand-listed, so a new +#: ``tests/test_*.py`` file is picked up without this script changing. +#: +#: ``tests.test_tier_guard_xdist`` is excluded on purpose: with +#: ``pytest-xdist`` installed (the ``test`` extra pulls it in, and this +#: script's own CI job installs that extra for the other root modules' +#: sake), its ``XdistSubprocessTests`` spawns a real, deliberately slow +#: ``pytest -n auto --dist load`` subprocess that the pytest-based +#: ``test``/``integration``/``gate`` jobs already exercise -- duplicating +#: that cost here would buy nothing against the ordering class of defect +#: this script exists to catch. +_EXCLUDED_ROOT_MODULES = frozenset({'tests.test_tier_guard_xdist'}) + + +def _dotted(path: 'pathlib.Path') -> 'str': + """``path`` as the dotted module name :class:`unittest.TestLoader` wants.""" + return '.'.join(path.relative_to(ROOT).with_suffix('').parts) + + +def root_modules() -> 'tuple[str, ...]': + """Every root-level ``tests/test_*.py`` module but the excluded one.""" + return tuple(sorted( + name for name in (_dotted(path) for path in sorted(TESTS_ROOT.glob('test_*.py'))) + if name not in _EXCLUDED_ROOT_MODULES + )) + + +def leg_modules(directory: 'str') -> 'tuple[str, ...]': + """Every unit-tier ``test_*.py`` module under ``tests/``. + + Args: + directory: name of a direct subdirectory of :data:`TESTS_ROOT`. + + Returns: + Dotted module names, in path-sorted order. + + Raises: + SystemExit: ``directory`` is not a subdirectory of :data:`TESTS_ROOT`. + + """ + leg_root = TESTS_ROOT / directory + if not leg_root.is_dir(): + raise SystemExit(f'no such tests/ subdirectory: tests/{directory}') + + return tuple( + _dotted(path) for path in sorted(leg_root.rglob('test_*.py')) + if is_unit_tier(path) + ) + + +def build_suite(directory: 'str') -> 'unittest.TestSuite': + """The combined suite for one leg: ``directory``'s own tests, then root's. + + See the module docstring for why that order, not the reverse. + + """ + loader = unittest.TestLoader() + suite = unittest.TestSuite() + for name in leg_modules(directory): + suite.addTests(loader.loadTestsFromName(name)) + for name in root_modules(): + suite.addTests(loader.loadTestsFromName(name)) + return suite + + +def main(argv: 'list[str] | None' = None) -> 'int': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + 'directory', + help="a direct subdirectory of tests/ to run alongside the root-level " + "modules, e.g. 'protocols'", + ) + parser.add_argument( + '-v', '--verbose', action='store_true', + help='pass verbosity 2 to unittest.TextTestRunner instead of the default 1', + ) + args = parser.parse_args(argv) + + start = time.monotonic() + suite = build_suite(args.directory) + runner = unittest.TextTestRunner(verbosity=2 if args.verbose else 1) + result: 'unittest.TestResult | None' = None + try: + result = runner.run(suite) + return 0 if result.wasSuccessful() else 1 + finally: + elapsed = time.monotonic() - start + # In a ``finally`` so a leg that *fails*, and a leg that unwinds on an + # exception -- including the ``KeyboardInterrupt`` CPython's default + # SIGINT handler raises, which :class:`unittest.case._Outcome`'s + # ``testPartExecutor`` re-raises rather than swallowing -- still + # reports its elapsed time. + tally = ('interrupted before a result was available' if result is None else + f'{result.testsRun} test(s), {len(result.failures)} failure(s), ' + f'{len(result.errors)} error(s)') + print(f'tests/{args.directory} + {len(root_modules())} root module(s): ' + f'{tally}, {elapsed:.1f}s elapsed') + + +if __name__ == '__main__': + sys.exit(main())