diff --git a/.github/.release-please-config.json b/.github/.release-please-config.json index f001082e..2af75de6 100644 --- a/.github/.release-please-config.json +++ b/.github/.release-please-config.json @@ -41,6 +41,12 @@ "include-component-in-tag": true, "changelog-path": "CHANGELOG.md" }, + "plugin/skip": { + "package-name": "testo/skip", + "component": "skip", + "include-component-in-tag": true, + "changelog-path": "CHANGELOG.md" + }, "plugin/test": { "package-name": "testo/test", "component": "test", diff --git a/.github/workflows/split-publish.yml b/.github/workflows/split-publish.yml index d6bdea0d..3adfd878 100644 --- a/.github/workflows/split-publish.yml +++ b/.github/workflows/split-publish.yml @@ -34,6 +34,7 @@ on: # yamllint disable-line rule:truthy - 'lifecycle-[0-9]*' - 'repeat-[0-9]*' - 'retry-[0-9]*' + - 'skip-[0-9]*' - 'test-[0-9]*' name: 📦 Split publish diff --git a/composer.json b/composer.json index 652996fd..88ae398b 100644 --- a/composer.json +++ b/composer.json @@ -50,6 +50,7 @@ "testo/lifecycle": "^0.1.6", "testo/repeat": "^0.1.9", "testo/retry": "^0.1.5", + "testo/skip": "0.1 - 1", "testo/test": "^0.1.7", "yiisoft/injector": "^1.2" }, @@ -104,6 +105,7 @@ "Tests\\Lifecycle\\": "plugin/lifecycle/tests/", "Tests\\Repeat\\": "plugin/repeat/tests/", "Tests\\Retry\\": "plugin/retry/tests/", + "Tests\\Skip\\": "plugin/skip/tests/", "Tests\\Test\\": "plugin/test/tests/" }, "files": [ @@ -129,6 +131,7 @@ "testo/lifecycle": "0.1.x-dev", "testo/repeat": "0.1.x-dev", "testo/retry": "0.1.x-dev", + "testo/skip": "0.1.x-dev", "testo/test": "0.1.x-dev" } } diff --git a/core/Pipeline/Attribute/CaseInterceptable.php b/core/Pipeline/Attribute/CaseInterceptable.php new file mode 100644 index 00000000..eca73826 --- /dev/null +++ b/core/Pipeline/Attribute/CaseInterceptable.php @@ -0,0 +1,21 @@ +definition->reflection === null + $classAttributes = $info->definition->reflection === null ? [] : Reflection::fetchClassAttributes( class: $info->definition->reflection, @@ -92,22 +96,39 @@ class: $info->definition->reflection, flags: \ReflectionAttribute::IS_INSTANCEOF, ); - if ($attrs === []) { + # Test-level attributes join the case pipeline only when they ask for it explicitly. + $testAttributes = []; + foreach ($info->definition->tests->getTests() as $definition) { + $testAttributes = [...$testAttributes, ...Reflection::fetchFunctionAttributes( + function: $definition->reflection, + attributeClass: CaseInterceptable::class, + flags: \ReflectionAttribute::IS_INSTANCEOF, + )]; + } + + if ($classAttributes === [] && $testAttributes === []) { # No attributes, continue to next interceptor return $next($info); } - $attrs = \array_map( + $instantiate = static fn(array $attrs): array => \array_values(\array_map( static function (\ReflectionAttribute $a): Interceptable { /** @var Interceptable */ return $a->newInstance(); }, $attrs, - ); + )); + $classAttributes = $instantiate($classAttributes); + $testAttributes = $instantiate($testAttributes); # Merge and instantiate attributes - $interceptors = $this->interceptorProvider->fromAttributes(TestCaseRunInterceptor::class, ...$attrs); - $info = $info->withAttributes(self::groupAttributes($attrs)); + $interceptors = $this->interceptorProvider->fromAttributes( + TestCaseRunInterceptor::class, + ...$classAttributes, + ...$testAttributes, + ); + # Only class attributes describe the case itself; test attributes stay on their tests. + $classAttributes === [] or $info = $info->withAttributes(self::groupAttributes($classAttributes)); /** @var callable(CaseInfo): CaseResult $pipeline */ $pipeline = $next instanceof Pipeline diff --git a/plugin/lifecycle/tests/Feature/FullySkippedCaseFeatureTest.php b/plugin/lifecycle/tests/Feature/FullySkippedCaseFeatureTest.php new file mode 100644 index 00000000..a07798ed --- /dev/null +++ b/plugin/lifecycle/tests/Feature/FullySkippedCaseFeatureTest.php @@ -0,0 +1,82 @@ +status, Status::Skipped); + Assert::same(FullySkippedFunctionState::$beforeClassCalls - $beforeClass, 1); + Assert::same(FullySkippedFunctionState::$afterClassCalls - $afterClass, 1); + # No test of the case ran, so the per-test hooks never fired. + Assert::same(FullySkippedFunctionState::$beforeTestCalls - $beforeTest, 0); + Assert::same(FullySkippedFunctionState::$afterTestCalls - $afterTest, 0); + } + + /** + * The class-based analog: hooks are the non-tests prefilled from the case's class reflection + * and must keep firing for a fully skipped class exactly as before. + */ + public function classHooksRunForFullySkippedClassCase(): void + { + $beforeClass = FullySkippedClassStub::$beforeClassCalls; + $afterClass = FullySkippedClassStub::$afterClassCalls; + + $result = TestRunner::runTest([FullySkippedClassStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::same(FullySkippedClassStub::$beforeClassCalls - $beforeClass, 1); + Assert::same(FullySkippedClassStub::$afterClassCalls - $afterClass, 1); + } +} diff --git a/plugin/lifecycle/tests/Stub/FullySkipped/FullySkippedClassStub.php b/plugin/lifecycle/tests/Stub/FullySkipped/FullySkippedClassStub.php new file mode 100644 index 00000000..0839ca2e --- /dev/null +++ b/plugin/lifecycle/tests/Stub/FullySkipped/FullySkippedClassStub.php @@ -0,0 +1,43 @@ + + TESTO +

+ +

Skip attribute plugin

+ +
+ +[![Documentation](https://img.shields.io/badge/Documentation-blue?style=for-the-badge&logo=gitbook&logoColor=white)](https://php-testo.github.io) +[![Support on Boosty](https://img.shields.io/static/v1?style=for-the-badge&label=&message=Sponsorship&logo=Boosty&logoColor=white&color=%23F15F2C)](https://boosty.to/roxblnfk) + +
+ +
+ +> [!IMPORTANT] +> ## 🪞 This is a read-only mirror. +> +> Active development of the Testo project lives in [**php-testo/testo**](https://github.com/php-testo/testo) under `plugin/skip/`. This repository is **automatically synchronized** from there on every release. +> +> File issues and pull requests in the [main monorepo](https://github.com/php-testo/testo/issues), not here. + +## About + +Marks a test, a test class or a test function as skipped without deleting or hiding it. The test is not executed, but stays in every report as Skipped with its reason, so parked tests remain visible until someone returns to them. + +The skip is declared ahead of time, next to the test; skipping at runtime from the test body is covered by the core `SkipTest` exception instead. + +## Install + +```bash +composer require --dev testo/skip +``` + +[![PHP](https://img.shields.io/packagist/php-v/testo/skip.svg?style=flat-square&logo=php)](https://packagist.org/packages/testo/skip) +[![Latest Version on Packagist](https://img.shields.io/packagist/v/testo/skip.svg?style=flat-square&logo=packagist)](https://packagist.org/packages/testo/skip) +[![License](https://img.shields.io/packagist/l/testo/skip.svg?style=flat-square)](https://github.com/php-testo/testo/blob/1.x/LICENSE.md) +[![Total Downloads](https://img.shields.io/packagist/dt/testo/skip.svg?style=flat-square)](https://packagist.org/packages/testo/skip/stats) diff --git a/plugin/skip/Skip.php b/plugin/skip/Skip.php new file mode 100644 index 00000000..4af6bde4 --- /dev/null +++ b/plugin/skip/Skip.php @@ -0,0 +1,67 @@ + {reason}` when + * a reason is given. The JUnit, TeamCity and HTML reporters show it; the terminal does not. + * + * Runtime contract: + * + * - The skipped test never enters the per-test pipeline: `#[BeforeTest]`/`#[AfterTest]`, + * data providers, `#[Retry]`/`#[Repeat]`, fibers and coverage never engage. A data-driven + * test yields a single Skipped entry. + * - `#[BeforeClass]`/`#[AfterClass]` still run, also when every test of the case is skipped. + * - The case class is not constructed for a skipped test. + * - A run consisting only of skipped tests is successful (exit code 0). + * - The attribute is inert on a non-test method and on `#[Bench]`/`#[TestInline]` targets. + * + * No registration is needed: the attribute wires {@see SkipInterceptor} itself, from a class, a + * method or a function alike. + * + * For skipping at runtime — from the test body, based on the environment — throw {@see SkipTest} + * instead; the `is skipped via #[Skip]` marker tells the two apart in reports. + * + * @api + */ +#[\Attribute(\Attribute::TARGET_CLASS | \Attribute::TARGET_METHOD | \Attribute::TARGET_FUNCTION)] +#[FallbackInterceptor(SkipInterceptor::class)] +final readonly class Skip implements CaseInterceptable +{ + /** + * @param string $reason Why the test is skipped. A reference to an issue + * (`'flaky on CI, see ISSUE-123'`) keeps the skip reviewable. + */ + public function __construct( + public string $reason = '', + ) {} +} diff --git a/plugin/skip/composer.json b/plugin/skip/composer.json new file mode 100644 index 00000000..0b3ef9c6 --- /dev/null +++ b/plugin/skip/composer.json @@ -0,0 +1,42 @@ +{ + "name": "testo/skip", + "description": "Skip attribute plugin for the Testo testing framework.", + "license": "BSD-3-Clause", + "type": "library", + "keywords": [ + "testo", + "skip", + "test" + ], + "authors": [ + { + "name": "Aleksei Gagarin (roxblnfk)", + "homepage": "https://github.com/roxblnfk" + } + ], + "funding": [ + { + "type": "boosty", + "url": "https://boosty.to/roxblnfk" + } + ], + "require": { + "php": ">=8.2", + "testo/testo": "0.11 - 1" + }, + "autoload": { + "psr-4": { + "Testo\\Skip\\": "src/" + }, + "files": [ + "Skip.php" + ] + }, + "minimum-stability": "dev", + "prefer-stable": true, + "extra": { + "branch-alias": { + "dev-1.x": "1.x-dev" + } + } +} diff --git a/plugin/skip/src/Internal/SkipInterceptor.php b/plugin/skip/src/Internal/SkipInterceptor.php new file mode 100644 index 00000000..0e69f877 --- /dev/null +++ b/plugin/skip/src/Internal/SkipInterceptor.php @@ -0,0 +1,159 @@ +findSkipped($info); + + if ($skipped === []) { + return $next($info); + } + + # Deactivated, not discarded: the definitions are shared, and the synthetic results below + # are the only delivery of these tests. + foreach ($skipped as [$definition, $_]) { + $definition->active = false; + } + + $inner = $info->batchRunner; + return $next($info->withBatchRunner( + function (array $handlers) use ($inner, $info, $skipped): array { + $results = $inner === null + ? \array_map(static fn(callable $handler): TestResult => $handler(), $handlers) + : $inner($handlers); + + foreach ($skipped as $name => [$definition, $attribute]) { + $results[] = $this->reportSkipped($info, $name, $definition, $attribute); + } + + return $results; + }, + )); + } + + /** + * `{testId} is skipped via #[Skip]`, extended with ` ==> {reason}` when a reason is given. + * The test id is the string `--filter` takes back. + */ + private static function reason(TestInfo $info, Skip $attribute): string + { + $message = "{$info->identity->fqn()} is skipped via #[Skip]"; + + return $attribute->reason === '' ? $message : "{$message} ==> {$attribute->reason}"; + } + + /** + * A method/function-level `#[Skip]` wins over the class-level one; the class-level attribute + * is inherited from parents and traits. + * + * @return array + */ + private function findSkipped(CaseInfo $info): array + { + $classAttribute = null; + $reflection = $info->definition->reflection; + if ($reflection !== null) { + $attributes = Reflection::fetchClassAttributes($reflection, attributeClass: Skip::class, limit: 1); + $attributes === [] or $classAttribute = $attributes[0]->newInstance(); + } + + $skipped = []; + # Active tests only: a test deactivated by --filter/--group is not part of this run and + # must not resurface as Skipped. + foreach ($info->definition->tests->getTests() as $name => $definition) { + $attributes = Reflection::fetchFunctionAttributes( + $definition->reflection, + attributeClass: Skip::class, + limit: 1, + ); + $attribute = $attributes === [] ? $classAttribute : $attributes[0]->newInstance(); + + $attribute === null or $skipped[$name] = [$definition, $attribute]; + } + + return $skipped; + } + + private function reportSkipped( + CaseInfo $case, + string $name, + TestDefinition $definition, + Skip $attribute, + ): TestResult { + $testInfo = (new TestInfo(name: $name, caseInfo: $case, testDefinition: $definition)) + ->withAttributes([Skip::class => [$attribute]]); + + $this->eventDispatcher->dispatch(new TestPipelineStarting($testInfo)); + + $result = new TestResult( + info: $testInfo, + status: Status::Skipped, + failure: new SkipTest(self::reason($testInfo, $attribute)), + attributes: ['duration' => 0, 'description' => $definition->getDescription()], + summary: Summary::forTest(Status::Skipped), + ); + + $this->eventDispatcher->dispatch(new TestPipelineFinished($testInfo, $result)); + + return $result; + } +} diff --git a/plugin/skip/tests/Feature/SkipFallbackStandaloneTest.php b/plugin/skip/tests/Feature/SkipFallbackStandaloneTest.php new file mode 100644 index 00000000..f2d1846d --- /dev/null +++ b/plugin/skip/tests/Feature/SkipFallbackStandaloneTest.php @@ -0,0 +1,70 @@ +with(new NamingConventionPlugin()), + ), + ], + ))->run(); + + /** @var array $tests */ + $tests = []; + foreach ($run as $suite) { + foreach ($suite as $case) { + foreach ($case as $test) { + $tests[$test->info->name] = $test; + } + } + } + + Assert::count($tests, 2); + Assert::true(StandaloneSkippedTest::$enabledRan); + Assert::same($tests['testEnabled']->status, Status::Passed); + + $skipped = $tests['testSkipped']; + Assert::same($skipped->status, Status::Skipped); + Assert::instanceOf($skipped->failure, SkipTest::class); + Assert::same( + $skipped->failure->getMessage(), + StandaloneSkippedTest::class . '::testSkipped is skipped via #[Skip] ==> standalone method is skipped', + ); + } +} diff --git a/plugin/skip/tests/Feature/SkipFeatureTest.php b/plugin/skip/tests/Feature/SkipFeatureTest.php new file mode 100644 index 00000000..be53a868 --- /dev/null +++ b/plugin/skip/tests/Feature/SkipFeatureTest.php @@ -0,0 +1,321 @@ +status, Status::Skipped); + Assert::instanceOf($result->failure, SkipTest::class); + Assert::same( + $result->failure?->getMessage(), + SkipMethodStub::class . '::skipped is skipped via #[Skip] ==> broken by the pricing rework, see ISSUE-123', + ); + } + + public function emptyReasonFallsBackToGeneratedMessage(): void + { + $result = TestRunner::runTest([SkipMethodStub::class, 'skippedNoReason']); + + Assert::same($result->status, Status::Skipped); + Assert::same( + $result->failure?->getMessage(), + SkipMethodStub::class . '::skippedNoReason is skipped via #[Skip]', + ); + } + + public function controlNeighborNextToSkippedTestsStillRuns(): void + { + $result = TestRunner::runTest([SkipMethodStub::class, 'enabled']); + + Assert::same($result->status, Status::Passed); + } + + public function classLevelSkipSkipsEveryTestWithClassReason(): void + { + $first = TestRunner::runTest([SkipClassLevelStub::class, 'firstSkipped']); + $second = TestRunner::runTest([SkipClassLevelStub::class, 'secondSkipped']); + + Assert::same($first->status, Status::Skipped); + Assert::same($second->status, Status::Skipped); + Assert::true(\str_ends_with((string) $first->failure?->getMessage(), ' ==> the whole case is skipped')); + Assert::true(\str_ends_with((string) $second->failure?->getMessage(), ' ==> the whole case is skipped')); + } + + public function methodReasonWinsOverClassReason(): void + { + $own = TestRunner::runTest([SkipClassAndMethodStub::class, 'ownReason']); + $inherited = TestRunner::runTest([SkipClassAndMethodStub::class, 'classReason']); + + Assert::true(\str_ends_with((string) $own->failure?->getMessage(), ' ==> method-specific reason')); + Assert::true(\str_ends_with((string) $inherited->failure?->getMessage(), ' ==> class-wide reason')); + } + + /** + * The method-level attribute wins as a whole: an empty method reason is not filled in + * from the class reason. + */ + public function emptyMethodReasonStillWinsOverClassReason(): void + { + $result = TestRunner::runTest([SkipClassAndMethodStub::class, 'emptyOwnReason']); + + Assert::same($result->status, Status::Skipped); + Assert::same( + $result->failure?->getMessage(), + SkipClassAndMethodStub::class . '::emptyOwnReason is skipped via #[Skip]', + ); + } + + public function functionalTestUsesFunctionFqnInMessage(): void + { + $result = TestRunner::runTest('Tests\Skip\Stub\Skip\skippedFunction'); + + Assert::same($result->status, Status::Skipped); + Assert::same( + $result->failure?->getMessage(), + 'Tests\Skip\Stub\Skip\skippedFunction is skipped via #[Skip] ==> functional test is skipped', + ); + } + + /** + * The function-based analog of the control neighbor: an enabled function of a partially + * skipped file still runs through the batch runner the interceptor installs on the case, and + * passes. + */ + public function controlNeighborFunctionNextToSkippedFunctionStillRuns(): void + { + $result = TestRunner::runTest('Tests\Skip\Stub\Skip\enabledFunction'); + + Assert::same($result->status, Status::Passed); + } + + /** + * The origin contract for downstream consumers: a result skipped by `#[Skip]` carries the + * attribute instances in `$result->info`, unlike a runtime `throw SkipTest` skip. + */ + public function skippedResultCarriesOriginAttribute(): void + { + $result = TestRunner::runTest([SkipMethodStub::class, 'skipped']); + + $origin = $result->info->getAttribute(Skip::class); + Assert::array($origin)->hasCount(1); + Assert::instanceOf($origin[0], Skip::class); + } + + /** + * The skipped test is filtered out before the case runs: class-level hooks fire as usual + * (once per directory run), per-test hooks fire only for the enabled control test. + */ + public function classHooksRunButTestHooksDoNot(): void + { + $beforeClass = SkipWithHooksStub::$beforeClassCalls; + $afterClass = SkipWithHooksStub::$afterClassCalls; + $beforeTest = SkipWithHooksStub::$beforeTestCalls; + $afterTest = SkipWithHooksStub::$afterTestCalls; + + $result = TestRunner::runTest([SkipWithHooksStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::same(SkipWithHooksStub::$beforeClassCalls - $beforeClass, 1); + Assert::same(SkipWithHooksStub::$afterClassCalls - $afterClass, 1); + # Only the enabled control test of the case went through the per-test pipeline. + Assert::same(SkipWithHooksStub::$beforeTestCalls - $beforeTest, 1); + Assert::same(SkipWithHooksStub::$afterTestCalls - $afterTest, 1); + } + + public function fullySkippedCaseWithoutHooksIsNeverInstantiated(): void + { + $result = TestRunner::runTest([SkipConstructorSpyStub::class, 'firstSkipped']); + + Assert::same($result->status, Status::Skipped); + Assert::false(SkipConstructorSpyStub::$constructed); + } + + /** + * Documented caveat: a non-static class-level hook builds the class even when every + * test is skipped — pinned so a future change is conscious, not accidental. + */ + public function nonStaticClassHookStillBuildsTheClass(): void + { + $constructions = SkipNonStaticHookStub::$constructions; + + $result = TestRunner::runTest([SkipNonStaticHookStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::same(SkipNonStaticHookStub::$constructions - $constructions, 1); + } + + public function classLevelSkipIsInheritedFromParent(): void + { + $result = TestRunner::runTest([SkipChildStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::true(\str_ends_with((string) $result->failure?->getMessage(), ' ==> inherited from the parent class')); + } + + public function classLevelSkipIsInheritedFromTrait(): void + { + $result = TestRunner::runTest([SkipTraitStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::true(\str_ends_with((string) $result->failure?->getMessage(), ' ==> inherited from the trait')); + } + + /** + * A method-level `#[Skip]` follows the prototype chain like `#[Group]` does: an overriding + * method without the attribute is still skipped, with the parent's reason. + */ + public function methodLevelSkipIsInheritedByOverridingMethod(): void + { + $result = TestRunner::runTest([SkipOverridingMethodStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::true(\str_ends_with((string) $result->failure?->getMessage(), ' ==> inherited from the overridden method')); + } + + /** + * A data-driven skipped test yields a single Skipped node: the provider is never called + * (not once across all directory runs of this class), no `MultipleResult` aggregate is + * attached. + */ + public function dataProviderIsNotCalledForSkippedTest(): void + { + $result = TestRunner::runTest([SkipWithDataProviderStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::null($result->getAttribute(MultipleResult::class)); + Assert::same(SkipWithDataProviderStub::$providerCalls, 0); + } + + /** + * The positive control on the enabled neighbor proves that `#[Retry]` does engage in this + * run — its first attempt fails and the second passes — so the zero on the skipped test is + * the skip at work, not a retry plugin that never ran. + */ + public function retryDoesNotEngageForSkippedTest(): void + { + $attempts = SkipWithRetryStub::$attempts; + $enabledAttempts = SkipWithRetryStub::$enabledAttempts; + + $result = TestRunner::runTest([SkipWithRetryStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::same(SkipWithRetryStub::$attempts - $attempts, 0); + Assert::same(SkipWithRetryStub::$enabledAttempts - $enabledAttempts, 2); + } + + /** + * Same shape for `#[Repeat]`: the enabled neighbor runs all three of its repetitions, the + * skipped test not even once. + */ + public function repeatDoesNotEngageForSkippedTest(): void + { + $enabledRuns = SkipWithRepeatStub::$enabledRuns; + + $result = TestRunner::runTest([SkipWithRepeatStub::class, 'skipped']); + + Assert::same($result->status, Status::Skipped); + Assert::false(SkipWithRepeatStub::$bodyRan); + Assert::same(SkipWithRepeatStub::$enabledRuns - $enabledRuns, 3); + } + + /** + * The common ground of the hook/provider/retry/repeat checks above: a skipped test never + * enters the per-test pipeline at all. A spy interceptor on that pipeline sees the + * enabled neighbors of the directory and none of the skipped tests. + */ + public function skippedTestsNeverEnterThePerTestPipeline(): void + { + $offset = \count(PipelineEntrySpyPlugin::$entered); + + TestRunner::runTest([SkipMethodStub::class, 'skipped']); + + $entered = \array_slice(PipelineEntrySpyPlugin::$entered, $offset); + Assert::array($entered) + ->contains(SkipMethodStub::class . '::enabled') + ->contains('Tests\Skip\Stub\Skip\enabledFunction') + ->notContains(SkipMethodStub::class . '::skipped') + ->notContains(SkipMethodStub::class . '::skippedNoReason') + ->notContains(SkipWithHooksStub::class . '::skipped') + ->notContains(SkipWithDataProviderStub::class . '::skipped') + ->notContains(SkipWithRetryStub::class . '::skipped') + ->notContains(SkipWithRepeatStub::class . '::skipped') + ->notContains(SkipInFiberStub::class . '::skipped') + ->notContains(SkipOverridingMethodStub::class . '::skipped') + ->notContains('Tests\Skip\Stub\Skip\skippedFunction'); + } + + /** + * Fiber compatibility: the skip interceptor wraps the fiber batch runner instead of + * replacing it. The round-robin interleaving of the two enabled tests is produced only by + * the case scheduler — run sequentially, their `\Fiber::suspend()` would throw and the + * log would stop short — while the skipped test is still skipped. + */ + public function fiberBatchRunnerSurvivesTheWrap(): void + { + $offset = \count(SkipInFiberStub::$log); + + $skipped = TestRunner::runTest([SkipInFiberStub::class, 'skipped']); + + Assert::same($skipped->status, Status::Skipped); + Assert::same( + \array_slice(SkipInFiberStub::$log, $offset), + ['first.1', 'second.1', 'first.2', 'second.2'], + ); + } +} diff --git a/plugin/skip/tests/Feature/SkipSummaryTest.php b/plugin/skip/tests/Feature/SkipSummaryTest.php new file mode 100644 index 00000000..9c193f42 --- /dev/null +++ b/plugin/skip/tests/Feature/SkipSummaryTest.php @@ -0,0 +1,91 @@ +summary; + Assert::same($summary->count(Status::Passed), 1); + Assert::same($summary->count(Status::Failed), 1); + Assert::same($summary->count(Status::Skipped), 2); + # Four tests total: the skipped data-driven one is counted once, not once per data set. + Assert::same($summary->total(), 4); + Assert::same($run->status, Status::Failed); + } + + /** + * A run consisting only of {@see Skip}-marked tests is a success: {@see Status::Skipped} + * is neither a success nor a failure, so nothing fails the run. + * + * The same run pins one result per skipped test. Every `#[Skip]` occurrence of the case + * spawns its own {@see SkipInterceptor} through the fallback alias; a second delivery would + * show up here as an inflated total and an extra name. + */ + public function runOfOnlySkippedTestsIsSuccessfulAndDeliveredOnce(): void + { + $run = self::run(__DIR__ . '/../Stub/SkipSummary/OnlySkipped'); + + Assert::same($run->status, Status::Passed); + Assert::same($run->summary->count(Status::Skipped), 2); + Assert::same($run->summary->total(), 2); + $cases = []; + foreach ($run as $suite) { + foreach ($suite as $case) { + $cases[] = $case; + } + } + # The directory holds one class with two skipped tests. + Assert::count($cases, 1); + $names = \array_map( + static fn(TestResult $result): string => $result->info->name, + \iterator_to_array($cases[0], preserve_keys: false), + ); + \sort($names); + Assert::same($names, ['firstSkipped', 'secondSkipped']); + } + + private static function run(string $path): RunResult + { + return Application::createFromConfig(new ApplicationConfig( + src: [], + suites: [ + new SuiteConfig( + 'SkipSummary', + location: new FinderConfig(include: [$path]), + ), + ], + ))->run(); + } +} diff --git a/plugin/skip/tests/Stub/PipelineEntrySpyPlugin.php b/plugin/skip/tests/Stub/PipelineEntrySpyPlugin.php new file mode 100644 index 00000000..2b493ef5 --- /dev/null +++ b/plugin/skip/tests/Stub/PipelineEntrySpyPlugin.php @@ -0,0 +1,42 @@ + */ + public static array $entered = []; + + #[\Override] + public function configure(Container $container): void + { + $container->get(InterceptorCollector::class)->addInterceptor( + new class implements TestRunInterceptor { + #[\Override] + public function runTest(TestInfo $info, callable $next): TestResult + { + PipelineEntrySpyPlugin::$entered[] = $info->identity->fqn(); + + return $next($info); + } + }, + ); + } +} diff --git a/plugin/skip/tests/Stub/Skip/SkipChildStub.php b/plugin/skip/tests/Stub/Skip/SkipChildStub.php new file mode 100644 index 00000000..7d2de11a --- /dev/null +++ b/plugin/skip/tests/Stub/Skip/SkipChildStub.php @@ -0,0 +1,19 @@ + */ + public static array $log = []; + + #[Skip('skipped inside a fiber-driven case')] + public function skipped(): void + { + throw new \LogicException('Must never run: the test is skipped.'); + } + + public function first(): void + { + self::$log[] = 'first.1'; + \Fiber::suspend(); + self::$log[] = 'first.2'; + + # Round-robin: after the yield, "second" has had its first step in between. + Assert::same(\array_slice(self::$log, -3), ['first.1', 'second.1', 'first.2']); + } + + public function second(): void + { + self::$log[] = 'second.1'; + \Fiber::suspend(); + self::$log[] = 'second.2'; + + Assert::same(\array_slice(self::$log, -4), ['first.1', 'second.1', 'first.2', 'second.2']); + } +} diff --git a/plugin/skip/tests/Stub/Skip/SkipMarkerTrait.php b/plugin/skip/tests/Stub/Skip/SkipMarkerTrait.php new file mode 100644 index 00000000..a373127c --- /dev/null +++ b/plugin/skip/tests/Stub/Skip/SkipMarkerTrait.php @@ -0,0 +1,13 @@ + + */ + public static function provide(): array + { + ++self::$providerCalls; + + return [ + 'one' => [1], + 'two' => [2], + ]; + } + + #[Skip('data-driven test is skipped as a whole')] + #[DataProvider('provide')] + public function skipped(int $value): void + { + throw new \LogicException('Must never run: the test is skipped.'); + } +} diff --git a/plugin/skip/tests/Stub/Skip/SkipWithHooksStub.php b/plugin/skip/tests/Stub/Skip/SkipWithHooksStub.php new file mode 100644 index 00000000..ebda9cb3 --- /dev/null +++ b/plugin/skip/tests/Stub/Skip/SkipWithHooksStub.php @@ -0,0 +1,67 @@ +firstAttemptFailed) { + $this->firstAttemptFailed = true; + throw new \RuntimeException('First attempt fails by design.'); + } + } +} diff --git a/plugin/skip/tests/Stub/Skip/skip_functions.php b/plugin/skip/tests/Stub/Skip/skip_functions.php new file mode 100644 index 00000000..083c59bd --- /dev/null +++ b/plugin/skip/tests/Stub/Skip/skip_functions.php @@ -0,0 +1,26 @@ + + */ + public static function provide(): iterable + { + yield [1]; + yield [2]; + } + + public function passes(): void + { + Assert::true(true); + } + + public function fails(): void + { + # Controlled failure: the skipped tests must not hide it from the totals. + Assert::true(false); + } + + #[Skip('skipped in the mixed case')] + public function skipped(): void + { + throw new \LogicException('Must never run: the test is skipped.'); + } + + #[Skip('data-driven test skipped as a whole')] + #[DataProvider('provide')] + public function skippedDataDriven(int $value): void + { + throw new \LogicException('Must never run: the test is skipped.'); + } +} diff --git a/plugin/skip/tests/Stub/SkipSummary/OnlySkipped/OnlySkippedStub.php b/plugin/skip/tests/Stub/SkipSummary/OnlySkipped/OnlySkippedStub.php new file mode 100644 index 00000000..b979159f --- /dev/null +++ b/plugin/skip/tests/Stub/SkipSummary/OnlySkipped/OnlySkippedStub.php @@ -0,0 +1,26 @@ +runTestCase($info, self::coreNext($seenTests)); + + Assert::same($seenTests, ['enabled']); + } + + /** + * The skipped tests still come back in the case result — as synthetic Skipped results + * with a SkipTest failure and their own `Summary::forTest(Status::Skipped)`, since no core + * runner produces one for them. + */ + public function returnsSyntheticSkippedResults(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped', 'enabled'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + $skipped = self::findResult($result, 'skipped'); + Assert::same($skipped->status, Status::Skipped); + Assert::instanceOf($skipped->failure, SkipTest::class); + Assert::same($skipped->summary->count(Status::Skipped), 1); + Assert::same($result->summary->count(Status::Skipped), 1); + Assert::same($result->summary->count(Status::Passed), 1); + } + + public function composesReasonAfterGeneratedPart(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::same( + self::findResult($result, 'skipped')->failure?->getMessage(), + SkipMixedMethodsFixture::class + . '::skipped is skipped via #[Skip] ==> broken by the pricing rework, see ISSUE-123', + ); + } + + /** + * An empty reason falls back to the generated part alone — no reporter ever shows an + * empty skip message. + */ + public function fallsBackToGeneratedMessageWithoutReason(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skippedNoReason'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::same( + self::findResult($result, 'skippedNoReason')->failure?->getMessage(), + SkipMixedMethodsFixture::class . '::skippedNoReason is skipped via #[Skip]', + ); + } + + /** + * The origin contract: a result skipped by `#[Skip]` carries the attribute instances in its + * info, so downstream consumers can tell a declarative skip from a runtime one. + */ + public function stampsOriginAttributeOnSyntheticInfo(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped', 'enabled'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + $origin = self::findResult($result, 'skipped')->info->getAttribute(Skip::class); + Assert::array($origin)->hasCount(1); + Assert::instanceOf($origin[0], Skip::class); + Assert::same($origin[0]->reason, 'broken by the pricing rework, see ISSUE-123'); + Assert::null(self::findResult($result, 'enabled')->info->getAttribute(Skip::class)); + } + + public function classLevelSkipSkipsEveryTest(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipClassLevelFixture::class, 'first', 'second'); + $seenTests = null; + + $result = $interceptor->runTestCase($info, self::coreNext($seenTests)); + + Assert::same($seenTests, []); + Assert::same(self::findResult($result, 'first')->status, Status::Skipped); + Assert::same(self::findResult($result, 'second')->status, Status::Skipped); + } + + /** + * The method-level attribute wins as a whole: an empty method reason is not filled in + * from the class reason. + */ + public function methodReasonWinsOverClassReason(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipClassLevelFixture::class, 'first', 'second', 'third'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::true(\str_ends_with( + (string) self::findResult($result, 'first')->failure?->getMessage(), + ' ==> entire case is skipped', + )); + Assert::true(\str_ends_with( + (string) self::findResult($result, 'second')->failure?->getMessage(), + ' ==> method beats class', + )); + Assert::same( + self::findResult($result, 'third')->failure?->getMessage(), + SkipClassLevelFixture::class . '::third is skipped via #[Skip]', + ); + } + + /** + * A case with no skipped tests passes through untouched: same test set, no batch runner + * installed. + */ + public function passesThroughCaseWithoutSkippedTests(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'enabled'); + $batchRunner = false; + $seenTests = null; + + $interceptor->runTestCase($info, static function (CaseInfo $inner) use (&$batchRunner, &$seenTests): CaseResult { + $batchRunner = $inner->batchRunner; + $seenTests = \array_keys($inner->definition->tests->getTests()); + return new CaseResult(results: [], status: Status::Passed); + }); + + Assert::same($seenTests, ['enabled']); + Assert::null($batchRunner); + } + + /** + * A batch runner already installed by an outer interceptor (e.g. testo/fiber's) keeps + * driving the remaining tests — the wrapper wraps it instead of replacing it. + */ + public function wrapsExistingBatchRunnerInsteadOfReplacing(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $innerRunnerCalls = 0; + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped', 'enabled') + ->withBatchRunner(static function (array $handlers) use (&$innerRunnerCalls): array { + ++$innerRunnerCalls; + return \array_map(static fn(callable $handler): TestResult => $handler(), $handlers); + }); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::same($innerRunnerCalls, 1); + Assert::same(self::findResult($result, 'enabled')->status, Status::Passed); + Assert::same(self::findResult($result, 'skipped')->status, Status::Skipped); + } + + /** + * Reporters render test lines from the pipeline events: Starting before Finished, both + * carrying the same address, so a reporter keyed on the identity closes what it opened. + */ + public function dispatchesPipelineEventsForSkippedTests(): void + { + $dispatcher = self::createDispatcher(); + $interceptor = new SkipInterceptor($dispatcher); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + $events = $dispatcher->dispatched; + Assert::count($events, 2); + [$starting, $finished] = $events; + Assert::instanceOf($starting, TestPipelineStarting::class); + Assert::instanceOf($finished, TestPipelineFinished::class); + Assert::same($starting->testInfo->name, 'skipped'); + Assert::same($finished->testInfo->identity, $starting->testInfo->identity); + Assert::same($finished->testResult, self::findResult($result, 'skipped')); + } + + /** + * The terminal renders a test's PHPDoc description from the result attributes (as the + * regular test path stamps it), so the synthetic result must carry it too. + */ + public function carriesDescriptionInSyntheticResult(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped'); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::same( + self::findResult($result, 'skipped')->attributes['description'], + 'Checks that order totals include the reworked pricing.', + ); + } + + /** + * A skipped test is deactivated, not discarded: it leaves the case's active test set — the + * only set the core runs — yet stays a member of the case for anything that reads them all. + */ + public function skippedTestIsDeactivatedNotDiscarded(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfo(SkipMixedMethodsFixture::class, 'skipped', 'enabled'); + + $interceptor->runTestCase($info, self::coreNext()); + + $tests = $info->definition->tests; + Assert::array($tests->getTests())->hasKeys('enabled')->doesNotHaveKeys('skipped'); + Assert::array($tests->getTests(active: false))->hasKeys('skipped'); + Assert::array($tests->all())->hasKeys('skipped', 'enabled'); + } + + /** + * `#[Skip]` on a non-test member is inert. A case is prefilled with every member of the class + * — helpers and lifecycle hooks included — and the interceptor walks its tests only, so an + * attribute on a non-test has nothing to act on. + */ + public function skipOnANonTestMemberIsInert(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfoWith(SkipMixedMethodsFixture::class, [ + 'skipped' => new TestDefinition( + new \ReflectionMethod(SkipMixedMethodsFixture::class, 'skipped'), + isTest: false, + ), + 'enabled' => new TestDefinition(new \ReflectionMethod(SkipMixedMethodsFixture::class, 'enabled')), + ]); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::same($result->summary->count(Status::Skipped), 0); + Assert::same($result->summary->count(Status::Passed), 1); + Assert::same(self::findResult($result, 'enabled')->status, Status::Passed); + } + + /** + * A test an earlier filter already dropped is not resurrected as Skipped: `--filter`/`--group` + * deactivate at location time, and reporting such a test would put it back into a run it was + * excluded from. + */ + public function alreadyFilteredTestIsNotReportedAsSkipped(): void + { + $interceptor = new SkipInterceptor(self::createDispatcher()); + $info = self::createCaseInfoWith(SkipMixedMethodsFixture::class, [ + 'skipped' => new TestDefinition( + new \ReflectionMethod(SkipMixedMethodsFixture::class, 'skipped'), + active: false, + ), + 'enabled' => new TestDefinition(new \ReflectionMethod(SkipMixedMethodsFixture::class, 'enabled')), + ]); + + $result = $interceptor->runTestCase($info, self::coreNext()); + + Assert::same($result->summary->count(Status::Skipped), 0); + Assert::same($result->summary->count(Status::Passed), 1); + Assert::same(self::findResult($result, 'enabled')->status, Status::Passed); + } + + /** + * `#[Skip]` is a plain-test feature: the interceptor declares `testType: TestType::Test`, + * so on a bench or inline case the type filter drops it and the attribute is inert. + */ + public function declaresTestTypeScopingSkipToPlainTests(): void + { + $attributes = (new \ReflectionClass(SkipInterceptor::class)) + ->getAttributes(InterceptorOptions::class); + + Assert::count($attributes, 1); + Assert::same($attributes[0]->newInstance()->testType, TestType::Test); + } + + /** + * The rest of the placement contract: `ORDER_DEFAULT` is the slot the class docblock claims + * (outer to the lifecycle interceptor, inner to the fiber one), and `ConflictPolicy::First` + * is what collapses the instances the fallback alias spawns per `#[Skip]` occurrence. + */ + public function declaresOrderAndConflictPolicy(): void + { + $attributes = (new \ReflectionClass(SkipInterceptor::class)) + ->getAttributes(InterceptorOptions::class); + + Assert::count($attributes, 1); + # Both values equal the InterceptorOptions defaults, so pin that they are written out. + Assert::array($attributes[0]->getArguments())->hasKeys('order', 'onConflict'); + $options = $attributes[0]->newInstance(); + Assert::same($options->order, InterceptorOptions::ORDER_DEFAULT); + Assert::same($options->onConflict, ConflictPolicy::First); + } + + private static function createDispatcher(): EventDispatcherInterface + { + return new class implements EventDispatcherInterface { + /** @var list */ + public array $dispatched = []; + + #[\Override] + public function dispatch(object $event): object + { + $this->dispatched[] = $event; + return $event; + } + }; + } + + /** + * @param class-string $class + * @param non-empty-string ...$methods + */ + private static function createCaseInfo(string $class, string ...$methods): CaseInfo + { + $definitions = []; + foreach ($methods as $method) { + $definitions[$method] = new TestDefinition(new \ReflectionMethod($class, $method)); + } + + return self::createCaseInfoWith($class, $definitions); + } + + /** + * The same case built from definitions that carry their own flags — the shape a case has after + * prefilling (non-test members) or after an earlier filter deactivated one of its tests. + * + * @param class-string $class + * @param array $definitions + */ + private static function createCaseInfoWith(string $class, array $definitions): CaseInfo + { + $caseDefinition = new CaseDefinition( + name: $class, + type: 'test', + file: Path::create(__FILE__), + reflection: new \ReflectionClass($class), + tests: TestDefinitions::fromArray(...$definitions), + ); + + return new CaseInfo(definition: $caseDefinition, suiteIdentity: new SuiteIdentity('Test/Unit')); + } + + /** + * A `$next` that mimics the core case loop: runs the surviving tests as Passed through + * the case's batch runner (or inline without one) and aggregates the case summary. + * + * @param list|null $seenTests Filled with the test names that survived to `$next`. + */ + private static function coreNext(?array &$seenTests = null): \Closure + { + return static function (CaseInfo $info) use (&$seenTests): CaseResult { + $seenTests = \array_keys($info->definition->tests->getTests()); + + $handlers = []; + foreach ($info->definition->tests->getTests() as $name => $definition) { + $handlers[] = static fn(): TestResult => new TestResult( + info: new TestInfo(name: $name, caseInfo: $info, testDefinition: $definition), + status: Status::Passed, + summary: Summary::forTest(Status::Passed), + ); + } + + $runner = $info->batchRunner; + /** @var list $results */ + $results = $runner === null + ? \array_map(static fn(\Closure $handler): TestResult => $handler(), $handlers) + : $runner($handlers); + + return new CaseResult( + results: $results, + status: Status::Passed, + summary: Summary::combine(\array_map(static fn(TestResult $r): Summary => $r->summary, $results)), + ); + }; + } + + private static function findResult(CaseResult $result, string $name): TestResult + { + foreach ($result as $testResult) { + if ($testResult->info->name === $name) { + return $testResult; + } + } + + throw new \LogicException("No result for test {$name}."); + } +} diff --git a/plugin/skip/tests/Unit/SkipAttributeTest.php b/plugin/skip/tests/Unit/SkipAttributeTest.php new file mode 100644 index 00000000..3be2c744 --- /dev/null +++ b/plugin/skip/tests/Unit/SkipAttributeTest.php @@ -0,0 +1,83 @@ +reason, ''); + } + + public function customReason(): void + { + $skip = new Skip('flaky on CI, see ISSUE-123'); + + Assert::same($skip->reason, 'flaky on CI, see ISSUE-123'); + } + + /** + * Exactly class, method and function — and nothing else, so no `IS_REPEATABLE`. + */ + public function targetsClassMethodAndFunctionOnly(): void + { + $attributes = (new \ReflectionClass(Skip::class))->getAttributes(\Attribute::class); + + Assert::count($attributes, 1); + /** @var \Attribute $attribute */ + $attribute = $attributes[0]->newInstance(); + + Assert::same( + $attribute->flags, + \Attribute::TARGET_CLASS | \Attribute::TARGET_METHOD | \Attribute::TARGET_FUNCTION, + ); + } + + /** + * The pipeline collects `Interceptable` attributes; without the marker a class-level + * `#[Skip]` would be invisible to the attributes interceptor. + */ + public function isInterceptable(): void + { + Assert::instanceOf(new Skip(), Interceptable::class); + } + + /** + * A method- or function-level `#[Skip]` has to reach the case pipeline, which is built from + * class attributes alone unless the attribute opts in. + */ + public function isCaseInterceptable(): void + { + Assert::instanceOf(new Skip(), CaseInterceptable::class); + } + + /** + * An `Interceptable` attribute must resolve to an interceptor, or the attributes + * interceptor throws at pipeline build time; the fallback names the handler. + */ + public function declaresSkipInterceptorAsFallback(): void + { + $attributes = (new \ReflectionClass(Skip::class))->getAttributes(FallbackInterceptor::class); + + Assert::count($attributes, 1); + Assert::same($attributes[0]->newInstance()->class, SkipInterceptor::class); + } +} diff --git a/plugin/skip/tests/suites.php b/plugin/skip/tests/suites.php new file mode 100644 index 00000000..63b4cded --- /dev/null +++ b/plugin/skip/tests/suites.php @@ -0,0 +1,22 @@ +definition->tests->getTests()` and set + `$definition->active = false` on each of those. Deactivated, not discarded: `getTests()` then + yields only the rest, and those are the tests the core runs. +- Hand back their results yourself, from `CaseInfo::withBatchRunner`: **wrap** the runner already on + the case (testo/fiber may have set one), never replace it — and run the handlers yourself when the + case carries none — then append one synthetic `TestResult` per skipped test after the inner + runner returns. +- Dispatch `TestPipelineStarting`/`TestPipelineFinished` around each synthetic result, or reporters + never render its line, and stamp `summary: Summary::forTest(Status::Skipped)` on it — a result that + never passes through the test runner is not counted for you. + +The shipped implementation of exactly this shape is `Testo\Skip\Internal\SkipInterceptor` in +`plugin/skip`, serving the `#[Skip]` attribute (whose contract is in the `testo-write-tests` skill). +Read it as a reference — it is `@internal` (and `final`), don't import it. + ## Container scopes — provision per-case / per-suite resources `$container->scope($closure)` runs `$closure` in a **child scope**: services bound inside live only for @@ -243,9 +263,21 @@ $method = $info->testDefinition->reflection; $optedOut = $method->getAttributes(WithoutTransaction::class) !== []; ``` +An attribute can also bring its own interceptor, so users need no plugin registration at all — +`#[Retry]`, `#[Repeat]` and `#[Skip]` ship this way. Implement `Testo\Pipeline\Attribute\Interceptable` +and name the handler with `#[FallbackInterceptor(MyInterceptor::class)]` (repeatable — one per +pipeline position); the core instantiates the interceptor with the attribute instance as a constructor +argument when the attribute is found on a class (case and test pipelines) or on a test (test pipeline +only). If a test-level attribute has to act on the **case** pipeline — take that test out before the +class-level hooks, say — implement `Testo\Pipeline\Attribute\CaseInterceptable` instead: one interceptor +instance is spawned per attribute occurrence, so declare `ConflictPolicy::First` in +`#[InterceptorOptions]` to keep a single one. + ## Pitfalls - **Skipping**: return a `Status::Skipped` `TestResult`; never `throw SkipTest` from an interceptor. + From a **case** interceptor still call `$next` — deactivate the tests you skip and append their + results through the batch runner. - **Cleanup**: wrap `$next()` in `try/finally`; a later interceptor may throw. - **State**: prefer pipeline attributes / container scope over mutable interceptor fields. - **Listeners** observe; **interceptors** change behaviour. Don't try to alter a run from a listener. diff --git a/skills/testo-write-tests/SKILL.md b/skills/testo-write-tests/SKILL.md index c76bd286..b8773386 100644 --- a/skills/testo-write-tests/SKILL.md +++ b/skills/testo-write-tests/SKILL.md @@ -136,6 +136,49 @@ Constraints: - Subclasses work: `class MissingExtensionSkip extends SkipTest {}` is still recognized. - Return type stays `void`, or `never` if the throw is unconditional. +## Skipping a test with #[Skip] + +To skip a test declaratively — without running any of its code — put `Testo\Skip` (from the +`testo/skip` plugin) on the test method (inherited by an overriding +method that does not repeat it), the class (skips every test of the case; inherited from parents +and traits, a method-level reason wins), or a free function: + +```php +use Testo\Skip; + +#[Test] +#[Skip('broken by the pricing rework, see ISSUE-123')] +public function calculatesTotal(): void { /* ... */ } // reported as Skipped, body never runs +``` + +The test is reported as `Status::Skipped` and counted in the totals; its reason travels in the +result's failure message `{testId} is skipped via #[Skip] ==> {reason}` (without ` ==> ...` when +the reason is empty). The JUnit, TeamCity and HTML reports show that message; the terminal prints +the skipped line without it, and the compact `--json` report only counts the test in +`totals.skipped`. + +`reason` is optional and the attribute is not repeatable — but **always pass a reason that points +at an issue** (`#[Skip('flaky on CI, see ISSUE-123')]`); a bare `#[Skip]` is how a skipped test rots +unreviewed. The attribute needs no plugin registration: it wires its own interceptor, from a class, +a method or a function alike. + +Which skipping tool to reach for: + +| Tool | Decided by | Visibility | Use when | +|---|---|---|---| +| `#[Skip('...')]` | code, ahead of time | always reported; reason in JUnit/TeamCity/HTML | the test is knowingly broken, tracked in an issue, and must be returned to | +| `throw SkipTest` | test body, at runtime | reported when the run gets there | test isn't applicable in this environment | +| `#[Group]` + `--group=!x` | runner invocation | invisible — filtered out of reports | a category you sometimes don't run | + +Runtime contract of `#[Skip]`: the test never enters the per-test pipeline, so +`#[BeforeTest]`/`#[AfterTest]`, data providers, `#[Retry]`/`#[Repeat]` and coverage never +engage, and a data-driven test yields a single Skipped entry (the provider is not called). +`#[BeforeClass]`/`#[AfterClass]` still run (also when every test of the case is skipped). A +skipped test never requires an instance of the case class: a fully skipped class is built only +when a non-static class-level hook forces it, while enabled neighbors construct it as usual. A +run of only `#[Skip]`-marked tests is a success (exit 0). `#[Skip]` applies to plain tests only: +on a `#[Bench]` or `#[TestInline]` target it is inert — the benchmark or inline case runs as usual. + ## Tests that intentionally perform no assertions A test that finishes successfully without recording a single assertion is reported as diff --git a/testo.php b/testo.php index 0ad33f9b..989d63cf 100644 --- a/testo.php +++ b/testo.php @@ -67,6 +67,7 @@ require 'plugin/lifecycle/tests/suites.php', require 'plugin/repeat/tests/suites.php', require 'plugin/retry/tests/suites.php', + require 'plugin/skip/tests/suites.php', require 'plugin/test/tests/suites.php', require 'tests/Testo/suites.php', require 'tests/Application/suites.php', diff --git a/tests/Core/Pipeline/AttributesInterceptorTest.php b/tests/Core/Pipeline/AttributesInterceptorTest.php index 1bd61323..0db0eb0f 100644 --- a/tests/Core/Pipeline/AttributesInterceptorTest.php +++ b/tests/Core/Pipeline/AttributesInterceptorTest.php @@ -14,7 +14,9 @@ use Testo\Core\Context\TestResult; use Testo\Core\Definition\CaseDefinition; use Testo\Core\Definition\TestDefinition; +use Testo\Core\Definition\TestDefinitions; use Testo\Core\Value\Status; +use Testo\Pipeline\Attribute\CaseInterceptable; use Testo\Pipeline\Attribute\Interceptable; use Testo\Pipeline\Attribute\FallbackInterceptor; use Testo\Pipeline\Internal\AttributesInterceptor; @@ -317,6 +319,56 @@ public function runTestCasePreparesPipelineAroundClosureNext(): void Assert::same($result->status, Status::Flaky); } + public function runTestCaseRunsCaseInterceptorOfAMethodLevelCaseInterceptableAttribute(): void + { + $caseInfo = $this->makeCaseInfoWithTest(TestWithMethodCaseInterceptableAttribute::class); + + $interceptor = new AttributesInterceptor($this->createInterceptorProvider()); + + $captured = null; + $result = $interceptor->runTestCase($caseInfo, function (CaseInfo $info) use (&$captured): CaseResult { + $captured = $info; + return new CaseResult([], Status::Passed); + }); + + // The case fallback interceptor ran for a method-level attribute: it rewrote the status. + Assert::same($result->status, Status::Flaky); + // A test-level attribute describes its test, not the case: nothing is stamped on the CaseInfo. + Assert::same($captured->attributes, []); + } + + public function runTestCaseIgnoresAMethodLevelInterceptableWithoutCaseOptIn(): void + { + // Same case-interceptor fallback, but the attribute is a plain Interceptable: from a method it + // must stay out of the case pipeline (a per-test attribute must not reconfigure the whole case). + $caseInfo = $this->makeCaseInfoWithTest(TestWithMethodPlainInterceptableCaseAttribute::class); + + $interceptor = new AttributesInterceptor($this->createInterceptorProvider()); + $terminal = new CaseResult([], Status::Passed); + + $captured = null; + $result = $interceptor->runTestCase($caseInfo, function (CaseInfo $info) use (&$captured, $terminal): CaseResult { + $captured = $info; + return $terminal; + }); + + Assert::same($result, $terminal); + Assert::same($captured, $caseInfo); + } + + public function runTestCaseSkipsDeactivatedTestsWhenCollectingCaseInterceptableAttributes(): void + { + $caseInfo = $this->makeCaseInfoWithTest(TestWithMethodCaseInterceptableAttribute::class, active: false); + + $interceptor = new AttributesInterceptor($this->createInterceptorProvider()); + $terminal = new CaseResult([], Status::Passed); + + $result = $interceptor->runTestCase($caseInfo, static fn(CaseInfo $info): CaseResult => $terminal); + + // A filtered-out test is not part of the run, so its attribute must not shape the case. + Assert::same($result, $terminal); + } + public function runTestPreparesPipelineAroundClosureNext(): void { $caseInfo = $this->makeCaseInfo(new \ReflectionClass(TestWithClassInterceptableAttribute::class)); @@ -364,6 +416,25 @@ private function makeTestInfo(string $class, bool $classReflection): TestInfo return new TestInfo('test', $caseInfo, $testDefinition); } + /** + * A case without class attributes whose single test `test` is registered in the definitions. + * + * @param class-string $class + */ + private function makeCaseInfoWithTest(string $class, bool $active = true): CaseInfo + { + $definition = new TestDefinition(new \ReflectionMethod($class, 'test')); + $definition->active = $active; + + return new CaseInfo(new CaseDefinition( + name: 'TestCase', + type: 'unit', + file: Path::create(__FILE__), + reflection: new \ReflectionClass($class), + tests: TestDefinitions::fromArray(test: $definition), + ), new SuiteIdentity('Core/Pipeline')); + } + private function makeCaseInfo(?\ReflectionClass $reflection): CaseInfo { return new CaseInfo(new CaseDefinition( @@ -417,6 +488,18 @@ final class TestWithRepeatedInterceptableAttribute public function test(): void {} } +final class TestWithMethodCaseInterceptableAttribute +{ + #[TestMethodCaseInterceptableAttribute] + public function test(): void {} +} + +final class TestWithMethodPlainInterceptableCaseAttribute +{ + #[TestMethodPlainInterceptableCaseAttribute] + public function test(): void {} +} + #[\Attribute(\Attribute::TARGET_CLASS)] #[FallbackInterceptor(TestTagRunInterceptor::class)] final class TestClassInterceptableAttribute implements Interceptable {} @@ -433,6 +516,14 @@ final class TestCaseInterceptableAttribute implements Interceptable {} #[FallbackInterceptor(TestTagRunInterceptor::class)] final class TestRepeatableInterceptableAttribute implements Interceptable {} +#[\Attribute(\Attribute::TARGET_METHOD)] +#[FallbackInterceptor(TestTagCaseRunInterceptor::class)] +final class TestMethodCaseInterceptableAttribute implements CaseInterceptable {} + +#[\Attribute(\Attribute::TARGET_METHOD)] +#[FallbackInterceptor(TestTagCaseRunInterceptor::class)] +final class TestMethodPlainInterceptableCaseAttribute implements Interceptable {} + /** * Distinguishable effect: tags the result so a real pass-through differs from a no-op. */ diff --git a/tests/Output/Unit/JUnit/JUnitWriterTest.php b/tests/Output/Unit/JUnit/JUnitWriterTest.php index 77a4e4cc..613f387b 100644 --- a/tests/Output/Unit/JUnit/JUnitWriterTest.php +++ b/tests/Output/Unit/JUnit/JUnitWriterTest.php @@ -18,6 +18,7 @@ use Testo\Core\Context\TestResult; use Testo\Core\Definition\CaseDefinition; use Testo\Core\Definition\TestDefinition; +use Testo\Core\Exception\SkipTest; use Testo\Core\Value\Status; use Testo\Output\JUnit\Internal\JUnitWriter; use Testo\Test; @@ -163,6 +164,52 @@ public function skippedTestRendersSkippedElement(): void Assert::count($xml->testsuite->testcase->skipped, 1); } + /** + * The failure message is the single source of truth for the skip reason: every producer + * (a runtime throw, a declarative `#[Skip]`) delivers it the same way, and the writer + * renders it as the `message` of ``. + */ + #[Covers(JUnitWriter::class)] + public function skippedTestCarriesTheReasonFromTheFailureMessage(): void + { + $writer = new JUnitWriter(); + $writer->startSuite('MySuite'); + $writer->addTestResult(self::makeResult( + 'passingTest', + Status::Skipped, + failure: new SkipTest('sqlite extension is missing'), + )); + $writer->finishSuite(); + + $xml = self::loadXml($writer->generate('Testo')); + + $skipped = $xml->testsuite->testcase->skipped; + Assert::count($skipped, 1); + Assert::same((string) $skipped['message'], 'sqlite extension is missing'); + } + + /** + * No reason — no `message`: an empty attribute would read as an empty reason. + */ + #[Covers(JUnitWriter::class)] + public function skippedTestWithoutAReasonOmitsTheMessage(): void + { + $writer = new JUnitWriter(); + $writer->startSuite('MySuite'); + $writer->addTestResult(self::makeResult( + 'passingTest', + Status::Skipped, + failure: new SkipTest(), + )); + $writer->finishSuite(); + + $xml = self::loadXml($writer->generate('Testo')); + + $skipped = $xml->testsuite->testcase->skipped; + Assert::count($skipped, 1); + Assert::null($skipped['message']); + } + public function cancelledTestCountsAsSkipped(): void { // Arrange