diff --git a/.github/workflows/duplicated_code.yaml b/.github/workflows/duplicated_code.yaml new file mode 100644 index 00000000000..28dc9845b79 --- /dev/null +++ b/.github/workflows/duplicated_code.yaml @@ -0,0 +1,29 @@ +name: Duplicated Code + +on: + pull_request: null + push: + branches: + - "main" + +env: + COMPOSER_ROOT_VERSION: "dev-main" + +jobs: + duplicated_code: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v5 + + - + uses: shivammathur/setup-php@v2 + with: + php-version: 8.4 + coverage: none + + - run: composer install --no-progress --ansi + + # fails when a large copy-pasted block (>= 400 tokens) is added + - run: composer duplicated-code diff --git a/composer.json b/composer.json index 59154c70a29..061e2c073b9 100644 --- a/composer.json +++ b/composer.json @@ -70,7 +70,8 @@ "src" ], "Rector\\Utils\\": "utils", - "Rector\\Utils\\PHPStan\\": "utils/phpstan/src" + "Rector\\Utils\\PHPStan\\": "utils/phpstan/src", + "Rector\\Utils\\Duplicates\\": "utils/duplicates/src" }, "files": [ "src/functions/node_helper.php" @@ -83,6 +84,7 @@ "tests" ], "Rector\\Utils\\PHPStan\\Tests\\": "utils/phpstan/tests", + "Rector\\Utils\\Duplicates\\Tests\\": "utils/duplicates/tests", "E2e\\Parallel\\Reflection\\Resolver\\": "e2e/parallel-reflection-resolver/src/", "Rector\\Scripts\\": "scripts/src" }, @@ -104,11 +106,12 @@ "@phpstan", "@test" ], - "test": "vendor/bin/fastunit tests rules-tests utils/phpstan/tests", + "test": "vendor/bin/fastunit tests rules-tests utils/phpstan/tests utils/duplicates/tests", "check-cs": "vendor/bin/ecs check --ansi", "fix-cs": "vendor/bin/ecs check --fix --ansi", "phpstan": "vendor/bin/phpstan analyse --ansi --memory-limit=512M", "rector": "bin/rector process --ansi", + "duplicated-code": "php utils/duplicates/bin/duplicates.php --min-lines 5 --min-tokens 400 src rules", "preload": "php build/build-preload.php .", "release": "vendor/bin/rng --from-commit X --to-commit Y --remote-repository rectorphp/rector-symfony --remote-repository rectorphp/rector-doctrine --remote-repository rectorphp/rector-phpunit" }, diff --git a/utils/duplicates/README.md b/utils/duplicates/README.md new file mode 100644 index 00000000000..0950b62da83 --- /dev/null +++ b/utils/duplicates/README.md @@ -0,0 +1,19 @@ +# Duplicates + +Token-based copy-paste detector, a small clone of [phpcpd](https://github.com/phpcpd-next/phpcpd). + +It tokenizes PHP with `token_get_all()`, strips whitespace and comments, then finds exact duplicate token spans with a Rabin-Karp rolling hash. + +## Usage + +```bash +php utils/duplicates/bin/duplicates.php src rules +``` + +Options: + +- `--min-lines` minimum lines of a clone (default `5`) +- `--min-tokens` minimum tokens of a clone (default `70`) +- `--fuzzy` ignore variable names when matching + +Exit code is `1` when clones are found, `0` otherwise. diff --git a/utils/duplicates/bin/duplicates.php b/utils/duplicates/bin/duplicates.php new file mode 100644 index 00000000000..fa06fd252f9 --- /dev/null +++ b/utils/duplicates/bin/duplicates.php @@ -0,0 +1,109 @@ +#!/usr/bin/env php +...' . PHP_EOL); + exit(1); +} + +$filePaths = []; +foreach ($paths as $path) { + if (is_file($path)) { + $filePaths[] = $path; + continue; + } + + if (! is_dir($path)) { + continue; + } + + $finder = new Finder() + ->files() + ->in($path) + ->name('*.php'); + + foreach ($finder as $fileInfo) { + $realPath = $fileInfo->getRealPath(); + if ($realPath === false) { + continue; + } + + $filePaths[] = $realPath; + } +} + +if ($filePaths === []) { + echo 'No PHP files found in the given paths.' . PHP_EOL; + exit(0); +} + +$cloneDetector = new CloneDetector($minLines, $minTokens, $fuzzy); +$clones = $cloneDetector->detect($filePaths); + +if ($clones === []) { + echo sprintf('[OK] No duplicates found in %d files', count($filePaths)) . PHP_EOL; + exit(0); +} + +$duplicatedLines = 0; +foreach ($clones as $clone) { + echo sprintf( + ' - %s:%d-%d (%d lines, %d tokens)', + $clone->firstFile->filePath, + $clone->firstFile->startLine, + $clone->firstFile->endLine, + $clone->lines, + $clone->tokens + ) . PHP_EOL; + echo sprintf( + ' %s:%d-%d', + $clone->secondFile->filePath, + $clone->secondFile->startLine, + $clone->secondFile->endLine + ) . PHP_EOL; + echo PHP_EOL; + + $duplicatedLines += $clone->lines; +} + +fwrite(STDERR, sprintf( + '[ERROR] Found %d clones with %d duplicated lines in %d scanned files', + count($clones), + $duplicatedLines, + count($filePaths) +) . PHP_EOL); + +exit(1); diff --git a/utils/duplicates/src/CloneDetector.php b/utils/duplicates/src/CloneDetector.php new file mode 100644 index 00000000000..62629984989 --- /dev/null +++ b/utils/duplicates/src/CloneDetector.php @@ -0,0 +1,196 @@ + + */ + private const array IGNORED_TOKENS = [ + T_INLINE_HTML => true, + T_COMMENT => true, + T_DOC_COMMENT => true, + T_OPEN_TAG => true, + T_OPEN_TAG_WITH_ECHO => true, + T_CLOSE_TAG => true, + T_WHITESPACE => true, + ]; + + /** + * Bytes contributed by each kept token to the signature: 1 type byte + 4 CRC32 bytes. + */ + private const int BYTES_PER_TOKEN = 5; + + /** + * First seen location per window hash: hash => [filePath, tokenIndex]. + * + * @var array + */ + private array $hashes = []; + + /** + * Per-file token line numbers, kept for span line lookup. + * + * @var array + */ + private array $fileLines = []; + + /** + * @var CodeClone[] + */ + private array $clones = []; + + public function __construct( + private readonly int $minLines, + private readonly int $minTokens, + private readonly bool $fuzzy + ) { + } + + /** + * @param string[] $filePaths + * @return CodeClone[] + */ + public function detect(array $filePaths): array + { + foreach ($filePaths as $filePath) { + $this->processFile($filePath); + } + + return $this->clones; + } + + private function processFile(string $filePath): void + { + $source = file_get_contents($filePath); + if ($source === false) { + return; + } + + [$signature, $lines] = $this->tokenize($source); + $this->fileLines[$filePath] = $lines; + + $tokenCount = count($lines); + if ($tokenCount < $this->minTokens) { + return; + } + + $lastWindow = $tokenCount - $this->minTokens; + $windowBytes = $this->minTokens * self::BYTES_PER_TOKEN; + + $found = false; + $firstToken = 0; + $originFile = ''; + $originToken = 0; + + for ($i = 0; $i <= $lastWindow; ++$i) { + $hash = substr(md5(substr($signature, $i * self::BYTES_PER_TOKEN, $windowBytes), true), 0, 8); + + if (isset($this->hashes[$hash])) { + if (! $found) { + $found = true; + $firstToken = $i; + [$originFile, $originToken] = $this->hashes[$hash]; + } + + continue; + } + + if ($found) { + $this->recordClone($originFile, $originToken, $filePath, $firstToken, $i); + $found = false; + } + + $this->hashes[$hash] = [$filePath, $i]; + } + + if ($found) { + $this->recordClone($originFile, $originToken, $filePath, $firstToken, $lastWindow + 1); + } + } + + private function recordClone( + string $originFile, + int $originToken, + string $currentFile, + int $currentToken, + int $mismatchWindow + ): void { + $windowCount = $mismatchWindow - $currentToken; + $tokenSpan = $windowCount + $this->minTokens - 1; + + $originLines = $this->fileLines[$originFile]; + $currentLines = $this->fileLines[$currentFile]; + + $originStartLine = $originLines[$originToken]; + $originEndLine = $originLines[min($originToken + $tokenSpan - 1, count($originLines) - 1)]; + + $currentStartLine = $currentLines[$currentToken]; + $currentEndLine = $currentLines[min($currentToken + $tokenSpan - 1, count($currentLines) - 1)]; + + $numberOfLines = $originEndLine - $originStartLine + 1; + if ($numberOfLines < $this->minLines) { + return; + } + + $this->clones[] = new CodeClone( + new CodeCloneFile($originFile, $originStartLine, $originEndLine), + new CodeCloneFile($currentFile, $currentStartLine, $currentEndLine), + $numberOfLines, + $tokenSpan + ); + } + + /** + * Builds the token signature and the parallel line map for a source string. + * + * @return array{string, int[]} + */ + private function tokenize(string $source): array + { + $signature = ''; + $lines = []; + $currentLine = 1; + + foreach (token_get_all($source) as $token) { + if (is_array($token)) { + $tokenId = $token[0]; + $tokenText = $token[1]; + $currentLine = $token[2]; + + if (isset(self::IGNORED_TOKENS[$tokenId])) { + $currentLine += substr_count($tokenText, "\n"); + continue; + } + + if ($this->fuzzy && $tokenId === T_VARIABLE) { + $tokenText = '$'; + } + + $signature .= chr($tokenId & 255) . pack('N*', crc32($tokenText)); + $lines[] = $currentLine; + $currentLine += substr_count($tokenText, "\n"); + + continue; + } + + $signature .= chr(0) . pack('N*', crc32($token)); + $lines[] = $currentLine; + } + + return [$signature, $lines]; + } +} diff --git a/utils/duplicates/src/ValueObject/CodeClone.php b/utils/duplicates/src/ValueObject/CodeClone.php new file mode 100644 index 00000000000..ea89ea758ef --- /dev/null +++ b/utils/duplicates/src/ValueObject/CodeClone.php @@ -0,0 +1,16 @@ +detect([ + __DIR__ . '/Fixture/first_duplicate.php.inc', + __DIR__ . '/Fixture/second_duplicate.php.inc', + ]); + + $this->assertCount(1, $clones); + + $codeClone = $clones[0]; + $this->assertStringEndsWith('first_duplicate.php.inc', $codeClone->firstFile->filePath); + $this->assertStringEndsWith('second_duplicate.php.inc', $codeClone->secondFile->filePath); + $this->assertGreaterThanOrEqual(3, $codeClone->lines); + } + + public function testDoesNotReportUniqueCode(): void + { + $cloneDetector = new CloneDetector(3, 25, false); + + $clones = $cloneDetector->detect([ + __DIR__ . '/Fixture/first_duplicate.php.inc', + __DIR__ . '/Fixture/unique.php.inc', + ]); + + $this->assertSame([], $clones); + } +} diff --git a/utils/duplicates/tests/Fixture/first_duplicate.php.inc b/utils/duplicates/tests/Fixture/first_duplicate.php.inc new file mode 100644 index 00000000000..dded1cbdc0d --- /dev/null +++ b/utils/duplicates/tests/Fixture/first_duplicate.php.inc @@ -0,0 +1,13 @@ +