Skip to content

perf(files): avoid materializing directory listings in scanner #64269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
joshtrichards wants to merge 3 commits into
base: master
Choose a base branch
from
+198 −71
Open

perf(files): avoid materializing directory listings in scanner #64269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37385ba
perf(files): avoid materializing directory listings during scans
joshtrichards Sep 11, 2026
576963d
test(files): cover lazy directory scanning
joshtrichards Sep 11, 2026
4c063ff
test(files): fix typo in ScannerTest property
joshtrichards Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 120 additions & 70 deletions lib/private/Files/Cache/Scanner.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,96 +443,146 @@ protected function scanChildren(string $path, $recursive, int $reuse, int $folde
}

/**
* Process the current directory before recursively scanning child directories.
*
* Keeping this work in a separate method releases the current directory's local
* state before scanChildren() descends into the child queue.
*
* @param bool|IScanner::SCAN_RECURSIVE_INCOMPLETE $recursive
*/
private function handleChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float &$size, bool &$etagChanged): array {
// we put this in its own function so it cleans up the memory before we start recursing
private function handleChildren(
string $path,
$recursive,
int $reuse,
int $folderId,
bool $lock,
int|float &$size,
bool &$etagChanged,
): array {
$existingChildren = $this->getExistingChildren($folderId);
$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));

if (count($existingChildren) === 0 && count($newChildren) === 0) {
// no need to do a transaction
return [];
}

if ($this->useTransactions) {
$this->connection->beginTransaction();
}

$processingStarted = false;
$exceptionOccurred = false;
$childQueue = [];
$newChildNames = [];
foreach ($newChildren as $fileMeta) {
$permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions'];
if ($permissions === 0) {
continue;
}
$originalFile = $fileMeta['name'];
$file = trim(Filesystem::normalizePath($originalFile), '/');
if (trim($originalFile, '/') !== $file) {
// encoding mismatch, might require compatibility wrapper
Server::get(LoggerInterface::class)->debug('Scanner: Skipping non-normalized file name "' . $originalFile . '" in path "' . $path . '".', ['app' => 'core']);
$this->emit('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', [$path ? $path . '/' . $originalFile : $originalFile]);
// skip this entry
continue;
}

$newChildNames[] = $file;
$child = $path ? $path . '/' . $file : $file;
try {
$existingData = $existingChildren[$file] ?? false;
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);
if ($data) {
if ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE) {
$childQueue[$child] = [$data['fileid'], $data['size']];
} elseif ($data['mimetype'] === 'httpd/unix-directory' && $recursive === self::SCAN_RECURSIVE_INCOMPLETE && $data['size'] === -1) {
// only recurse into folders which aren't fully scanned
$childQueue[$child] = [$data['fileid'], $data['size']];
} elseif ($data['size'] === -1) {
$size = -1;
} elseif ($size !== -1) {
$size += $data['size'];
try {
foreach ($this->storage->getDirectoryContent($path) as $fileMeta) {
// Avoid opening a transaction for an empty directory with no cached children.
if (!$processingStarted) {
if ($this->useTransactions) {
$this->connection->beginTransaction();
}
$processingStarted = true;
}

$permissions = $fileMeta['scan_permissions'] ?? $fileMeta['permissions'];
if ($permissions === 0) {
continue;
}

$originalFile = $fileMeta['name'];
$file = trim(Filesystem::normalizePath($originalFile), '/');

if (trim($originalFile, '/') !== $file) {
// Non-normalized names cannot be addressed consistently through the cache; may require compatibility wrapper.
Server::get(LoggerInterface::class)->debug(
'Scanner: Skipping non-normalized file name "' . $originalFile . '" in path "' . $path . '".',
['app' => 'core']
);
$this->emit(
'\OC\Files\Cache\Scanner',
'normalizedNameMismatch',
[$path ? $path . '/' . $originalFile : $originalFile]
);
continue;
}

$newChildNames[] = $file;
$child = $path ? $path . '/' . $file : $file;

try {
$existingData = $existingChildren[$file] ?? false;
$data = $this->scanFile($child, $reuse, $folderId, $existingData, $lock, $fileMeta);

if ($data) {
if (
$data['mimetype'] === 'httpd/unix-directory'
&& $recursive === self::SCAN_RECURSIVE
) {
$childQueue[$child] = [$data['fileid'], $data['size']];
} elseif (
$data['mimetype'] === 'httpd/unix-directory'
&& $recursive === self::SCAN_RECURSIVE_INCOMPLETE
&& $data['size'] === -1
) {
// In incomplete scans, recurse only into folders that still need scanning.
$childQueue[$child] = [$data['fileid'], $data['size']];
} elseif ($data['size'] === -1) {
$size = -1;
} elseif ($size !== -1) {
$size += $data['size'];
}

if (isset($data['etag_changed']) && $data['etag_changed']) {
$etagChanged = true;
if (isset($data['etag_changed']) && $data['etag_changed']) {
$etagChanged = true;
}
}
} catch (Exception $ex) {
// A concurrent scanner may have inserted this entry already.
if ($this->useTransactions) {
$this->connection->rollback();
$this->connection->beginTransaction();
}
Server::get(LoggerInterface::class)->debug(
'Exception while scanning file "' . $child . '"',
['app' => 'core', 'exception' => $ex]
);
$exceptionOccurred = true;
} catch (LockedException $e) {
if ($this->useTransactions) {
$this->connection->rollback();
}
throw $e;
}
} catch (Exception $ex) {
// might happen if inserting duplicate while a scanning
// process is running in parallel
// log and ignore
}

if (!$processingStarted && $existingChildren === []) {
return [];
}

$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);

// Removed cached children still require a transaction when the listing is empty.
if ($removedChildren !== [] && !$processingStarted) {
if ($this->useTransactions) {
$this->connection->rollback();
$this->connection->beginTransaction();
}
Server::get(LoggerInterface::class)->debug('Exception while scanning file "' . $child . '"', [
'app' => 'core',
'exception' => $ex,
]);
$exceptionOccurred = true;
} catch (LockedException $e) {
if ($this->useTransactions) {
$this->connection->rollback();
}
throw $e;
$processingStarted = true;
}

foreach ($removedChildren as $childName) {
$child = $path ? $path . '/' . $childName : $childName;
$this->removeFromCache((string)$child);
}

if ($processingStarted && $this->useTransactions) {
$this->connection->commit();
}
} catch (\Throwable $e) {
// Lazy directory listings can fail after the transaction has started.
if ($this->useTransactions && $this->connection->inTransaction()) {
$this->connection->rollBack();
}

throw $e;
}
$removedChildren = \array_diff(array_keys($existingChildren), $newChildNames);
foreach ($removedChildren as $childName) {
$child = $path ? $path . '/' . $childName : $childName;
$this->removeFromCache((string)$child);
}
if ($this->useTransactions) {
$this->connection->commit();
}

if ($exceptionOccurred) {
// It might happen that the parallel scan process has already
// inserted mimetypes but those weren't available yet inside the transaction
// To make sure to have the updated mime types in such cases,
// we reload them here
// Reload MIME types that may have been inserted by a concurrent scan.
Server::get(IMimeTypeLoader::class)->reset();
}

return $childQueue;
}

Expand Down
79 changes: 78 additions & 1 deletion tests/lib/Files/Cache/ScannerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,38 @@
use OCP\Server;
use Test\TestCase;

final class LazyDirectoryStorage extends Temporary {
public bool $firstChildProcessed = false;
public bool $resumedBeforeFirstChildProcessed = false;
public bool $throwAfterFirstEntry = false;

#[\Override]
public function getDirectoryContent(string $directory): \Traversable {
$first = true;

foreach (parent::getDirectoryContent($directory) as $entry) {
yield $entry;

if (!$first) {
continue;
}

$first = false;

if (!$this->firstChildProcessed) {
$this->resumedBeforeFirstChildProcessed = true;
}

if ($this->throwAfterFirstEntry) {
throw new \RuntimeException('Directory listing failed after the first entry');
}
}
}
}

/**
* Class ScannerTest
*
*
* @package Test\Files\Cache
*/
#[\PHPUnit\Framework\Attributes\Group('DB')]
Expand Down Expand Up @@ -125,6 +153,55 @@ public function testFolder(): void {
$this->assertEquals($cachedDataFolder2['size'], $cachedDataText2['size']);
}

public function testDirectoryContentIsConsumedLazily(): void {
$storage = new LazyDirectoryStorage();
$storage->file_put_contents('file.txt', 'content');

$scanner = new Scanner($storage);
$scanner->listen(
'\OC\Files\Cache\Scanner',
'scanFile',
function (string $path) use ($storage): void {
if ($path === 'file.txt') {
$storage->firstChildProcessed = true;
}
},
);

$scanner->scan('');

$this->assertTrue($storage->firstChildProcessed);
$this->assertFalse($storage->resumedBeforeFirstChildProcessed);
$this->assertTrue($storage->getCache()->inCache('file.txt'));
}

public function testDirectoryContentFailureRollsBackTransaction(): void {
$storage = new LazyDirectoryStorage();
$storage->file_put_contents('file.txt', 'content');
$storage->throwAfterFirstEntry = true;

$scanner = new Scanner($storage);
$connection = Server::get(IDBConnection::class);

$this->expectException(\RuntimeException::class);
$this->expectExceptionMessage('Directory listing failed after the first entry');

try {
$scanner->scan('');
} finally {
$this->assertFalse($connection->inTransaction());
}
}

public function testEmptyDirectoryScanDoesNotLeaveTransactionOpen(): void {
$storage = new LazyDirectoryStorage([]);
$scanner = new Scanner($storage);

$scanner->scan('');

$this->assertFalse(Server::get(IDBConnection::class)->inTransaction());
}

public function testShallow(): void {
$this->fillTestFolders();

Expand Down
Loading