[FEATURE] Ordered UIDs filter — return collections in externally reso…

Classes/Domain/Repository/CommonRepository.php

@@ -5,8 +5,11 @@
 namespace SourceBroker\T3api\Domain\Repository;
 
 use SourceBroker\T3api\Domain\Model\ApiFilter;
+use SourceBroker\T3api\Domain\Model\CollectionOperation;
 use SourceBroker\T3api\Domain\Model\OperationInterface;
+use SourceBroker\T3api\Exception\MissingCollectionOperationException;
 use SourceBroker\T3api\Filter\FilterInterface;
+use SourceBroker\T3api\Filter\QueryModifierInterface;
 use SourceBroker\T3api\Security\FilterAccessChecker;
 use SourceBroker\T3api\Service\StorageService;
 use Symfony\Component\HttpFoundation\Request;
@@ -89,6 +92,7 @@ public function findFiltered(array $apiFilters, Request $request): QueryInterfac
 
         $query = $this->createQuery();
         $constraintGroups = [];
+        $queryModifiers = [];
 
         $apiFilters = $this->filterGrantedFilters($apiFilters);
         $apiFilters = $this->filterAndSortApiFiltersByQueryParams($apiFilters, $queryParams);
@@ -111,6 +115,10 @@ public function findFiltered(array $apiFilters, Request $request): QueryInterfac
                     [$constraint]
                 );
             }
+
+            if ($filter instanceof QueryModifierInterface) {
+                $queryModifiers[] = [$filter, $apiFilter];
+            }
         }
 
         $constraints = [];
@@ -122,6 +130,26 @@ public function findFiltered(array $apiFilters, Request $request): QueryInterfac
             $query->matching($query->logicalAnd(...$constraints));
         }
 
+        // Let query-modifier filters adjust the fully-constrained query — e.g. apply an ORDER BY
+        // the QOM constraint model cannot express (ORDER BY FIELD(uid, ...) for a relevance
+        // ranking). Runs after matching() so the query already carries every constraint.
+        if ($queryModifiers !== []) {
+            $operation = $this->operation ?? null;
+            if (!$operation instanceof CollectionOperation) {
+                throw new MissingCollectionOperationException(
+                    sprintf(
+                        'Query modifiers require a repository built for a collection operation, `%s` given.',
+                        get_debug_type($operation)
+                    ),
+                    1784102345678
+                );
+            }
+
+            foreach ($queryModifiers as [$filter, $apiFilter]) {
+                $filter->modifyQuery($query, $apiFilter, $operation);
+            }
+        }
+
         return $query;
     }
 

Classes/Exception/MissingCollectionOperationException.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+namespace SourceBroker\T3api\Exception;
+
+/**
+ * Thrown when query-modifier filters are applied on a repository that was not built for
+ * a collection operation (see CommonRepository::findFiltered()).
+ */
+final class MissingCollectionOperationException extends \RuntimeException {}

Classes/Filter/AbstractOrderedUidsFilter.php

@@ -0,0 +1,118 @@
+<?php
+
+declare(strict_types=1);
+
+namespace SourceBroker\T3api\Filter;
+
+use SourceBroker\T3api\Domain\Model\ApiFilter;
+use SourceBroker\T3api\Domain\Model\CollectionOperation;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Persistence\Generic\Qom\ConstraintInterface;
+use TYPO3\CMS\Extbase\Persistence\Generic\Query;
+use TYPO3\CMS\Extbase\Persistence\Generic\Storage\Typo3DbQueryParser;
+use TYPO3\CMS\Extbase\Persistence\QueryInterface;
+
+/**
+ * Base for filters that resolve the filter value to an ordered list of record UIDs — e.g. a
+ * relevance-ranked result from an external search engine, a recommendation service or a curated
+ * list — and want that order reflected in the collection result.
+ *
+ * A concrete filter only implements resolveOrderedUids(). This base then:
+ *  - constrains the collection to those UIDs (uid IN (...)) via filterProperty(), and
+ *  - as a QueryModifierInterface, re-applies the order as ORDER BY FIELD(uid, ...) once t3api has
+ *    combined every filter's constraints (Extbase's ordering API cannot express FIELD()).
+ *
+ * The FIELD ordering is handed to the query as a Doctrine QueryBuilder statement; it survives
+ * pagination and the count query because AbstractCollectionResponse gives the paginated query its
+ * own QueryBuilder clone.
+ *
+ * Several such filters compose: each one's uid IN (...) constraint applies (the collection is the
+ * intersection of all matches), the first modifier's ranking becomes the primary ordering and the
+ * following ones are appended as tie-breakers on the already-present QueryBuilder statement.
+ */
+abstract class AbstractOrderedUidsFilter extends AbstractFilter implements QueryModifierInterface
+{
+    /**
+     * Keyed by the ApiFilter declaration (spl_object_id): filters are singletons, so a resource
+     * declaring one filter class under several parameter names shares a single instance —
+     * per-declaration keying keeps the resolved UID lists from overwriting each other between
+     * filterProperty() and modifyQuery().
+     *
+     * @var array<int, int[]>
+     */
+    private array $orderedUidsPerDeclaration = [];
+
+    /**
+     * Resolve the matching record UIDs, in the order they should appear, for the filter value(s).
+     *
+     * @param mixed $values the raw filter value(s) from the request
+     * @return int[]|null null → no lookup performed (filter inactive, matches everything);
+     *                    [] → lookup performed but nothing matched (matches nothing);
+     *                    int[] → matching UIDs in the order to apply
+     */
+    abstract protected function resolveOrderedUids($values, ApiFilter $apiFilter): ?array;
+
+    public function filterProperty(
+        string $property,
+        $values,
+        QueryInterface $query,
+        ApiFilter $apiFilter
+    ): ?ConstraintInterface {
+        $uids = $this->resolveOrderedUids($values, $apiFilter);
+        if ($uids === null) {
+            unset($this->orderedUidsPerDeclaration[spl_object_id($apiFilter)]);
+
+            return null;
+        }
+
+        $this->orderedUidsPerDeclaration[spl_object_id($apiFilter)] = $uids;
+
+        // The appended 0 guards the empty case: in('uid', []) is invalid SQL; uid 0 never matches.
+        return $query->in('uid', array_merge($uids, [0]));
+    }
+
+    public function modifyQuery(QueryInterface $query, ApiFilter $apiFilter, CollectionOperation $operation): void
+    {
+        $orderedUids = $this->orderedUidsPerDeclaration[spl_object_id($apiFilter)] ?? [];
+        unset($this->orderedUidsPerDeclaration[spl_object_id($apiFilter)]);
+
+        if ($orderedUids === [] || !$query instanceof Query) {
+            return;
+        }
+
+        $statement = $query->getStatement();
+        if ($statement !== null && $statement->getStatement() instanceof QueryBuilder) {
+            // An earlier query modifier already rewrote this query into a QueryBuilder statement.
+            // Reconverting the QOM would silently discard its work — append this ordering to the
+            // existing builder instead, as a tie-breaker after the earlier modifier's ordering.
+            $queryBuilder = $statement->getStatement();
+            $queryBuilder->getConcreteQueryBuilder()
+                ->addOrderBy($this->buildFieldOrderExpression($queryBuilder, $query, $orderedUids));
+
+            return;
+        }
+
+        $queryBuilder = GeneralUtility::makeInstance(Typo3DbQueryParser::class)
+            ->convertQueryToDoctrineQueryBuilder($query);
+
+        // The ordering goes to the concrete Doctrine QueryBuilder: TYPO3's facade would quote the
+        // whole FIELD() expression as an identifier, and orderBy() on the concrete builder also
+        // replaces any ordering Typo3DbQueryParser derived from the Extbase query.
+        $queryBuilder->getConcreteQueryBuilder()
+            ->orderBy($this->buildFieldOrderExpression($queryBuilder, $query, $orderedUids));
+
+        $query->statement($queryBuilder);
+    }
+
+    /**
+     * @param int[] $orderedUids
+     */
+    private function buildFieldOrderExpression(QueryBuilder $queryBuilder, Query $query, array $orderedUids): string
+    {
+        return 'FIELD('
+            . $queryBuilder->quoteIdentifier($this->getTableName($query) . '.uid') . ', '
+            . implode(',', array_map('intval', $orderedUids))
+            . ')';
+    }
+}

Classes/Filter/FilterInterface.php

@@ -10,6 +10,12 @@
 
 interface FilterInterface
 {
+    /**
+     * Planned for the next major version (v6): a CollectionOperation parameter will be added here
+     * (as in QueryModifierInterface::modifyQuery()), mirroring API Platform's filter contract
+     * where apply() receives the Operation. It cannot happen within 5.x — it would break every
+     * existing custom filter implementation.
+     */
     public function filterProperty(
         string $property,
         $values,

Classes/Filter/QueryModifierInterface.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace SourceBroker\T3api\Filter;
+
+use SourceBroker\T3api\Domain\Model\ApiFilter;
+use SourceBroker\T3api\Domain\Model\CollectionOperation;
+use TYPO3\CMS\Extbase\Persistence\QueryInterface;
+
+/**
+ * A filter that, in addition to (or instead of) contributing a constraint via
+ * FilterInterface::filterProperty(), post-processes the whole query after every filter's
+ * constraints have been combined and applied (see CommonRepository::findFiltered()).
+ *
+ * This is the seam for adjustments the QOM constraint model cannot express — most notably a
+ * custom ORDER BY such as ORDER BY FIELD(uid, ...) for a relevance ranking, which needs the
+ * query to already carry all constraints. Filters implementing this run in a second pass, in the
+ * same (request-driven) order as filterProperty().
+ *
+ * Composability caveat: adjustments made through the QOM API compose across modifiers, but a
+ * modifier that ends in $query->statement(...) effectively replaces the whole query — Extbase
+ * executes the statement and ignores QOM-level changes made by later modifiers, and a later
+ * statement(...) call overwrites an earlier one (last one wins). A statement-based modifier
+ * should therefore check $query->getStatement() first and mutate the QueryBuilder already
+ * present instead of reconverting, as AbstractOrderedUidsFilter does.
+ */
+interface QueryModifierInterface
+{
+    public function modifyQuery(QueryInterface $query, ApiFilter $apiFilter, CollectionOperation $operation): void;
+}

Classes/Response/AbstractCollectionResponse.php

@@ -7,6 +7,8 @@
 use GoldSpecDigital\ObjectOrientedOAS\Objects\Schema;
 use SourceBroker\T3api\Domain\Model\CollectionOperation;
 use Symfony\Component\HttpFoundation\Request;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
+use TYPO3\CMS\Extbase\Persistence\Generic\Query;
 use TYPO3\CMS\Extbase\Persistence\QueryInterface;
 
 abstract class AbstractCollectionResponse
@@ -56,7 +58,26 @@ protected function applyPagination(): QueryInterface
             return $this->query;
         }
 
-        return (clone $this->query)
+        $paginatedQuery = clone $this->query;
+
+        // When the query is backed by a raw Doctrine QueryBuilder statement (e.g. a custom
+        // filter or operation handler that needs an ORDER BY the Extbase QOM cannot express,
+        // such as ORDER BY FIELD(uid, ...) for a relevance ranking), the shallow clone above
+        // still shares the very same QueryBuilder instance with $this->query. Extbase applies
+        // this (paginated) query's limit/offset onto that shared builder at execution time,
+        // and it then leaks into the separate, unpaginated count() query used by
+        // getTotalItems() — turning its "SELECT COUNT(*) ... LIMIT n OFFSET m" into an empty
+        // result, because the single count row is skipped by the offset. Give the paginated
+        // query its own clone of the builder so the limit/offset stay isolated to the members
+        // query and never reach the count query. No-op for normal QOM queries (no statement).
+        if ($paginatedQuery instanceof Query) {
+            $statement = $paginatedQuery->getStatement();
+            if ($statement !== null && $statement->getStatement() instanceof QueryBuilder) {
+                $paginatedQuery->statement(clone $statement->getStatement(), $statement->getBoundVariables());
+            }
+        }
+
+        return $paginatedQuery
             ->setLimit($pagination->getNumberOfItemsPerPage())
             ->setOffset($pagination->getOffset());
     }

Classes/Service/SerializerMetadataService.php

@@ -35,12 +35,15 @@ public static function generateAutoloadForClass(string $class): void
         }
 
         foreach (self::getClassHierarchy($class) as $reflectionClass) {
-            if (in_array($reflectionClass->getName(), self::$runtimeGeneratedCache, true)) {
+            $generatedMetadataFile = self::getAutogeneratedFilePath($reflectionClass->getName());
+
+            // Keyed by target file, not class name: the file path contains Environment::getVarPath(),
+            // which changes per test instance while this static cache spans the whole PHP process -
+            // a class-name key would wrongly skip generation into a fresh var path.
+            if (in_array($generatedMetadataFile, self::$runtimeGeneratedCache, true)) {
                 continue;
             }
 
-            $generatedMetadataFile = self::getAutogeneratedFilePath($reflectionClass->getName());
-
             $classMergedMetadata = array_replace_recursive(
                 self::getForClass($reflectionClass),
                 self::getMetadataFromMetadataDirs($reflectionClass->getName())
@@ -53,7 +56,7 @@ public static function generateAutoloadForClass(string $class): void
             );
             rename($tmpFile, $generatedMetadataFile);
 
-            self::$runtimeGeneratedCache[] = $reflectionClass->getName();
+            self::$runtimeGeneratedCache[] = $generatedMetadataFile;
         }
     }
 

Documentation/Filtering/CustomFilters/Index.rst

@@ -66,6 +66,42 @@ It may be useful, but not required, to extend class ``\SourceBroker\T3api\Filter
 .. note::
     Instance of your custom filter will be created using Extbase's ObjectManager, so you can inject into it any other services if you need them.
 
+.. _filtering_custom-filters_query-modifiers:
+
+Query modifiers
+===============
+
+A constraint returned from ``filterProperty`` is sometimes not enough — for example when the filter needs to apply a custom ``ORDER BY`` which has to take the complete query into account. For such cases a custom filter can additionally implement ``\SourceBroker\T3api\Filter\QueryModifierInterface``:
+
+.. code-block:: php
+
+   interface QueryModifierInterface
+   {
+       public function modifyQuery(QueryInterface $query, ApiFilter $apiFilter, CollectionOperation $operation): void;
+   }
+
+Method ``modifyQuery`` accepts 3 arguments:
+
+- $query (``TYPO3\CMS\Extbase\Persistence\QueryInterface``) - Instance of Extbase's query, already carrying the combined constraints of all filters.
+- $apiFilter (``SourceBroker\T3api\Domain\Model\ApiFilter``) - The filter declaration being applied.
+- $operation (``SourceBroker\T3api\Domain\Model\CollectionOperation``) - The collection operation being served, so the filter can act per endpoint (path, resource, attributes).
+
+After the constraints of all filters have been combined and applied to the query, t3api calls ``modifyQuery`` on every filter implementing this interface. The filters run in the same request-driven order as ``filterProperty``. Query modifiers only ever run for collection ``GET`` operations — the same scope in which filters apply at all.
+
+.. important::
+    Modifications done through Extbase's query API compose across multiple modifiers. A modifier
+    which replaces the query with a Doctrine QueryBuilder statement (``$query->statement(...)``)
+    does not: Extbase then executes only the statement, so QOM-level changes made by later
+    modifiers are ignored, and a later ``statement()`` call overwrites an earlier one (last one
+    wins). A statement-based modifier should check ``$query->getStatement()`` first and mutate
+    the QueryBuilder already present instead of reconverting — see
+    ``\SourceBroker\T3api\Filter\AbstractOrderedUidsFilter`` for a reference implementation.
+
+A ready-made base class using this seam is :ref:`AbstractOrderedUidsFilter <filtering_ordered-uids-filter>`, which returns a collection in the order of an externally resolved UID list (e.g. search results ordered by relevance).
+
+Registration
+============
+
 To use your new custom filter is it just needed to pass it as first parameter into ``@ApiFilter`` annotation:
 
 .. code-block:: php

Documentation/Filtering/Index.rst

@@ -117,6 +117,7 @@ There are few build-in filters. See the next pages.
 
    BuiltinFilters/Index
    CustomFilters/Index
+   OrderedUidsFilter/Index
    SqlInOperator/Index
    SqlOrOperator/Index