Add ALL and EXACTLY_ONE array filter operators for MongoDB and Postgres

document-store/src/integrationTest/resources/query/array_operators/array_match_test.json

@@ -0,0 +1,36 @@
+[
+  {
+    "_id": 1,
+    "name": "Document A",
+    "tags": ["red", "blue"]
+  },
+  {
+    "_id": 2,
+    "name": "Document B",
+    "tags": ["red", "blue", "green"]
+  },
+  {
+    "_id": 3,
+    "name": "Document C",
+    "tags": ["red"]
+  },
+  {
+    "_id": 4,
+    "name": "Document D",
+    "tags": ["yellow"]
+  },
+  {
+    "_id": 5,
+    "name": "Document E",
+    "tags": []
+  },
+  {
+    "_id": 6,
+    "name": "Document F"
+  },
+  {
+    "_id": 7,
+    "name": "Document G",
+    "tags": ["red", "red"]
+  }
+]

document-store/src/integrationTest/resources/query/array_operators/nested_array_elements_test.json

@@ -0,0 +1,5 @@
+[
+  { "_id": 1, "name": "NestedMulti", "tags": [["red", "blue"], "green"] },
+  { "_id": 2, "name": "ScalarSingle", "tags": ["red"] },
+  { "_id": 3, "name": "NestedSingle", "tags": [["red"]] }
+]

document-store/src/integrationTest/resources/query/array_operators/nested_array_match_test.json

@@ -0,0 +1,58 @@
+[
+  {
+    "_id": 1,
+    "name": "Document A",
+    "props": {
+      "metadata": {
+        "colors": ["red", "blue"]
+      }
+    }
+  },
+  {
+    "_id": 2,
+    "name": "Document B",
+    "props": {
+      "metadata": {
+        "colors": ["red", "blue", "green"]
+      }
+    }
+  },
+  {
+    "_id": 3,
+    "name": "Document C",
+    "props": {
+      "metadata": {
+        "colors": ["red"]
+      }
+    }
+  },
+  {
+    "_id": 4,
+    "name": "Document D",
+    "props": {
+      "metadata": {
+        "colors": ["yellow"]
+      }
+    }
+  },
+  {
+    "_id": 5,
+    "name": "Document E",
+    "props": {
+      "metadata": {
+        "colors": []
+      }
+    }
+  },
+  {
+    "_id": 6,
+    "name": "Document F",
+    "props": {
+      "metadata": {}
+    }
+  },
+  {
+    "_id": 7,
+    "name": "Document G"
+  }
+]

document-store/src/main/java/org/hypertrace/core/documentstore/expression/impl/ArrayRelationalFilterExpression.java

@@ -11,7 +11,23 @@
 import org.hypertrace.core.documentstore.parser.FilterTypeExpressionVisitor;
 
 /**
- * Expression representing a condition for filtering on array fields
+ * Expression representing a condition for filtering on array fields.
+ *
+ * <p>When the inner filter uses {@code IN}, that operator is <em>element membership</em>, not array
+ * equality. Combined with {@link ArrayOperator}:
+ *
+ * <ul>
+ *   <li>{@code ALL}: the RHS set is a subset of the stored array (every listed value appears in
+ *       the array).
+ *   <li>{@code EXACTLY_ONE}: the stored array has length 1 and that element is in the RHS set.
+ * </ul>
+ *
+ * Order and duplicates are ignored (set semantics).
+ *
+ * <p>The inner {@code IN} RHS is a scalar list — arrays of arrays cannot be expressed. The
+ * operators apply to the outermost array only: a stored element that is itself an array is opaque
+ * and never matches a scalar RHS value, and length/cardinality checks count top-level elements
+ * only.
  *
  * <p>Example: If color is an array field <code>
  * ANY(color) IN ('Blue', 'Green')

document-store/src/main/java/org/hypertrace/core/documentstore/expression/operators/ArrayOperator.java

@@ -2,5 +2,29 @@
 
 public enum ArrayOperator {
   ANY,
-  // Can support ALL and NONE later
+  /**
+   * With an inner {@code IN} filter, {@code IN} is element membership (not array equality). {@code
+   * ALL} means the RHS set is a subset of the stored array: every listed value appears in the
+   * array. Order and duplicates are ignored (set semantics), e.g. {@code [red, red] ALL [red]} is
+   * true.
+   *
+   * <p>The RHS is a scalar list — arrays of arrays cannot be expressed. The operator applies to
+   * the outermost array only: a stored element that is itself an array is opaque and never matches
+   * a scalar RHS value.
+   */
+  ALL,
+  /**
+   * With an inner {@code IN} filter, {@code IN} is element membership (not array equality). {@code
+   * EXACTLY_ONE} means the stored array has length 1 <em>and</em> that single element is in the
+   * RHS set. Order and duplicates in the RHS are ignored (set semantics). The cardinality check is
+   * on the raw element count, not distinct values, e.g. {@code [red, red] EXACTLY_ONE [red]} is
+   * false because the array has two elements.
+   *
+   * <p>The RHS is a scalar list — arrays of arrays cannot be expressed. The operator applies to
+   * the outermost array only: a stored element that is itself an array is opaque and never matches
+   * a scalar RHS value, and the length check counts top-level elements only.
+   */
+  EXACTLY_ONE,
+  // Future consideration: an EXACTLY operator for set equality - the array contains exactly the
+  // filter values, no more and no less.
 }

document-store/src/main/java/org/hypertrace/core/documentstore/mongo/query/parser/MongoArrayFilterParser.java

@@ -6,10 +6,14 @@
 import static org.hypertrace.core.documentstore.mongo.query.parser.filter.MongoStandardExprRelationalFilterParser.EXPR;
 
 import com.google.common.collect.Maps;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import org.hypertrace.core.documentstore.expression.impl.ArrayFilterExpression;
+import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;
+import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
 import org.hypertrace.core.documentstore.expression.operators.ArrayOperator;
+import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression;
 import org.hypertrace.core.documentstore.expression.type.SelectTypeExpression;
 import org.hypertrace.core.documentstore.mongo.MongoUtils;
 import org.hypertrace.core.documentstore.mongo.query.parser.filter.MongoRelationalFilterParserFactory.MongoRelationalFilterContext;
@@ -19,8 +23,16 @@ class MongoArrayFilterParser {
   private static final String MAP = "$map";
   private static final String INPUT = "input";
   private static final String IF_NULL = "$ifNull";
+  private static final String COND = "$cond";
+  private static final String IS_ARRAY = "$isArray";
   private static final String AS = "as";
   private static final String IN = "in";
+  private static final String SET_IS_SUBSET = "$setIsSubset";
+  private static final String SIZE = "$size";
+  private static final String EQ = "$eq";
+  private static final String IN_OPERATOR = "$in";
+  private static final String ARRAY_ELEM_AT = "$arrayElemAt";
+  private static final String AND = "$and";
 
   private static final Map<ArrayOperator, String> OPERATOR_MAP =
       Maps.immutableEnumMap(Map.ofEntries(entry(ANY, ANY_ELEMENT_TRUE)));
@@ -39,6 +51,17 @@ class MongoArrayFilterParser {
   }
 
   Map<String, Object> parse(final ArrayFilterExpression arrayFilterExpression) {
+    switch (arrayFilterExpression.getOperator()) {
+      case ALL:
+        return parseAllOperator(arrayFilterExpression);
+      case EXACTLY_ONE:
+        return parseOneOperator(arrayFilterExpression);
+      default:
+        return parseAnyOperator(arrayFilterExpression);
+    }
+  }
+
+  private Map<String, Object> parseAnyOperator(final ArrayFilterExpression arrayFilterExpression) {
     final String operator =
         Optional.ofNullable(OPERATOR_MAP.get(arrayFilterExpression.getOperator()))
             .orElseThrow(
@@ -103,9 +126,90 @@ Map<String, Object> parse(final ArrayFilterExpression arrayFilterExpression) {
                     entry(INPUT, Map.of(IF_NULL, new Object[] {mapInput, new Object[0]})),
                     entry(AS, alias),
                     entry(IN, filter))));
+    return wrapInExprIfNeeded(arrayFilter);
+  }
+
+  /*
+  {
+    "$expr": {
+      "$setIsSubset": [
+        ["Blue", "Green"],
+        { "$cond": [{ "$isArray": "$colors" }, "$colors", []] }
+      ]
+    }
+  }
+   */
+  private Map<String, Object> parseAllOperator(final ArrayFilterExpression arrayFilterExpression) {
+    final Object mapInput = getDollarPrefixedArraySource(arrayFilterExpression);
+    final List<?> values = getFilterValues(arrayFilterExpression);
+
+    final Map<String, Object> setIsSubset =
+        Map.of(SET_IS_SUBSET, List.of(values, arrayOrEmpty(mapInput)));
+    return wrapInExprIfNeeded(setIsSubset);
+  }
+
+  /*
+  {
+    "$expr": {
+      "$and": [
+        { "$eq": [{ "$size": { "$cond": [{ "$isArray": "$colors" }, "$colors", []] } }, 1] },
+        { "$in": [{ "$arrayElemAt": [{ "$cond": [{ "$isArray": "$colors" }, "$colors", []] }, 0] }, ["Blue", "Green"]] }
+      ]
+    }
+  }
+   */
+  private Map<String, Object> parseOneOperator(final ArrayFilterExpression arrayFilterExpression) {
+    final Object mapInput = getDollarPrefixedArraySource(arrayFilterExpression);
+    final List<?> values = getFilterValues(arrayFilterExpression);
+    final Map<String, Object> arrayWithDefault = arrayOrEmpty(mapInput);
+
+    final Map<String, Object> sizeIsOne = Map.of(EQ, List.of(Map.of(SIZE, arrayWithDefault), 1));
+    final Map<String, Object> firstElementMatches =
+        Map.of(IN_OPERATOR, List.of(Map.of(ARRAY_ELEM_AT, List.of(arrayWithDefault, 0)), values));
+
+    return wrapInExprIfNeeded(Map.of(AND, List.of(sizeIsOne, firstElementMatches)));
+  }
+
+  /*
+   * Guards against missing, null and non-array (e.g. scalar) field values: $setIsSubset/$size
+   * error out on a non-array operand, whereas Postgres simply does not match such documents.
+   * $isArray is false for null/missing values, so this also subsumes $ifNull.
+   */
+  private Map<String, Object> arrayOrEmpty(final Object mapInput) {
+    return Map.of(COND, List.of(Map.of(IS_ARRAY, mapInput), mapInput, List.of()));
+  }
+
+  private String getDollarPrefixedArraySource(final ArrayFilterExpression arrayFilterExpression) {
+    final MongoSelectTypeExpressionParser wrappingParser =
+        new MongoDollarPrefixingIdempotentParser(relationalFilterContext.lhsParser());
+    return arrayFilterExpression.getArraySource().accept(wrappingParser);
+  }
+
+  private List<?> getFilterValues(final ArrayFilterExpression arrayFilterExpression) {
+    final FilterTypeExpression filter = arrayFilterExpression.getFilter();
+    if (!(filter instanceof RelationalExpression)) {
+      throw new UnsupportedOperationException(
+          "Array operator "
+              + arrayFilterExpression.getOperator()
+              + " only supports a relational filter with a constant list of values, got: "
+              + filter);
+    }
+
+    final SelectTypeExpression rhs = ((RelationalExpression) filter).getRhs();
+    if (!(rhs instanceof ConstantExpression)) {
+      throw new UnsupportedOperationException(
+          "Array operator "
+              + arrayFilterExpression.getOperator()
+              + " requires a constant list of values, got: "
+              + rhs);
+    }
+
+    final Object value = ((ConstantExpression) rhs).getValue();
+    return value instanceof List ? (List<?>) value : List.of(value);
+  }
+
+  private Map<String, Object> wrapInExprIfNeeded(final Map<String, Object> filter) {
     // If already wrapped inside `$expr` avoid wrapping again
-    return INSIDE_EXPR.equals(relationalFilterContext.location())
-        ? arrayFilter
-        : Map.of(EXPR, arrayFilter);
+    return INSIDE_EXPR.equals(relationalFilterContext.location()) ? filter : Map.of(EXPR, filter);
   }
 }