From 847fab9d078543b5db059df4c440fb22f6b7ca21 Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Wed, 2 Sep 2026 11:53:18 +0000 Subject: [PATCH 1/3] fix(hstore): don't push sysprop-only range queries down to the store A sort-key prefix/range traversal reaches HstoreTable.queryByRange() with sysprop conditions only (owner vertex, direction, label, sort values) -- all already enforced by the scan key range. queryByRange() still pushed the serialized ConditionQuery down unconditionally, so the store tried to decode row property values it cannot parse (the server writes raw values, the store-side reader expects a self-describing (cardinality<<6)|dataType byte) and every such query failed with errors like: 'Can't construct Cardinality from code 0' / 'Unsupported data type UNKNOWN'. Apply the same prepareConditionQuery() guard that queryByPrefix() already uses: push the query down only when user-prop conditions remain. Also make both prepare methods operate on a copy of the origin query instead of mutating it via resetConditions() -- core still uses the origin query for its own result filtering after the scan returns. Part of #3090 (interim mitigation; the versioned sinking codec is tracked separately there). Co-Authored-By: Claude Fable 5 --- .../backend/store/hstore/HstoreTable.java | 31 ++- .../backend/store/hstore/HstoreTableTest.java | 248 ++++++++++++++++++ 2 files changed, 266 insertions(+), 13 deletions(-) diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index c6e0823288..8b26cf5c47 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -568,8 +568,11 @@ private ConditionQuery prepareConditionQuery(ConditionQuery conditionQuery) { } } if (newConditions.size() > 0) { - conditionQuery.resetConditions(newConditions); - return conditionQuery; + // NOTE: copy before reset, the origin query is still used by core + // for result filtering after the backend scan returns + ConditionQuery pushdown = conditionQuery.copy(); + pushdown.resetConditions(newConditions); + return pushdown; } else { return null; } @@ -594,8 +597,10 @@ private ConditionQuery prepareConditionQueryList(ConditionQuery conditionQuery) } } if (newConditions.size() > 0) { - conditionQuery.resetConditions(newConditions); - return conditionQuery; + // NOTE: copy before reset, see prepareConditionQuery() + ConditionQuery pushdown = conditionQuery.copy(); + pushdown.resetConditions(newConditions); + return pushdown; } else { return null; } @@ -642,16 +647,16 @@ protected BackendColumnIterator queryByRange(Session session, } if (origin instanceof ConditionQuery && (query.resultType().isEdge() || query.resultType().isVertex())) { - cq = (ConditionQuery) query.originQuery(); - - // LOG.debug("query {} with ownerKeyFrom: {}, ownerKeyTo: {}, " + - // "keyFrom: {}, keyTo: {}, " + - // "scanType: {}, conditionQuery: {}", - // this.table(), bytes2String(ownerStart), - // bytes2String(ownerEnd), bytes2String(start), - // bytes2String(end), type, cq.bytes()); + // Same guard as queryByPrefix(): only push the query down to the + // store when user-prop conditions remain. A sort-key prefix/range + // query keeps sysprop conditions only (owner vertex, direction, + // label, sort values), which are already enforced by the key + // range, and the store-side row decoder cannot parse the raw + // property layout written by the server (see issue #3090). + cq = prepareConditionQuery((ConditionQuery) origin); + byte[] queryBytes = cq == null ? null : cq.bytes(); return session.scan(this.table(), ownerStart, - ownerEnd, start, end, type, cq.bytes(), position); + ownerEnd, start, end, type, queryBytes, position); } return session.scan(this.table(), ownerStart, ownerEnd, start, end, type, null, position); diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java index b61b65a768..8f89f0ec07 100644 --- a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java @@ -18,21 +18,29 @@ package org.apache.hugegraph.backend.store.hstore; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hugegraph.backend.id.Id.IdType; import org.apache.hugegraph.backend.id.IdGenerator; import org.apache.hugegraph.backend.page.PageInfo; import org.apache.hugegraph.backend.page.PageState; +import org.apache.hugegraph.backend.query.Condition; +import org.apache.hugegraph.backend.query.ConditionQuery; import org.apache.hugegraph.backend.query.IdRangeQuery; import org.apache.hugegraph.backend.query.Query; import org.apache.hugegraph.backend.store.BackendEntry; import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; import org.apache.hugegraph.backend.store.BackendEntryIterator; +import org.apache.hugegraph.store.HgOwnerKey; import org.apache.hugegraph.store.client.util.HgStoreClientConst; import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.Directions; +import org.apache.hugegraph.type.define.GraphMode; +import org.apache.hugegraph.type.define.HugeKeys; import org.junit.Assert; import org.junit.Test; @@ -124,6 +132,59 @@ public void testRangeScanBudgetIncludesOneLookaheadRecord() { Assert.assertEquals(14L, HstoreTable.rangeScanBudget(query)); } + @Test + public void testRangeQueryWithoutUserpropsDoesNotPushConditions() { + // Sort-key prefix/range queries keep sysprop conditions only (owner + // vertex, direction, label, sort values); those are enforced by the + // key range already and must not be pushed to the store, whose row + // decoder cannot parse the server's raw property layout (issue #3090) + ConditionQuery origin = new ConditionQuery(HugeType.EDGE); + origin.eq(HugeKeys.OWNER_VERTEX, IdGenerator.of("v1")); + origin.eq(HugeKeys.DIRECTION, Directions.OUT); + origin.eq(HugeKeys.LABEL, IdGenerator.of(1L)); + origin.gte(HugeKeys.SORT_VALUES, "ETC!"); + origin.lt(HugeKeys.SORT_VALUES, "ETC~"); + int before = origin.conditions().size(); + + ScanRecordingSession session = new ScanRecordingSession(); + this.newTestTable().queryByRange(session, edgeRangeQuery(origin)); + + Assert.assertTrue(session.scanCalled); + Assert.assertNull(session.lastQueryBytes); + Assert.assertEquals(before, origin.conditions().size()); + } + + @Test + public void testRangeQueryWithUserpropsPushesCopyAndKeepsOrigin() { + ConditionQuery origin = new ConditionQuery(HugeType.EDGE); + origin.eq(HugeKeys.OWNER_VERTEX, IdGenerator.of("v1")); + origin.query(Condition.eq(IdGenerator.of(7L), 100)); + int before = origin.conditions().size(); + + ScanRecordingSession session = new ScanRecordingSession(); + this.newTestTable().queryByRange(session, edgeRangeQuery(origin)); + + Assert.assertTrue(session.scanCalled); + Assert.assertNotNull(session.lastQueryBytes); + // the pushed-down query is a copy: the origin query keeps all its + // conditions for core-side filtering after the scan returns + Assert.assertEquals(before, origin.conditions().size()); + } + + private HstoreTable newTestTable() { + HstoreTable table = new HstoreTable("hugegraph", "g+oe"); + table.ownerByQueryDelegate = (type, id) -> new byte[]{0}; + return table; + } + + private static IdRangeQuery edgeRangeQuery(ConditionQuery origin) { + return new IdRangeQuery(HugeType.EDGE_OUT, origin, + IdGenerator.of(keyBytes(1), IdType.STRING), + true, + IdGenerator.of(keyBytes(9), IdType.STRING), + false); + } + private static IdRangeQuery rangeIndexQuery() { return new IdRangeQuery(HugeType.RANGE_INT_INDEX, null, IdGenerator.of(keyBytes(1), IdType.STRING), @@ -139,6 +200,193 @@ private static byte[] keyBytes(int key) { return bytes; } + private static final class ScanRecordingSession extends HstoreSessions.Session { + + private boolean scanCalled = false; + private byte[] lastQueryBytes = null; + + @Override + public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, + byte[] ownerKeyTo, byte[] keyFrom, + byte[] keyTo, int scanType, + byte[] query, byte[] position) { + this.scanCalled = true; + this.lastQueryBytes = query; + return new TestColumnIterator(); + } + + @Override + public void open() { + } + + @Override + public void close() { + } + + @Override + public Object commit() { + return null; + } + + @Override + public void rollback() { + } + + @Override + public boolean hasChanges() { + return false; + } + + @Override + public void createTable(String tableName) { + } + + @Override + public void dropTable(String tableName) { + } + + @Override + public boolean existsTable(String tableName) { + return true; + } + + @Override + public void truncateTable(String tableName) { + } + + @Override + public void deleteGraph() { + } + + @Override + public Pair keyRange(String table) { + return null; + } + + @Override + public void put(String table, byte[] ownerKey, byte[] key, + byte[] value) { + } + + @Override + public void increase(String table, byte[] ownerKey, byte[] key, + byte[] value) { + } + + @Override + public void delete(String table, byte[] ownerKey, byte[] key) { + } + + @Override + public void deletePrefix(String table, byte[] ownerKey, byte[] key) { + } + + @Override + public void deleteRange(String table, byte[] ownerKeyFrom, + byte[] ownerKeyTo, byte[] keyFrom, + byte[] keyTo) { + } + + @Override + public byte[] get(String table, byte[] key) { + return new byte[0]; + } + + @Override + public byte[] get(String table, byte[] ownerKey, byte[] key) { + return new byte[0]; + } + + @Override + public BackendColumnIterator scan(String table) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator scan(String table, byte[] ownerKey, + byte[] prefix) { + throw new UnsupportedOperationException(); + } + + @Override + public List scan(String table, + List keys, + int scanType, long limit, + byte[] query) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendEntry.BackendIterator scan( + String table, Iterator keys, int scanType, + Query queryParam, byte[] query) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, + byte[] ownerKeyTo, byte[] keyFrom, + byte[] keyTo, int scanType) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, + byte[] ownerKeyTo, byte[] keyFrom, + byte[] keyTo, int scanType, + byte[] query) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator scan(String table, int codeFrom, + int codeTo, int scanType, + byte[] query) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator scan(String table, int codeFrom, + int codeTo, int scanType, + byte[] query, byte[] position) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator scan(String table, + byte[] conditionQueryToByte) { + throw new UnsupportedOperationException(); + } + + @Override + public BackendColumnIterator getWithBatch(String table, + List keys) { + throw new UnsupportedOperationException(); + } + + @Override + public void merge(String table, byte[] ownerKey, byte[] key, + byte[] value) { + } + + @Override + public void setMode(GraphMode mode) { + } + + @Override + public void truncate() throws Exception { + } + + @Override + public void beginTx() { + } + + @Override + public int getActiveStoreSize() { + return 0; + } + } + private static final class TestColumnIterator implements BackendColumnIterator { From 107287266895611d49c7dfa6e7e35166c2f4256e Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Wed, 2 Sep 2026 18:38:27 +0000 Subject: [PATCH 2/3] fix(hstore): clear pushdown back reference; assert pushed payload content in test Review follow-up (thanks @bitflicker64): - setOriginQuery(null) on the pushdown copy in both prepare methods -- the copy() back reference nested the origin query into the serialized payload (measured by reviewer: 2275 -> 657 bytes); nothing store-side reads it - test now decodes the pushed bytes via ConditionQuery.fromBytes and asserts the user-prop condition survives, OWNER_VERTEX is dropped and the back reference is cleared Co-Authored-By: Claude Fable 5 --- .../apache/hugegraph/backend/store/hstore/HstoreTable.java | 5 ++++- .../hugegraph/backend/store/hstore/HstoreTableTest.java | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index 8b26cf5c47..90d7a498a4 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -569,9 +569,11 @@ private ConditionQuery prepareConditionQuery(ConditionQuery conditionQuery) { } if (newConditions.size() > 0) { // NOTE: copy before reset, the origin query is still used by core - // for result filtering after the backend scan returns + // for result filtering after the backend scan returns; drop the + // back reference so the serialized payload stays flat ConditionQuery pushdown = conditionQuery.copy(); pushdown.resetConditions(newConditions); + pushdown.setOriginQuery(null); return pushdown; } else { return null; @@ -600,6 +602,7 @@ private ConditionQuery prepareConditionQueryList(ConditionQuery conditionQuery) // NOTE: copy before reset, see prepareConditionQuery() ConditionQuery pushdown = conditionQuery.copy(); pushdown.resetConditions(newConditions); + pushdown.setOriginQuery(null); return pushdown; } else { return null; diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java index 8f89f0ec07..1db4f5faae 100644 --- a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java @@ -169,6 +169,12 @@ public void testRangeQueryWithUserpropsPushesCopyAndKeepsOrigin() { // the pushed-down query is a copy: the origin query keeps all its // conditions for core-side filtering after the scan returns Assert.assertEquals(before, origin.conditions().size()); + // pushed payload: user-prop condition survives, owner-vertex is + // dropped, and the back reference to the origin query is cleared + ConditionQuery pushed = ConditionQuery.fromBytes(session.lastQueryBytes); + Assert.assertNull(pushed.condition(HugeKeys.OWNER_VERTEX)); + Assert.assertFalse(pushed.userpropConditions().isEmpty()); + Assert.assertNull(pushed.originQuery()); } private HstoreTable newTestTable() { From 0ecc10a3d03c8227e755c1f59841d98a7ead2a0d Mon Sep 17 00:00:00 2001 From: Sebastian Gruza Date: Thu, 3 Sep 2026 16:04:15 +0000 Subject: [PATCH 3/3] fix(hstore): single scan exit in queryByRange; cover prepareConditionQueryList via queryByPrefixList Review follow-ups for #3184: - queryByRange(): compute the pushed query bytes in one place and end with a single session.scan(...) call; the ConditionQuery is block-local now. - HstoreTableTest: ScanRecordingSession records the owner-key list scan instead of throwing, and testPrefixListQueryPushesCopyAndKeepsOrigin drives prepareConditionQueryList() through queryByPrefixList() with a shared origin query: the origin keeps all conditions (including OWNER_VERTEX), the pushed payload drops OWNER_VERTEX, keeps LABEL and the user-prop condition, and has no back reference. --- .../backend/store/hstore/HstoreTable.java | 12 ++-- .../backend/store/hstore/HstoreTableTest.java | 62 ++++++++++++++++--- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java index 90d7a498a4..77262b91ab 100755 --- a/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java +++ b/hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreTable.java @@ -631,7 +631,6 @@ protected BackendColumnIterator queryByRange(Session session, type |= query.inclusiveEnd() ? Session.SCAN_LTE_END : Session.SCAN_LT_END; } - ConditionQuery cq; Query origin = query.originQuery(); byte[] position = null; byte[] ownerStart = this.ownerByQueryDelegate.apply(query.resultType(), @@ -648,6 +647,7 @@ protected BackendColumnIterator queryByRange(Session session, if (query.paging() && !query.page().isEmpty()) { position = PageState.fromString(query.page()).position(); } + byte[] queryBytes = null; if (origin instanceof ConditionQuery && (query.resultType().isEdge() || query.resultType().isVertex())) { // Same guard as queryByPrefix(): only push the query down to the @@ -656,13 +656,11 @@ protected BackendColumnIterator queryByRange(Session session, // label, sort values), which are already enforced by the key // range, and the store-side row decoder cannot parse the raw // property layout written by the server (see issue #3090). - cq = prepareConditionQuery((ConditionQuery) origin); - byte[] queryBytes = cq == null ? null : cq.bytes(); - return session.scan(this.table(), ownerStart, - ownerEnd, start, end, type, queryBytes, position); + ConditionQuery cq = prepareConditionQuery((ConditionQuery) origin); + queryBytes = cq == null ? null : cq.bytes(); } - return session.scan(this.table(), ownerStart, - ownerEnd, start, end, type, null, position); + return session.scan(this.table(), ownerStart, ownerEnd, start, end, + type, queryBytes, position); } static boolean shouldUseOrderedRangeScan(IdRangeQuery query) { diff --git a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java index 1db4f5faae..2f374fccf1 100644 --- a/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java +++ b/hugegraph-server/hugegraph-hstore/src/test/java/org/apache/hugegraph/backend/store/hstore/HstoreTableTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.backend.store.hstore; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -29,6 +30,7 @@ import org.apache.hugegraph.backend.page.PageState; import org.apache.hugegraph.backend.query.Condition; import org.apache.hugegraph.backend.query.ConditionQuery; +import org.apache.hugegraph.backend.query.IdPrefixQuery; import org.apache.hugegraph.backend.query.IdRangeQuery; import org.apache.hugegraph.backend.query.Query; import org.apache.hugegraph.backend.store.BackendEntry; @@ -177,6 +179,42 @@ public void testRangeQueryWithUserpropsPushesCopyAndKeepsOrigin() { Assert.assertNull(pushed.originQuery()); } + @Test + public void testPrefixListQueryPushesCopyAndKeepsOrigin() { + // prepareConditionQueryList() is reached from queryByPrefixList() and + // from the streaming query(Session, Iterator, String); one origin + // query is shared by every prefix query of the batch + ConditionQuery origin = new ConditionQuery(HugeType.EDGE); + origin.eq(HugeKeys.OWNER_VERTEX, IdGenerator.of("v1")); + origin.eq(HugeKeys.DIRECTION, Directions.OUT); + origin.eq(HugeKeys.LABEL, IdGenerator.of(1L)); + origin.query(Condition.eq(IdGenerator.of(7L), 100)); + int before = origin.conditions().size(); + List queries = Arrays.asList( + new IdPrefixQuery(origin, IdGenerator.of(keyBytes(1), + IdType.STRING)), + new IdPrefixQuery(origin, IdGenerator.of(keyBytes(2), + IdType.STRING))); + + ScanRecordingSession session = new ScanRecordingSession(); + List iterators = this.newTestTable() + .queryByPrefixList(session, queries, "g+oe"); + + Assert.assertTrue(session.scanCalled); + Assert.assertEquals(2, session.lastOwnerKeys.size()); + Assert.assertEquals(2, iterators.size()); + Assert.assertNotNull(session.lastQueryBytes); + // the shared origin query keeps every condition, including the + // owner vertex that the pushed copy drops + Assert.assertEquals(before, origin.conditions().size()); + Assert.assertNotNull(origin.condition(HugeKeys.OWNER_VERTEX)); + ConditionQuery pushed = ConditionQuery.fromBytes(session.lastQueryBytes); + Assert.assertNull(pushed.condition(HugeKeys.OWNER_VERTEX)); + Assert.assertNotNull(pushed.condition(HugeKeys.LABEL)); + Assert.assertFalse(pushed.userpropConditions().isEmpty()); + Assert.assertNull(pushed.originQuery()); + } + private HstoreTable newTestTable() { HstoreTable table = new HstoreTable("hugegraph", "g+oe"); table.ownerByQueryDelegate = (type, id) -> new byte[]{0}; @@ -210,6 +248,7 @@ private static final class ScanRecordingSession extends HstoreSessions.Session { private boolean scanCalled = false; private byte[] lastQueryBytes = null; + private List lastOwnerKeys = null; @Override public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, @@ -221,6 +260,21 @@ public BackendColumnIterator scan(String table, byte[] ownerKeyFrom, return new TestColumnIterator(); } + @Override + public List scan(String table, + List keys, + int scanType, long limit, + byte[] query) { + this.scanCalled = true; + this.lastQueryBytes = query; + this.lastOwnerKeys = keys; + List iterators = new ArrayList<>(); + for (int i = 0; i < keys.size(); i++) { + iterators.add(new TestColumnIterator()); + } + return iterators; + } + @Override public void open() { } @@ -314,14 +368,6 @@ public BackendColumnIterator scan(String table, byte[] ownerKey, throw new UnsupportedOperationException(); } - @Override - public List scan(String table, - List keys, - int scanType, long limit, - byte[] query) { - throw new UnsupportedOperationException(); - } - @Override public BackendEntry.BackendIterator scan( String table, Iterator keys, int scanType,