From 652c16494555c51358ac9f2972076587bb2f3c0d Mon Sep 17 00:00:00 2001 From: u70b3 Date: Thu, 27 Aug 2026 12:20:24 +0000 Subject: [PATCH 1/4] [feature](lance) Add durable Lance index job model and state machine Second sub-PR (PR3B) of slice 3 of the Lance index lifecycle design (apache/doris#66497, v5.1 contract): the durable job model behind the one-shot mutation lifecycle. No user-visible entry point; admission, dispatch, and FORCE land in follow-up PRs. Model pieces (design sections in parentheses): - LanceIndexJob: the minimal durable job record (7.2) - identity, creator, revision, bounded timestamps, persisted target identity and same-name fence key material, mutation intent, admitted dataset version with the ordered schema-contract-v1 representation (4.2), independent mutation/refresh states, typed result with bounded sanitized message, dispatch identity (backend id, BE process epoch, immutable invocation id, deadline), possible-live ownership with termination proof, and the FORCE audit fields (populated by PR3E). No credentials, no unbounded values (4.3/8). - Dual state machines: PENDING -> RUNNING -> COMMITTED|NOT_COMMITTED| UNKNOWN (6.1, UNKNOWN terminal with no outgoing transition) and the independent NOT_REQUIRED|REQUIRED|RUNNING|DONE|FAILED refresh state (6.2). - LanceIndexJobResultCode: the provider-result classification table (6.3) as typed codes plus one pure classify(); IF_CONDITION_NOOP only for DROP IF EXISTS + LANCE_ERR_NOT_FOUND. - Normalization v1 (4.1): index names via toLowerCase(Locale.ROOT); dataset locators via trim, lowercased scheme, trailing-slash strip, and rejection of credential-bearing or identity-less forms. - LanceIndexFenceKey: (catalog id, DIRECTORY provider, normalized locator, normalized index name); display name is persisted on the job, never in the key. toString hides the locator. --- .../lance/job/LanceIndexDatasetLocator.java | 109 ++++ .../lance/job/LanceIndexFenceKey.java | 92 +++ .../datasource/lance/job/LanceIndexJob.java | 616 ++++++++++++++++++ .../job/LanceIndexJobCompletionReason.java | 28 + .../lance/job/LanceIndexJobMutationState.java | 47 ++ .../lance/job/LanceIndexJobMutationType.java | 28 + .../lance/job/LanceIndexJobRefreshState.java | 41 ++ .../lance/job/LanceIndexJobResult.java | 112 ++++ .../lance/job/LanceIndexJobResultCode.java | 158 +++++ .../lance/job/LanceIndexNameNormalizer.java | 69 ++ .../lance/job/LanceIndexSchemaContract.java | 194 ++++++ .../lance/job/LanceIndexTerminationProof.java | 33 + 12 files changed, 1527 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java new file mode 100644 index 00000000000000..bed9a2dff04970 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import java.util.Locale; + +/** + * Dataset locator normalization v1 for the durable fence key. The rules, in + * order: + * + *
    + *
  1. trim surrounding whitespace;
    2. + *
  2. if a {@code scheme://} prefix is present, lowercase the scheme + * (aligned with the {@code LanceStorageProvider.schemeOf} precedent);
    3. + *
  3. a URL whose authority carries userinfo is rejected: credential-bearing + * URLs are never identity;
    4. + *
  4. a locator with a scheme but neither an authority nor a path (for + * example {@code "s3://"}) carries no identity and is rejected; an empty + * authority with a non-empty path ({@code "file:///x"}) stays legal;
    5. + *
  5. trailing {@code '/'} characters are removed, keeping the root + * (a scheme-less {@code "/"} stays {@code "/"});
    6. + *
  6. without a scheme the locator must be an absolute path (start with + * {@code '/'}), otherwise it is rejected.
    7. + *
+ * + *

The authority (host/bucket) and path keep their original case: bucket and + * path components are case-sensitive on the providers Doris supports, and + * normalization v1 deliberately does not define cross-alias equivalence. URI + * aliases and external writers replacing the dataset at the same URI are + * outside Doris serialization. + */ +public final class LanceIndexDatasetLocator { + private static final String SCHEME_SEPARATOR = "://"; + + private LanceIndexDatasetLocator() { + } + + /** + * Normalize a raw dataset locator into its durable identity form. + * + * @throws IllegalArgumentException if the locator is null/empty, carries + * userinfo, has an empty scheme, has neither an authority nor a + * path, or is a scheme-less relative path + */ + public static String normalize(String rawLocator) { + if (rawLocator == null) { + throw new IllegalArgumentException("dataset locator must not be null"); + } + String locator = rawLocator.trim(); + if (locator.isEmpty()) { + throw new IllegalArgumentException("dataset locator must not be empty"); + } + int separator = locator.indexOf(SCHEME_SEPARATOR); + if (separator < 0) { + if (!locator.startsWith("/")) { + throw new IllegalArgumentException( + "dataset locator without a scheme must be an absolute path: " + abbreviate(locator)); + } + return stripTrailingSlashes(locator, 1); + } + String scheme = locator.substring(0, separator); + if (scheme.isEmpty()) { + throw new IllegalArgumentException("dataset locator has an empty scheme: " + abbreviate(locator)); + } + String rest = locator.substring(separator + SCHEME_SEPARATOR.length()); + int pathStart = rest.indexOf('/'); + String authority = pathStart < 0 ? rest : rest.substring(0, pathStart); + if (authority.contains("@")) { + // Never persist or key on a credential-bearing URL. + throw new IllegalArgumentException( + "credential-bearing dataset locators are never identity (userinfo is not allowed)"); + } + String path = pathStart < 0 ? "" : stripTrailingSlashes(rest.substring(pathStart), 0); + if (authority.isEmpty() && path.isEmpty()) { + // "s3://" / "file://" carry no identity at all; "file:///x" (empty + // authority, non-empty path) is legal and does not reach this. + throw new IllegalArgumentException( + "dataset locator has neither an authority nor a path: " + abbreviate(locator)); + } + return scheme.toLowerCase(Locale.ROOT) + SCHEME_SEPARATOR + authority + path; + } + + private static String stripTrailingSlashes(String value, int minLength) { + int end = value.length(); + while (end > minLength && value.charAt(end - 1) == '/') { + end--; + } + return value.substring(0, end); + } + + private static String abbreviate(String locator) { + return locator.length() <= 64 ? locator : locator.substring(0, 64) + "..."; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java new file mode 100644 index 00000000000000..19cc9efe80ad45 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import java.util.Objects; + +/** + * The durable same-name target/fence key: + * 

+ * ( persisted catalog identity, provider = DIRECTORY,
+ *   normalized stable dataset locator, persisted normalized logical-index-name bytes )
+ *
+ * The display name is not part of the key; it is persisted on the job itself. + * This class is a derived in-memory index key and is not persisted directly. + */ +public final class LanceIndexFenceKey { + /** Provider of every job in this delivery slice, mapped from a filesystem (Directory) Lance catalog. */ + public static final String PROVIDER_DIRECTORY = "DIRECTORY"; + + private final long catalogId; + private final String provider; + private final String normalizedLocator; + private final String normalizedIndexName; + + public LanceIndexFenceKey(long catalogId, String provider, String normalizedLocator, String normalizedIndexName) { + this.catalogId = catalogId; + this.provider = Objects.requireNonNull(provider, "provider"); + this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator"); + this.normalizedIndexName = Objects.requireNonNull(normalizedIndexName, "normalizedIndexName"); + } + + public long getCatalogId() { + return catalogId; + } + + public String getProvider() { + return provider; + } + + public String getNormalizedLocator() { + return normalizedLocator; + } + + public String getNormalizedIndexName() { + return normalizedIndexName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof LanceIndexFenceKey)) { + return false; + } + LanceIndexFenceKey that = (LanceIndexFenceKey) o; + return catalogId == that.catalogId + && provider.equals(that.provider) + && normalizedLocator.equals(that.normalizedLocator) + && normalizedIndexName.equals(that.normalizedIndexName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, provider, normalizedLocator, normalizedIndexName); + } + + /** + * Deliberately omits the locator: fence-conflict messages may surface to + * users without target privileges and must not disclose it. + */ + @Override + public String toString() { + return "LanceIndexFenceKey{catalogId=" + catalogId + ", provider=" + provider + + ", normalizedIndexName=" + normalizedIndexName + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java new file mode 100644 index 00000000000000..52bb24ee2bc4e4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** + * The minimal durable Lance index job record. It contains only what the + * lifecycle invariants require: job identity/creator/revision and bounded + * timestamps; the persisted target identity and normalized same-name fence + * key; the mutation intent; the admitted dataset version and schema-contract + * representation; the mutation outcome, independent refresh state, typed + * result, and a bounded sanitized message; the dispatch identity (selected BE, + * BE process epoch, immutable invocation ID, deadline), possible-live + * ownership, and any matching termination proof; and the FORCE audit fields + * (populated only by the FORCE_RELEASE slice; here they are carried for + * durability and replay). + * + *

The record never carries secrets, raw/opaque provider data, or unbounded + * values: name/message/note/properties fields are length-checked at + * construction/mutation time. Credentials are resolved at execution time from + * the current catalog and are never persisted here. + * + *

Serialization is the standard Gson stream: {@code Text.writeString} of + * the JSON form. Every durable field carries a short {@link SerializedName}; + * fields missing in old data keep the safe in-class defaults, so replay stays + * tolerant of additive evolution. An instance is effectively immutable while + * published in the manager: every durable transition is applied to a private + * copy that is logged and then swapped in, so the object written to the edit + * log is never mutated afterwards. + */ +public class LanceIndexJob implements Writable { + /** Bound on the user/build properties JSON snapshot. */ + public static final int MAX_PROPERTIES_JSON_BYTES = 4096; + /** Bound on the FORCE_RELEASE operator note and the late-commit warning text. */ + public static final int MAX_FORCE_TEXT_BYTES = 1024; + + // ------------------------------------------------------------------ + // Identity + // ------------------------------------------------------------------ + @SerializedName(value = "jid") + private long jobId; + + @SerializedName(value = "cr") + private String creator; + + /** Bumped by +1 on every durable transition; callbacks and replay are revision-checked. */ + @SerializedName(value = "rev") + private long revision; + + @SerializedName(value = "ctm") + private long createTimeMs; + + @SerializedName(value = "utm") + private long updateTimeMs; + + // ------------------------------------------------------------------ + // Target / fence identity + // ------------------------------------------------------------------ + @SerializedName(value = "cid") + private long catalogId; + + /** Local database name, kept for privilege and FORCE resolution when the catalog still resolves. */ + @SerializedName(value = "dbn") + private String dbName; + + /** Local table name, kept for privilege and FORCE resolution when the catalog still resolves. */ + @SerializedName(value = "tbn") + private String tableName; + + @SerializedName(value = "prv") + private String provider = LanceIndexFenceKey.PROVIDER_DIRECTORY; + + @SerializedName(value = "loc") + private String normalizedLocator; + + @SerializedName(value = "din") + private String displayIndexName; + + @SerializedName(value = "nin") + private String normalizedIndexName; + + // ------------------------------------------------------------------ + // Mutation intent + // ------------------------------------------------------------------ + @SerializedName(value = "mt") + private LanceIndexJobMutationType mutationType = LanceIndexJobMutationType.CREATE; + + @SerializedName(value = "ine") + private boolean ifNotExists; + + @SerializedName(value = "ie") + private boolean ifExists; + + /** Logical Lance algorithm (IVF_PQ / BTREE / BITMAP); required for CREATE/REPLACE, nullable for DROP. */ + @SerializedName(value = "it") + private String indexType; + + @SerializedName(value = "cn") + private String columnName; + + @SerializedName(value = "pj") + private String propertiesJson; + + // ------------------------------------------------------------------ + // Admission snapshot + // ------------------------------------------------------------------ + @SerializedName(value = "adv") + private long admittedDatasetVersion; + + /** Nullable only for DROP, which does not revalidate an indexed-field contract. */ + @SerializedName(value = "sc") + private LanceIndexSchemaContract schemaContract; + + // ------------------------------------------------------------------ + // Dual state + // ------------------------------------------------------------------ + @SerializedName(value = "ms") + private LanceIndexJobMutationState mutationState = LanceIndexJobMutationState.UNKNOWN; + + /** + * Safe default for a corrupt record missing the key: REQUIRED holds the fence on a + * terminal record, mirroring the null fallback in {@link #isUnresolved()}. A legal + * record always carries the state explicitly (admission sets NOT_REQUIRED). + */ + @SerializedName(value = "rs") + private LanceIndexJobRefreshState refreshState = LanceIndexJobRefreshState.REQUIRED; + + // ------------------------------------------------------------------ + // Result + // ------------------------------------------------------------------ + @SerializedName(value = "res") + private LanceIndexJobResult result; + + // ------------------------------------------------------------------ + // Dispatch + // ------------------------------------------------------------------ + @SerializedName(value = "bid") + private Long backendId; + + @SerializedName(value = "bpe") + private Long beProcessEpoch; + + /** Immutable per-dispatch UUID; callbacks must present the matching identity. */ + @SerializedName(value = "iid") + private String invocationId; + + /** Bounds wait/runtime only; never proves termination and never releases a possible-live slot. */ + @SerializedName(value = "dlm") + private Long deadlineMs; + + @SerializedName(value = "plo") + private boolean possibleLiveOwned; + + @SerializedName(value = "tp") + private LanceIndexTerminationProof terminationProof = LanceIndexTerminationProof.NONE; + + // ------------------------------------------------------------------ + // FORCE_RELEASE audit (populated only by the FORCE slice; durable + replayable here) + // ------------------------------------------------------------------ + @SerializedName(value = "fr") + private boolean forceReleased; + + @SerializedName(value = "fa") + private String forceActor; + + @SerializedName(value = "ftm") + private Long forceTimeMs; + + @SerializedName(value = "fn") + private String forceNote; + + @SerializedName(value = "fw") + private String forceWarning; + + /** + * No-arg constructor for Gson replay only; missing fields keep the safe + * defaults declared above (UNKNOWN mutation state holds the fence, never + * the redispatchable PENDING; REQUIRED refresh state owes a refresh rather + * than silently releasing the fence). + */ + public LanceIndexJob() { + } + + /** + * Admission constructor: identity, intent, and the admission snapshot. The + * manager initializes the lifecycle fields when the job is admitted. + */ + public LanceIndexJob(long jobId, String creator, long catalogId, String dbName, String tableName, + String provider, String normalizedLocator, String displayIndexName, String normalizedIndexName, + LanceIndexJobMutationType mutationType, boolean ifNotExists, boolean ifExists, String indexType, + String columnName, String propertiesJson, long admittedDatasetVersion, + LanceIndexSchemaContract schemaContract) { + this.jobId = jobId; + this.creator = creator; + this.catalogId = catalogId; + this.dbName = dbName; + this.tableName = tableName; + this.provider = Objects.requireNonNull(provider, "provider"); + this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator"); + setDisplayIndexName(displayIndexName); + setNormalizedIndexName(normalizedIndexName); + this.mutationType = Objects.requireNonNull(mutationType, "mutationType"); + this.ifNotExists = ifNotExists; + this.ifExists = ifExists; + this.indexType = indexType; + this.columnName = columnName; + setPropertiesJson(propertiesJson); + this.admittedDatasetVersion = admittedDatasetVersion; + this.schemaContract = schemaContract; + } + + /** + * Copy used by the manager to stage a durable transition: the copy is + * mutated, written to the edit log, and then swapped in verbatim, so a + * published instance is never mutated after being logged. Every field is + * carried over; the {@code result} and {@code schemaContract} references + * are shared, which is safe because both are immutable values that are + * only ever replaced wholesale, never mutated in place. + */ + public LanceIndexJob(LanceIndexJob other) { + this.jobId = other.jobId; + this.creator = other.creator; + this.revision = other.revision; + this.createTimeMs = other.createTimeMs; + this.updateTimeMs = other.updateTimeMs; + this.catalogId = other.catalogId; + this.dbName = other.dbName; + this.tableName = other.tableName; + this.provider = other.provider; + this.normalizedLocator = other.normalizedLocator; + this.displayIndexName = other.displayIndexName; + this.normalizedIndexName = other.normalizedIndexName; + this.mutationType = other.mutationType; + this.ifNotExists = other.ifNotExists; + this.ifExists = other.ifExists; + this.indexType = other.indexType; + this.columnName = other.columnName; + this.propertiesJson = other.propertiesJson; + this.admittedDatasetVersion = other.admittedDatasetVersion; + this.schemaContract = other.schemaContract; + this.mutationState = other.mutationState; + this.refreshState = other.refreshState; + this.result = other.result; + this.backendId = other.backendId; + this.beProcessEpoch = other.beProcessEpoch; + this.invocationId = other.invocationId; + this.deadlineMs = other.deadlineMs; + this.possibleLiveOwned = other.possibleLiveOwned; + this.terminationProof = other.terminationProof; + this.forceReleased = other.forceReleased; + this.forceActor = other.forceActor; + this.forceTimeMs = other.forceTimeMs; + this.forceNote = other.forceNote; + this.forceWarning = other.forceWarning; + } + + // ------------------------------------------------------------------ + // Derived helpers + // ------------------------------------------------------------------ + + /** + * The durable same-name fence key of this job. + */ + public LanceIndexFenceKey fenceKey() { + return new LanceIndexFenceKey(catalogId, provider, normalizedLocator, normalizedIndexName); + } + + /** + * The per persisted table/locator quota identity. + */ + public LanceIndexJobQuota.TableQuotaKey getTableQuotaKey() { + return new LanceIndexJobQuota.TableQuotaKey(catalogId, normalizedLocator); + } + + /** + * Whether this job still holds its same-name fence and unresolved quota. + * Fence and quota are released together: PENDING/RUNNING always hold; a + * known terminal job holds until its required refresh is DONE (FAILED + * still holds, refresh may retry); UNKNOWN holds until a durable + * FORCE_RELEASE. A null state from a corrupt record is treated as UNKNOWN, + * the safe direction (fence retained, never redispatched). + */ + public boolean isUnresolved() { + LanceIndexJobMutationState ms = mutationState == null ? LanceIndexJobMutationState.UNKNOWN : mutationState; + switch (ms) { + case PENDING: + case RUNNING: + return true; + case UNKNOWN: + return !forceReleased; + case COMMITTED: + case NOT_COMMITTED: + default: + LanceIndexJobRefreshState rs = + refreshState == null ? LanceIndexJobRefreshState.REQUIRED : refreshState; + return rs == LanceIndexJobRefreshState.REQUIRED + || rs == LanceIndexJobRefreshState.RUNNING + || rs == LanceIndexJobRefreshState.FAILED; + } + } + + /** + * Whether this job still owns a possible-live worker slot: released only + * by a matching termination proof or a durable FORCE_RELEASE, never by a + * deadline. + */ + public boolean holdsPossibleLiveSlot() { + return possibleLiveOwned + && (terminationProof == null || terminationProof == LanceIndexTerminationProof.NONE) + && !forceReleased; + } + + // ------------------------------------------------------------------ + // Accessors. Setters for bounded text fields re-validate the bound. + // ------------------------------------------------------------------ + + public long getJobId() { + return jobId; + } + + public void setJobId(long jobId) { + this.jobId = jobId; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public long getRevision() { + return revision; + } + + public void setRevision(long revision) { + this.revision = revision; + } + + public long getCreateTimeMs() { + return createTimeMs; + } + + public void setCreateTimeMs(long createTimeMs) { + this.createTimeMs = createTimeMs; + } + + public long getUpdateTimeMs() { + return updateTimeMs; + } + + public void setUpdateTimeMs(long updateTimeMs) { + this.updateTimeMs = updateTimeMs; + } + + public long getCatalogId() { + return catalogId; + } + + public String getDbName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public String getProvider() { + return provider; + } + + public String getNormalizedLocator() { + return normalizedLocator; + } + + public String getDisplayIndexName() { + return displayIndexName; + } + + public final void setDisplayIndexName(String displayIndexName) { + LanceIndexNameNormalizer.validateDisplayName(displayIndexName); + this.displayIndexName = displayIndexName; + } + + public String getNormalizedIndexName() { + return normalizedIndexName; + } + + public final void setNormalizedIndexName(String normalizedIndexName) { + if (normalizedIndexName == null || normalizedIndexName.isEmpty()) { + throw new IllegalArgumentException("normalized index name must not be null or empty"); + } + if (normalizedIndexName.getBytes(StandardCharsets.UTF_8).length + > LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES) { + throw new IllegalArgumentException( + "normalized index name exceeds " + LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + " UTF-8 bytes"); + } + this.normalizedIndexName = normalizedIndexName; + } + + public LanceIndexJobMutationType getMutationType() { + return mutationType; + } + + public boolean isIfNotExists() { + return ifNotExists; + } + + public boolean isIfExists() { + return ifExists; + } + + public String getIndexType() { + return indexType; + } + + public void setIndexType(String indexType) { + this.indexType = indexType; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + public String getPropertiesJson() { + return propertiesJson; + } + + public final void setPropertiesJson(String propertiesJson) { + this.propertiesJson = checkBytes(propertiesJson, MAX_PROPERTIES_JSON_BYTES, "propertiesJson"); + } + + public long getAdmittedDatasetVersion() { + return admittedDatasetVersion; + } + + public LanceIndexSchemaContract getSchemaContract() { + return schemaContract; + } + + public LanceIndexJobMutationState getMutationState() { + return mutationState; + } + + public void setMutationState(LanceIndexJobMutationState mutationState) { + this.mutationState = Objects.requireNonNull(mutationState, "mutationState"); + } + + public LanceIndexJobRefreshState getRefreshState() { + return refreshState; + } + + public void setRefreshState(LanceIndexJobRefreshState refreshState) { + this.refreshState = Objects.requireNonNull(refreshState, "refreshState"); + } + + public LanceIndexJobResult getResult() { + return result; + } + + public void setResult(LanceIndexJobResult result) { + this.result = result; + } + + public Long getBackendId() { + return backendId; + } + + public void setBackendId(Long backendId) { + this.backendId = backendId; + } + + public Long getBeProcessEpoch() { + return beProcessEpoch; + } + + public void setBeProcessEpoch(Long beProcessEpoch) { + this.beProcessEpoch = beProcessEpoch; + } + + public String getInvocationId() { + return invocationId; + } + + public void setInvocationId(String invocationId) { + this.invocationId = invocationId; + } + + public Long getDeadlineMs() { + return deadlineMs; + } + + public void setDeadlineMs(Long deadlineMs) { + this.deadlineMs = deadlineMs; + } + + public boolean isPossibleLiveOwned() { + return possibleLiveOwned; + } + + public void setPossibleLiveOwned(boolean possibleLiveOwned) { + this.possibleLiveOwned = possibleLiveOwned; + } + + public LanceIndexTerminationProof getTerminationProof() { + return terminationProof; + } + + public void setTerminationProof(LanceIndexTerminationProof terminationProof) { + this.terminationProof = Objects.requireNonNull(terminationProof, "terminationProof"); + } + + public boolean isForceReleased() { + return forceReleased; + } + + public void setForceReleased(boolean forceReleased) { + this.forceReleased = forceReleased; + } + + public String getForceActor() { + return forceActor; + } + + public void setForceActor(String forceActor) { + this.forceActor = forceActor; + } + + public Long getForceTimeMs() { + return forceTimeMs; + } + + public void setForceTimeMs(Long forceTimeMs) { + this.forceTimeMs = forceTimeMs; + } + + public String getForceNote() { + return forceNote; + } + + public void setForceNote(String forceNote) { + this.forceNote = checkBytes(forceNote, MAX_FORCE_TEXT_BYTES, "forceNote"); + } + + public String getForceWarning() { + return forceWarning; + } + + public void setForceWarning(String forceWarning) { + this.forceWarning = checkBytes(forceWarning, MAX_FORCE_TEXT_BYTES, "forceWarning"); + } + + private static String checkBytes(String value, int maxBytes, String fieldName) { + if (value != null && value.getBytes(StandardCharsets.UTF_8).length > maxBytes) { + throw new IllegalArgumentException(fieldName + " exceeds " + maxBytes + " UTF-8 bytes"); + } + return value; + } + + // ------------------------------------------------------------------ + // Serialization (Gson stream style, see DropIndexPolicyLog) + // ------------------------------------------------------------------ + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static LanceIndexJob read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), LanceIndexJob.class); + } + + /** + * Deliberately omits the locator: job lookups must not disclose target + * details to callers without privilege, and log lines reuse this form. + */ + @Override + public String toString() { + return "LanceIndexJob{jobId=" + jobId + ", revision=" + revision + ", catalogId=" + catalogId + + ", db=" + dbName + ", table=" + tableName + ", index=" + displayIndexName + + ", mutationType=" + mutationType + ", mutationState=" + mutationState + + ", refreshState=" + refreshState + ", possibleLiveSlot=" + holdsPossibleLiveSlot() + + ", forceReleased=" + forceReleased + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java new file mode 100644 index 00000000000000..94bfcd320538d3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +/** + * Why a job reached its terminal state. NOOP is not a state: a typed + * post-dispatch DROP-not-found for DROP IF EXISTS is NOT_COMMITTED with + * completion reason IF_CONDITION_NOOP. + */ +public enum LanceIndexJobCompletionReason { + NONE, + IF_CONDITION_NOOP +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java new file mode 100644 index 00000000000000..9add2d7c20fbfa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +/** + * Compact durable mutation lifecycle: + * 

+ * PENDING -> RUNNING -> COMMITTED
+ *                     -> NOT_COMMITTED
+ *                     -> UNKNOWN
+ *
+ * All three outcomes are terminal. UNKNOWN has no outgoing transition: + * metadata never changes it to a known outcome, and FORCE_RELEASE (a later + * delivery slice) only releases the fence/quota/slot without rewriting the + * outcome. + */ +public enum LanceIndexJobMutationState { + /** The request and fence are durable; no execute send may have occurred. */ + PENDING, + /** The durable dispatch boundary has been crossed; the one-shot call may execute or may already have executed. */ + RUNNING, + /** A complete identity-matched typed success proves this job committed. */ + COMMITTED, + /** A complete trusted result proves this job did not commit. */ + NOT_COMMITTED, + /** Doris cannot safely prove whether this job committed. */ + UNKNOWN; + + public boolean isTerminal() { + return this == COMMITTED || this == NOT_COMMITTED || this == UNKNOWN; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java new file mode 100644 index 00000000000000..4d5d3e893ff071 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +/** + * The one-shot Lance index mutation a durable job intends to perform. + * REPLACE is a full same-name rebuild (CREATE OR REPLACE INDEX). + */ +public enum LanceIndexJobMutationType { + CREATE, + REPLACE, + DROP +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java new file mode 100644 index 00000000000000..f6ece16ac935aa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +/** + * Independent metadata-refresh state of a durable Lance index job. Stored and + * transitioned independently from the mutation state: refresh success/failure + * only sets DONE/FAILED and never changes the mutation outcome. + * 
+ * NOT_REQUIRED | REQUIRED -> RUNNING -> DONE | FAILED, FAILED -> RUNNING (retry)
+ *
+ * Refresh replays through the existing idempotent external-table refresh path; + * a failed refresh may be retried and still holds the same-name fence. + */ +public enum LanceIndexJobRefreshState { + /** No refresh is owed (e.g. a proven pre-invocation failure with no relevant metadata change). */ + NOT_REQUIRED, + /** A refresh of the authoritative metadata is owed before the fence may be released. */ + REQUIRED, + /** A refresh attempt is in flight. */ + RUNNING, + /** The required refresh finished; the fence may be released. */ + DONE, + /** The refresh attempt failed; retry through the idempotent path is allowed. */ + FAILED +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java new file mode 100644 index 00000000000000..49782f275b3c2d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import com.google.gson.annotations.SerializedName; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** + * The immutable typed result of the one-shot invocation, persisted on the job. + * Contains only what is needed to classify the invocation: the saved typed + * code, the classified completion reason, a bounded sanitized message, and + * whether trusted pre-invocation revalidation observed relevant external + * metadata advancement. Never carries secrets, raw provider responses, or + * unbounded text. + */ +public class LanceIndexJobResult { + public static final int MAX_MESSAGE_BYTES = 1024; + + @SerializedName(value = "rc") + private LanceIndexJobResultCode resultCode = LanceIndexJobResultCode.NO_TRUSTED_RESULT; + + @SerializedName(value = "cr") + private LanceIndexJobCompletionReason completionReason = LanceIndexJobCompletionReason.NONE; + + @SerializedName(value = "msg") + private String sanitizedMessage; + + @SerializedName(value = "ema") + private boolean externalMetadataAdvanced; + + /** + * No-arg constructor for Gson replay only; missing fields keep the safe + * defaults declared above. + */ + public LanceIndexJobResult() { + } + + public LanceIndexJobResult(LanceIndexJobResultCode resultCode, LanceIndexJobCompletionReason completionReason, + String sanitizedMessage, boolean externalMetadataAdvanced) { + this.resultCode = Objects.requireNonNull(resultCode, "resultCode"); + this.completionReason = completionReason == null ? LanceIndexJobCompletionReason.NONE : completionReason; + this.sanitizedMessage = checkMessageBytes(sanitizedMessage); + this.externalMetadataAdvanced = externalMetadataAdvanced; + } + + private static String checkMessageBytes(String message) { + if (message != null && message.getBytes(StandardCharsets.UTF_8).length > MAX_MESSAGE_BYTES) { + throw new IllegalArgumentException( + "sanitized message exceeds " + MAX_MESSAGE_BYTES + " UTF-8 bytes"); + } + return message; + } + + public LanceIndexJobResultCode getResultCode() { + return resultCode; + } + + public LanceIndexJobCompletionReason getCompletionReason() { + return completionReason; + } + + public String getSanitizedMessage() { + return sanitizedMessage; + } + + public boolean isExternalMetadataAdvanced() { + return externalMetadataAdvanced; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof LanceIndexJobResult)) { + return false; + } + LanceIndexJobResult that = (LanceIndexJobResult) o; + return externalMetadataAdvanced == that.externalMetadataAdvanced + && resultCode == that.resultCode + && completionReason == that.completionReason + && Objects.equals(sanitizedMessage, that.sanitizedMessage); + } + + @Override + public int hashCode() { + return Objects.hash(resultCode, completionReason, sanitizedMessage, externalMetadataAdvanced); + } + + @Override + public String toString() { + return "LanceIndexJobResult{resultCode=" + resultCode + ", completionReason=" + completionReason + + ", externalMetadataAdvanced=" + externalMetadataAdvanced + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java new file mode 100644 index 00000000000000..fad096b5087212 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +/** + * Typed result codes of the one-shot Lance index invocation. Only saved typed + * codes are ever classified; message text is never inspected to infer an + * outcome. Codes fall into three groups: + * + * + */ +public enum LanceIndexJobResultCode { + /** Pre-invocation: admitted dataset version / schema contract no longer matches. */ + PRE_INVOCATION_STALE_ADMISSION, + /** Pre-invocation: the recomputed contract is not a supported contract v1. */ + PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT, + /** Pre-invocation: credentials are known to be expired. */ + PRE_INVOCATION_CREDENTIAL_EXPIRED, + /** Pre-invocation: trusted busy / pre-FFI resource rejection. */ + PRE_INVOCATION_RESOURCE_REJECTED, + /** LANCE_OK from the one native invocation. */ + NATIVE_OK, + /** LANCE_ERR_COMMIT_CONFLICT: the commit lost a race; the external dataset advanced. */ + NATIVE_COMMIT_CONFLICT, + /** LANCE_ERR_NOT_FOUND after invocation. */ + NATIVE_NOT_FOUND, + /** LANCE_ERR_INVALID_ARGUMENT after invocation. */ + NATIVE_INVALID_ARGUMENT, + /** LANCE_ERR_NOT_SUPPORTED after invocation. */ + NATIVE_NOT_SUPPORTED, + /** LANCE_ERR_INDEX after invocation (also the coarse duplicate-CREATE error). */ + NATIVE_INDEX, + /** LANCE_ERR_IO after invocation. */ + NATIVE_IO, + /** LANCE_ERR_INTERNAL after invocation. */ + NATIVE_INTERNAL, + /** No complete trusted result exists; commitment cannot be proven either way. */ + NO_TRUSTED_RESULT; + + public boolean isPreInvocation() { + return this == PRE_INVOCATION_STALE_ADMISSION + || this == PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT + || this == PRE_INVOCATION_CREDENTIAL_EXPIRED + || this == PRE_INVOCATION_RESOURCE_REJECTED; + } + + /** + * Classify a complete saved result into the durable (mutationState, refreshState, + * completionReason) triple, following the provider-result classification table: + * + * + */ + public static Classification classify(LanceIndexJobMutationType mutationType, LanceIndexJobResultCode resultCode, + boolean ifExists, boolean externalMetadataAdvanced) { + if (resultCode == null) { + resultCode = NO_TRUSTED_RESULT; + } + switch (resultCode) { + case PRE_INVOCATION_STALE_ADMISSION: + case PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT: + case PRE_INVOCATION_CREDENTIAL_EXPIRED: + case PRE_INVOCATION_RESOURCE_REJECTED: + return new Classification(LanceIndexJobMutationState.NOT_COMMITTED, + externalMetadataAdvanced + ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED, + LanceIndexJobCompletionReason.NONE); + case NATIVE_OK: + return new Classification(LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.REQUIRED, + LanceIndexJobCompletionReason.NONE); + case NATIVE_COMMIT_CONFLICT: + return new Classification(LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.REQUIRED, + LanceIndexJobCompletionReason.NONE); + case NATIVE_NOT_FOUND: + if (mutationType == LanceIndexJobMutationType.DROP) { + return new Classification(LanceIndexJobMutationState.NOT_COMMITTED, + LanceIndexJobRefreshState.REQUIRED, + ifExists ? LanceIndexJobCompletionReason.IF_CONDITION_NOOP + : LanceIndexJobCompletionReason.NONE); + } + return new Classification(LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, + LanceIndexJobCompletionReason.NONE); + case NATIVE_INVALID_ARGUMENT: + case NATIVE_NOT_SUPPORTED: + case NATIVE_INDEX: + case NATIVE_IO: + case NATIVE_INTERNAL: + case NO_TRUSTED_RESULT: + default: + return new Classification(LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, + LanceIndexJobCompletionReason.NONE); + } + } + + /** + * The immutable outcome of {@link #classify}: the durable mutation state, the + * independent refresh obligation, and the completion reason. + */ + public static final class Classification { + private final LanceIndexJobMutationState mutationState; + private final LanceIndexJobRefreshState refreshState; + private final LanceIndexJobCompletionReason completionReason; + + private Classification(LanceIndexJobMutationState mutationState, LanceIndexJobRefreshState refreshState, + LanceIndexJobCompletionReason completionReason) { + this.mutationState = mutationState; + this.refreshState = refreshState; + this.completionReason = completionReason; + } + + public LanceIndexJobMutationState getMutationState() { + return mutationState; + } + + public LanceIndexJobRefreshState getRefreshState() { + return refreshState; + } + + public LanceIndexJobCompletionReason getCompletionReason() { + return completionReason; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java new file mode 100644 index 00000000000000..b467aa287f7f19 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import java.nio.charset.StandardCharsets; +import java.util.Locale; + +/** + * Logical index name normalization v1, the only definition: the UTF-8 result + * of Java {@code toLowerCase(Locale.ROOT)}. Both the display name and the + * normalized bytes are persisted on the job; Doris preserves display case, + * rejects new case-only duplicates, and fails mutation on ambiguous external + * case-only collisions. No normalization migration or mixed-version fence + * protocol exists. + */ +public final class LanceIndexNameNormalizer { + /** Bound on the persisted logical index name, aligned with the external string bound. */ + public static final int MAX_INDEX_NAME_BYTES = 1024; + + private LanceIndexNameNormalizer() { + } + + /** + * Normalization v1. The result is the identity bytes of the same-name fence key. + */ + public static String normalize(String displayName) { + if (displayName == null) { + throw new IllegalArgumentException("index name must not be null"); + } + return displayName.toLowerCase(Locale.ROOT); + } + + /** + * True when two display names differ only by case under normalization v1. + */ + public static boolean isCaseOnlyDuplicate(String displayA, String displayB) { + if (displayA == null || displayB == null) { + return false; + } + return !displayA.equals(displayB) && normalize(displayA).equals(normalize(displayB)); + } + + /** + * Validate the persisted display name: non-empty and within the UTF-8 byte bound. + */ + public static void validateDisplayName(String displayName) { + if (displayName == null || displayName.isEmpty()) { + throw new IllegalArgumentException("index name must not be null or empty"); + } + if (displayName.getBytes(StandardCharsets.UTF_8).length > MAX_INDEX_NAME_BYTES) { + throw new IllegalArgumentException("index name exceeds " + MAX_INDEX_NAME_BYTES + " UTF-8 bytes"); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java new file mode 100644 index 00000000000000..e38175e91c2658 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Persisted representation of schema contract v1: the ordered list of indexed + * fields captured at admission. Before native invocation the worker reopens + * the admitted dataset version, independently recomputes contract v1, and + * compares the ordered representation; a mismatch or unavailable version is a + * complete pre-invocation NOT_COMMITTED result. Equality is order-sensitive. + * + *

Building a contract from an Arrow schema belongs to admission (a later + * delivery slice); this class is only the durable, comparable representation. + */ +public class LanceIndexSchemaContract { + /** The only schema contract version defined. */ + public static final int SCHEMA_CONTRACT_VERSION_V1 = 1; + + @SerializedName(value = "scv") + private int schemaContractVersion = SCHEMA_CONTRACT_VERSION_V1; + + @SerializedName(value = "flds") + private List fields = new ArrayList<>(); + + /** + * No-arg constructor for Gson replay only. + */ + public LanceIndexSchemaContract() { + } + + public LanceIndexSchemaContract(List fields) { + this.schemaContractVersion = SCHEMA_CONTRACT_VERSION_V1; + this.fields = Collections.unmodifiableList(new ArrayList<>(Objects.requireNonNull(fields, "fields"))); + } + + public int getSchemaContractVersion() { + return schemaContractVersion; + } + + public List getFields() { + return fields == null ? Collections.emptyList() : Collections.unmodifiableList(fields); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof LanceIndexSchemaContract)) { + return false; + } + LanceIndexSchemaContract that = (LanceIndexSchemaContract) o; + return schemaContractVersion == that.schemaContractVersion && getFields().equals(that.getFields()); + } + + @Override + public int hashCode() { + return Objects.hash(schemaContractVersion, getFields()); + } + + @Override + public String toString() { + return "LanceIndexSchemaContract{version=" + schemaContractVersion + ", fields=" + getFields() + '}'; + } + + /** + * One indexed field of the ordered contract. {@code normalizedType} + * carries the relevant parameters such as decimal precision/scale and + * timestamp unit/time-zone semantics; unindexed fields are excluded from + * the contract. + */ + public static final class IndexedField { + @SerializedName(value = "fid") + private long fieldId; + + @SerializedName(value = "nn") + private String normalizedName; + + @SerializedName(value = "nt") + private String normalizedType; + + @SerializedName(value = "nul") + private boolean nullable; + + @SerializedName(value = "fsd") + private Integer fixedSizeListDimension; + + @SerializedName(value = "vet") + private String vectorElementType; + + @SerializedName(value = "ven") + private Boolean vectorElementNullable; + + /** + * No-arg constructor for Gson replay only. + */ + public IndexedField() { + } + + public IndexedField(long fieldId, String normalizedName, String normalizedType, boolean nullable, + Integer fixedSizeListDimension, String vectorElementType, Boolean vectorElementNullable) { + this.fieldId = fieldId; + this.normalizedName = Objects.requireNonNull(normalizedName, "normalizedName"); + this.normalizedType = Objects.requireNonNull(normalizedType, "normalizedType"); + this.nullable = nullable; + this.fixedSizeListDimension = fixedSizeListDimension; + this.vectorElementType = vectorElementType; + this.vectorElementNullable = vectorElementNullable; + } + + public long getFieldId() { + return fieldId; + } + + public String getNormalizedName() { + return normalizedName; + } + + public String getNormalizedType() { + return normalizedType; + } + + public boolean isNullable() { + return nullable; + } + + public Integer getFixedSizeListDimension() { + return fixedSizeListDimension; + } + + public String getVectorElementType() { + return vectorElementType; + } + + public Boolean getVectorElementNullable() { + return vectorElementNullable; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof IndexedField)) { + return false; + } + IndexedField that = (IndexedField) o; + return fieldId == that.fieldId + && nullable == that.nullable + && Objects.equals(normalizedName, that.normalizedName) + && Objects.equals(normalizedType, that.normalizedType) + && Objects.equals(fixedSizeListDimension, that.fixedSizeListDimension) + && Objects.equals(vectorElementType, that.vectorElementType) + && Objects.equals(vectorElementNullable, that.vectorElementNullable); + } + + @Override + public int hashCode() { + return Objects.hash(fieldId, normalizedName, normalizedType, nullable, fixedSizeListDimension, + vectorElementType, vectorElementNullable); + } + + @Override + public String toString() { + return "IndexedField{fieldId=" + fieldId + ", normalizedName=" + normalizedName + + ", normalizedType=" + normalizedType + ", nullable=" + nullable + + ", fixedSizeListDimension=" + fixedSizeListDimension + + ", vectorElementType=" + vectorElementType + + ", vectorElementNullable=" + vectorElementNullable + '}'; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java new file mode 100644 index 00000000000000..d5de9b5cbdb225 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +/** + * Evidence that releases a possible-live worker slot. Termination proof + * releases only that slot: it neither changes an UNKNOWN outcome nor releases + * the same-name fence. Deadlines bound wait/runtime but never prove + * termination. + */ +public enum LanceIndexTerminationProof { + /** No proof; the worker may still be running. */ + NONE, + /** The supervisor reaped the exact matching child process. */ + CHILD_REAPED, + /** The recorded BE process epoch no longer exists (the BE process was replaced). */ + BE_PROCESS_EPOCH_GONE +} From 2746b4133106be7cef9bf0df1ca5976fc075552c Mon Sep 17 00:00:00 2001 From: u70b3 Date: Thu, 27 Aug 2026 12:21:08 +0000 Subject: [PATCH 2/4] [feature](lance) Add master-owned Lance index job manager with fence and quota PR3B part 2: the master-owned job/fence manager (Appendix B seam) plus edit-log and image wiring. It deliberately reuses neither the generic scheduling JobManager nor internal IndexChangeJob: the external one-shot CAS, no-redispatch rule, same-name fence, and possible-live ownership required by the design are not provided by either. - LanceIndexJobManager: every durable transition shares one write-path shape - validate under the write lock (state legality, revision CAS, callback identity), append one upsert record, then apply the same record locally - so master and followers run identical apply logic. Fence and unresolved quota (table-locator/catalog/global, 5.4) live and die together per 6.4: held by PENDING/RUNNING, by terminal jobs until their required refresh is DONE, and by UNKNOWN until a durable FORCE_RELEASE; rejection precedes any durable write, leaving no job, no fence, and no record. - Replay per 7.3: replayUpsertJob is a verbatim replace with a monotonic-revision guard and performs no state transformation, so a follower tailing a live master keeps a fresh RUNNING record RUNNING. RUNNING without a complete terminal result becomes UNKNOWN only in the master-election sweep (Env.transferToMaster, after metadata replay and before master daemons start, mirroring the insertOverwriteManager.allTaskFail precedent) through the same identity-checked channel; refresh RUNNING is downgraded to REQUIRED so the idempotent external-table refresh can resume. Replay never redispatches and never calls lance-c again. - Wiring: OP_LANCE_INDEX_JOB_UPSERT = 500 (verified unique), JournalEntity/EditLog dispatch, a lanceIndexJobManager image module appended to PersistMetaModules (no FeMetaVersion bump; old images never invoke the load method and Env pre-initializes an empty manager). --- .../java/org/apache/doris/catalog/Env.java | 25 + .../lance/job/LanceIndexJobManager.java | 629 ++++++++++++++++++ .../lance/job/LanceIndexJobQuota.java | 187 ++++++ .../apache/doris/journal/JournalEntity.java | 6 + .../org/apache/doris/persist/EditLog.java | 10 + .../apache/doris/persist/OperationType.java | 3 + .../doris/persist/meta/MetaPersistMethod.java | 6 + .../persist/meta/PersistMetaModules.java | 2 +- 8 files changed, 867 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 43235d9ac51160..b1c17cec6aa786 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -109,6 +109,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; +import org.apache.doris.datasource.lance.job.LanceIndexJobManager; import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.deploy.DeployManager; @@ -568,6 +569,8 @@ public class Env { private InsertOverwriteManager insertOverwriteManager; + private LanceIndexJobManager lanceIndexJobManager; + private DNSCache dnsCache; private final NereidsSqlCacheManager sqlCacheManager; @@ -853,6 +856,7 @@ public Env(boolean isCheckpointCatalog) { this.mtmvService = new MTMVService(); this.eventProcessor = new EventProcessor(mtmvService); this.insertOverwriteManager = new InsertOverwriteManager(); + this.lanceIndexJobManager = new LanceIndexJobManager(); this.dnsCache = new DNSCache(); this.sqlCacheManager = new NereidsSqlCacheManager(); this.sortedPartitionsCacheManager = new NereidsSortedPartitionsCacheManager(); @@ -978,6 +982,10 @@ public InsertOverwriteManager getInsertOverwriteManager() { return insertOverwriteManager; } + public LanceIndexJobManager getLanceIndexJobManager() { + return lanceIndexJobManager; + } + public TabletScheduler getTabletScheduler() { return tabletScheduler; } @@ -1812,6 +1820,11 @@ private void transferToMaster() { insertOverwriteManager.allTaskFail(); + // A durable RUNNING Lance index job at this point may have lost its result with the + // old master: sweep it to UNKNOWN (and refresh RUNNING back to REQUIRED) before any + // master-only dispatcher could start. + lanceIndexJobManager.onTransferToMaster(); + toMasterProgress = "start daemon threads"; // coz current fe was not master fe and didn't get all fes' alive session report before, which cause @@ -2657,6 +2670,18 @@ public long saveDictionaryManager(CountingDataOutputStream out, long checksum) t return checksum; } + public long loadLanceIndexJobManager(DataInputStream in, long checksum) throws IOException { + this.lanceIndexJobManager = LanceIndexJobManager.read(in); + LOG.info("finished replay lance index job manager from image"); + return checksum; + } + + public long saveLanceIndexJobManager(CountingDataOutputStream out, long checksum) throws IOException { + this.lanceIndexJobManager.write(out); + LOG.info("finished save lance index job manager to image"); + return checksum; + } + // Only called by checkpoint thread // return the latest image file's absolute path public String saveImage() throws IOException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java new file mode 100644 index 00000000000000..4eae18093a492d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java @@ -0,0 +1,629 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Master-owned manager of the durable Lance index job records, the same-name + * fences, and the three-level unresolved-job quotas. It deliberately reuses + * neither the generic JobManager scheduling framework nor the internal + * IndexChangeJob machinery: the external one-shot CAS, no-redispatch rule, + * same-name fence, and possible-live ownership required here are not provided + * by either. + * + *

Every FE keeps the same in-memory image of {@link #jobs}: the master + * writes each durable transition to the edit log and then applies the same + * record locally; followers apply it from replay. All transitions share one + * write-path shape: + * 

+ * writeLock -> validate (state, revision CAS, callback identity)
+ *           -> writeEditLog(updated copy)   // fails only by System.exit
+ *           -> applyToMemory(updated copy)  // verbatim swap + fence/quota accounting
+ *
+ * A published {@link LanceIndexJob} is never mutated; a transition stages a + * copy, logs it, and swaps it in. {@link #replayUpsertJob(LanceIndexJob)} is a + * verbatim replace with a monotonic-revision guard and performs no state + * transformation at all: a follower tailing a live master must keep a fresh + * RUNNING record RUNNING. The only place a durable RUNNING without a complete + * matching terminal result becomes UNKNOWN is {@link #onTransferToMaster()}, + * the master-election sweep that runs after metadata replay and before any + * dispatcher could start. + * + *

Fence and quota live and die together ({@link LanceIndexJob#isUnresolved()}): + * released when a known terminal job's refresh is NOT_REQUIRED or DONE, or by + * a durable FORCE_RELEASE; an unresolved UNKNOWN holds both across failover, + * timeout, termination proof, and metadata observations. + * + *

The class starts no threads, so no checkpoint-thread guard is needed in + * the constructor; the derived fence index and quota counters are rebuilt in + * {@link #gsonPostProcess()} after image load. + */ +public class LanceIndexJobManager implements Writable, GsonPostProcessable { + private static final Logger LOG = LogManager.getLogger(LanceIndexJobManager.class); + + @SerializedName(value = "jobs") + private ConcurrentMap jobs = Maps.newConcurrentMap(); + + /** + * Derived: fence key -> jobId, holds the unresolved jobs that carry fence + * identity (identity-less corrupt records are kept out of the books). Rebuilt + * after replay/image load. + */ + private final Map fenceIndex = Maps.newHashMap(); + + /** Derived: three-level unresolved counters, rebuilt from the unresolved jobs. */ + private final LanceIndexJobQuota quota = new LanceIndexJobQuota(); + + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + + public LanceIndexJobManager() { + } + + private void readLock() { + lock.readLock().lock(); + } + + private void readUnlock() { + lock.readLock().unlock(); + } + + private void writeLock() { + lock.writeLock().lock(); + } + + private void writeUnlock() { + lock.writeLock().unlock(); + } + + /** + * Edit-log seam: the only place this manager writes the journal. Tests + * subclass and override to capture or swallow the record. + */ + protected void writeEditLog(LanceIndexJob job) { + Env.getCurrentEnv().getEditLog().logLanceIndexJob(job); + } + + // ------------------------------------------------------------------ + // Master-only write paths + // ------------------------------------------------------------------ + + /** + * Admit a new job: same-name fence CAS and unresolved-quota admission + * happen inside the write lock, before anything is logged. A rejection + * leaves no job, no fence, no quota charge, and no edit-log record. + * + * @throws DdlException on a fence conflict, a duplicate job id, or quota overload + */ + public void createJob(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) + throws DdlException { + Objects.requireNonNull(job, "job"); + writeLock(); + try { + if (jobs.containsKey(job.getJobId())) { + throw new DdlException("lance index job id already exists: " + job.getJobId()); + } + Long fencingJobId = fenceIndex.get(job.fenceKey()); + if (fencingJobId != null) { + // Never disclose the locator in the rejection (the caller may lack target privilege). + throw new DdlException("lance index '" + job.getDisplayIndexName() + + "' is fenced by unresolved job " + fencingJobId + + "; resolve that job (FORCE_RELEASE) before reusing the name"); + } + // Pure admission check; the charge itself happens in applyToMemory together with the fence, + // after the record is durable. An edit-log write failure exits the process, so no rollback exists. + if (!quota.hasCapacity(job, tableLimit, catalogLimit, globalLimit)) { + throw new DdlException("unresolved lance index job quota exceeded for index '" + + job.getDisplayIndexName() + "'; resolve or finish existing jobs first"); + } + long now = System.currentTimeMillis(); + // Stage a private copy and reset every lifecycle field: the caller's object is + // never published, and admission always starts from the same PENDING record no + // matter what the caller left in the lifecycle fields. + LanceIndexJob admitted = new LanceIndexJob(job); + admitted.setMutationState(LanceIndexJobMutationState.PENDING); + admitted.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED); + admitted.setRevision(0); + admitted.setCreateTimeMs(now); + admitted.setUpdateTimeMs(now); + admitted.setResult(null); + admitted.setBackendId(null); + admitted.setBeProcessEpoch(null); + admitted.setInvocationId(null); + admitted.setDeadlineMs(null); + admitted.setPossibleLiveOwned(false); + admitted.setTerminationProof(LanceIndexTerminationProof.NONE); + admitted.setForceReleased(false); + admitted.setForceActor(null); + admitted.setForceTimeMs(null); + admitted.setForceNote(null); + admitted.setForceWarning(null); + writeEditLog(admitted); + applyToMemory(admitted); + } finally { + writeUnlock(); + } + } + + /** + * PENDING -> RUNNING, the durable dispatch boundary. Compare-and-set on + * (jobId, revision): only a PENDING job at the expected revision may be + * dispatched, which is what makes redispatch after replay impossible. + * Records the dispatch identity (backend, BE process epoch, immutable + * invocation id, deadline) and takes the possible-live slot. + * + * @return false (with a warning) on any mismatch; the caller must not send + */ + public boolean markRunning(long jobId, long expectedRevision, long backendId, long beProcessEpoch, + String invocationId, long deadlineMs) { + if (StringUtils.isBlank(invocationId)) { + // A null/blank invocation identity would silently match a null field under + // Objects.equals in completeWithResult and defeat the stale-callback guard. + LOG.warn("reject markRunning for lance index job {}: invocation id is null or blank", jobId); + return false; + } + writeLock(); + try { + LanceIndexJob current = jobs.get(jobId); + if (current == null || current.getRevision() != expectedRevision + || current.getMutationState() != LanceIndexJobMutationState.PENDING) { + LOG.warn("reject markRunning for lance index job {}: expected revision {}, current {}", + jobId, expectedRevision, current); + return false; + } + LanceIndexJob updated = new LanceIndexJob(current); + updated.setMutationState(LanceIndexJobMutationState.RUNNING); + updated.setBackendId(backendId); + updated.setBeProcessEpoch(beProcessEpoch); + updated.setInvocationId(invocationId); + updated.setDeadlineMs(deadlineMs); + updated.setPossibleLiveOwned(true); + updated.setRevision(current.getRevision() + 1); + updated.setUpdateTimeMs(System.currentTimeMillis()); + writeEditLog(updated); + applyToMemory(updated); + return true; + } finally { + writeUnlock(); + } + } + + /** + * RUNNING -> terminal, from a worker/supervisor result. A callback must + * match the durable dispatch identity exactly (job revision, invocation + * id, and BE process epoch); a stale callback only logs a warning and + * changes nothing. The typed result is classified into (mutation state, + * refresh obligation, completion reason); message text is never inspected. + * A known terminal job whose refresh is NOT_REQUIRED releases fence and + * quota immediately; an UNKNOWN keeps both until a durable FORCE_RELEASE. + * + *

This is also the channel for marking a job UNKNOWN on an ambiguous + * result (result code NO_TRUSTED_RESULT), including the master-transfer + * sweep; no separate markUnknown API exists. + * + * @return false (with a warning) when the callback is stale or the job is not RUNNING + */ + public boolean completeWithResult(long jobId, long expectedRevision, String invocationId, Long beProcessEpoch, + LanceIndexJobResult result) { + Objects.requireNonNull(result, "result"); + writeLock(); + try { + LanceIndexJob current = jobs.get(jobId); + if (current == null || current.getRevision() != expectedRevision) { + LOG.warn("reject stale lance index job callback for job {}: expected revision {}, current {}", + jobId, expectedRevision, current); + return false; + } + if (current.getMutationState() != LanceIndexJobMutationState.RUNNING) { + LOG.warn("reject lance index job callback for job {} in state {}: only RUNNING accepts a result", + jobId, current.getMutationState()); + return false; + } + if (!Objects.equals(current.getInvocationId(), invocationId) + || !Objects.equals(current.getBeProcessEpoch(), beProcessEpoch)) { + LOG.warn("reject stale lance index job callback for job {}: invocation/epoch mismatch, current {}", + jobId, current); + return false; + } + LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify( + current.getMutationType(), result.getResultCode(), current.isIfExists(), + result.isExternalMetadataAdvanced()); + LanceIndexJob updated = new LanceIndexJob(current); + updated.setMutationState(classification.getMutationState()); + updated.setRefreshState(classification.getRefreshState()); + updated.setResult(new LanceIndexJobResult(result.getResultCode(), classification.getCompletionReason(), + result.getSanitizedMessage(), result.isExternalMetadataAdvanced())); + updated.setRevision(current.getRevision() + 1); + updated.setUpdateTimeMs(System.currentTimeMillis()); + writeEditLog(updated); + applyToMemory(updated); + return true; + } finally { + writeUnlock(); + } + } + + /** + * Refresh REQUIRED -> RUNNING, or FAILED -> RUNNING for a retry + * through the idempotent external-table refresh path. + */ + public boolean markRefreshRunning(long jobId, long expectedRevision) { + return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.RUNNING); + } + + /** + * Refresh RUNNING -> DONE. On a known terminal job this releases the + * same-name fence and the unresolved quota. + */ + public boolean markRefreshDone(long jobId, long expectedRevision) { + return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.DONE); + } + + /** + * Refresh RUNNING -> FAILED. The job keeps fence and quota; a later + * {@link #markRefreshRunning} retries through the idempotent path. + */ + public boolean markRefreshFailed(long jobId, long expectedRevision) { + return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.FAILED); + } + + private boolean transitionRefresh(long jobId, long expectedRevision, LanceIndexJobRefreshState target) { + writeLock(); + try { + LanceIndexJob current = jobs.get(jobId); + if (current == null || current.getRevision() != expectedRevision) { + LOG.warn("reject refresh transition to {} for lance index job {}: expected revision {}, current {}", + target, jobId, expectedRevision, current); + return false; + } + LanceIndexJobRefreshState from = current.getRefreshState(); + boolean legal = (target == LanceIndexJobRefreshState.RUNNING + && (from == LanceIndexJobRefreshState.REQUIRED || from == LanceIndexJobRefreshState.FAILED)) + || (target == LanceIndexJobRefreshState.DONE && from == LanceIndexJobRefreshState.RUNNING) + || (target == LanceIndexJobRefreshState.FAILED && from == LanceIndexJobRefreshState.RUNNING); + if (!legal) { + LOG.warn("reject illegal refresh transition {} -> {} for lance index job {}", + from, target, jobId); + return false; + } + LanceIndexJob updated = new LanceIndexJob(current); + updated.setRefreshState(target); + updated.setRevision(current.getRevision() + 1); + updated.setUpdateTimeMs(System.currentTimeMillis()); + writeEditLog(updated); + applyToMemory(updated); + return true; + } finally { + writeUnlock(); + } + } + + /** + * Record a matching termination proof for a job that still owns a + * possible-live slot. This releases only the slot: it never changes the + * mutation state and never releases the fence or quota. + */ + public boolean recordTerminationProof(long jobId, long expectedRevision, LanceIndexTerminationProof proof) { + Objects.requireNonNull(proof, "proof"); + writeLock(); + try { + LanceIndexJob current = jobs.get(jobId); + if (current == null || current.getRevision() != expectedRevision) { + LOG.warn("reject termination proof for lance index job {}: expected revision {}, current {}", + jobId, expectedRevision, current); + return false; + } + if (proof == LanceIndexTerminationProof.NONE || !current.isPossibleLiveOwned() + || current.getTerminationProof() != LanceIndexTerminationProof.NONE) { + LOG.warn("reject termination proof {} for lance index job {}: no possible-live slot owned, current {}", + proof, jobId, current); + return false; + } + LanceIndexJob updated = new LanceIndexJob(current); + updated.setTerminationProof(proof); + updated.setPossibleLiveOwned(false); + updated.setRevision(current.getRevision() + 1); + updated.setUpdateTimeMs(System.currentTimeMillis()); + writeEditLog(updated); + applyToMemory(updated); + return true; + } finally { + writeUnlock(); + } + } + + /** + * Master-election sweep, hooked from {@code Env.transferToMaster()} after + * metadata replay and before any master-only dispatcher could start. A + * durable RUNNING at this point means the terminal result may have been + * lost with the old master: the job becomes UNKNOWN through the same + * completeWithResult channel (result code NO_TRUSTED_RESULT, fence/quota/ + * possible-live ownership retained, never redispatched), and an in-flight + * refresh is downgraded to REQUIRED so the idempotent refresh retries. + * Both transitions are written to the edit log so followers converge. + */ + public void onTransferToMaster() { + List snapshot; + readLock(); + try { + snapshot = new ArrayList<>(jobs.values()); + } finally { + readUnlock(); + } + for (LanceIndexJob job : snapshot) { + if (job.getMutationState() == LanceIndexJobMutationState.RUNNING) { + boolean completed = completeWithResult(job.getJobId(), job.getRevision(), job.getInvocationId(), + job.getBeProcessEpoch(), + new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT, + LanceIndexJobCompletionReason.NONE, + "FE master transferred while the job was RUNNING; the result is not trusted", false)); + if (completed) { + LOG.info("lance index job {} transitioned RUNNING -> UNKNOWN on master transfer", + job.getJobId()); + } + } + if (job.getRefreshState() == LanceIndexJobRefreshState.RUNNING) { + downgradeRunningRefresh(job); + } + } + } + + private void downgradeRunningRefresh(LanceIndexJob sweepCandidate) { + writeLock(); + try { + LanceIndexJob current = jobs.get(sweepCandidate.getJobId()); + if (current == null || current.getRefreshState() != LanceIndexJobRefreshState.RUNNING) { + return; + } + LanceIndexJob updated = new LanceIndexJob(current); + updated.setRefreshState(LanceIndexJobRefreshState.REQUIRED); + updated.setRevision(current.getRevision() + 1); + updated.setUpdateTimeMs(System.currentTimeMillis()); + writeEditLog(updated); + applyToMemory(updated); + } finally { + writeUnlock(); + } + } + + // ------------------------------------------------------------------ + // Replay (all FEs) + // ------------------------------------------------------------------ + + /** + * Apply one durable record from the journal: a verbatim replace of the + * in-memory record plus derived-index accounting, with a monotonic + * revision guard so a stale record can never roll the state back. No state + * transformation happens here on purpose: a follower tailing a live master + * must keep a fresh RUNNING record RUNNING. Idempotent and tolerant of + * missing fields (Gson defaults); never throws on behalf of record content. + */ + public void replayUpsertJob(LanceIndexJob job) { + if (job == null) { + LOG.warn("ignore null lance index job record"); + return; + } + writeLock(); + try { + LanceIndexJob existing = jobs.get(job.getJobId()); + if (existing != null && job.getRevision() < existing.getRevision()) { + LOG.warn("ignore stale lance index job record for job {}: replayed revision {} < current {}", + job.getJobId(), job.getRevision(), existing.getRevision()); + return; + } + applyToMemory(job); + } finally { + writeUnlock(); + } + } + + /** + * Swap a staged record into memory and settle fence/quota accounting for + * the replaced record. Fence and quota always move together: a record + * holds both while {@link LanceIndexJob#isUnresolved()}, provided it + * carries fence identity (a corrupt identity-less record stays queryable + * but out of the books on both the charge and the release side). Caller + * holds the write lock. + */ + private void applyToMemory(LanceIndexJob job) { + LanceIndexJob old = jobs.put(job.getJobId(), job); + // Release only what the identity guard below booked: an identity-less corrupt + // record was stored without fence/quota accounting, so keying on it would throw. + if (old != null && old.isUnresolved() && hasFenceIdentity(old)) { + fenceIndex.remove(old.fenceKey(), old.getJobId()); + quota.release(old); + } + if (job.isUnresolved()) { + if (hasFenceIdentity(job)) { + Long displaced = fenceIndex.put(job.fenceKey(), job.getJobId()); + if (displaced != null && displaced.longValue() != job.getJobId()) { + // Only a corrupt journal can collide here; keep the smaller job id, + // the same rule as gsonPostProcess. + LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping {}", + displaced, job.getJobId(), Math.min(displaced, job.getJobId())); + if (displaced.longValue() < job.getJobId()) { + fenceIndex.put(job.fenceKey(), displaced); + } + } + quota.charge(job); + } else { + // Corrupt record tolerance: keep it queryable but out of the fence/quota books. + LOG.warn("lance index job {} lacks fence identity (provider/locator/name);" + + " stored without fence/quota accounting", job.getJobId()); + } + } + } + + private static boolean hasFenceIdentity(LanceIndexJob job) { + return job.getProvider() != null && job.getNormalizedLocator() != null + && job.getNormalizedIndexName() != null; + } + + // ------------------------------------------------------------------ + // Queries + // ------------------------------------------------------------------ + + public LanceIndexJob getJob(long jobId) { + readLock(); + try { + return jobs.get(jobId); + } finally { + readUnlock(); + } + } + + /** + * All jobs still holding a fence and unresolved quota: PENDING/RUNNING, + * unforced UNKNOWN, and known terminal jobs with unfinished refresh. + */ + public List getUnresolvedJobs() { + readLock(); + try { + List result = new ArrayList<>(); + for (LanceIndexJob job : jobs.values()) { + if (job != null && job.isUnresolved()) { + result.add(job); + } + } + return result; + } finally { + readUnlock(); + } + } + + /** + * Terminal jobs the refresh driver must pick up: refresh REQUIRED (waiting to + * run, including jobs downgraded by the master-transfer sweep) or FAILED + * (waiting for a retry). Both resume through the idempotent refresh path via + * {@link #markRefreshRunning}; a FAILED job invisible here would hold its + * fence forever with no retry channel. + */ + public List getJobsNeedingRefresh() { + readLock(); + try { + List result = new ArrayList<>(); + for (LanceIndexJob job : jobs.values()) { + if (job != null && job.getMutationState() != null && job.getMutationState().isTerminal() + && (job.getRefreshState() == LanceIndexJobRefreshState.REQUIRED + || job.getRefreshState() == LanceIndexJobRefreshState.FAILED)) { + result.add(job); + } + } + return result; + } finally { + readUnlock(); + } + } + + public boolean isFenceHeld(LanceIndexFenceKey fenceKey) { + readLock(); + try { + return fenceIndex.containsKey(fenceKey); + } finally { + readUnlock(); + } + } + + @VisibleForTesting + public LanceIndexJobQuota getQuota() { + return quota; + } + + @VisibleForTesting + public int getJobCount() { + readLock(); + try { + return jobs.size(); + } finally { + readUnlock(); + } + } + + // ------------------------------------------------------------------ + // Image serialization (whole-object Gson, IndexPolicyMgr style) + // ------------------------------------------------------------------ + + @Override + public void write(DataOutput out) throws IOException { + // Serialize against the declared base type: GsonUtils.BLOCK_INACCESSIBLE_JAVA refuses + // reflection on non-public runtime classes, so a subclassed manager (the test seam) + // would otherwise serialize as "null". + Text.writeString(out, GsonUtils.GSON.toJson(this, LanceIndexJobManager.class)); + } + + public static LanceIndexJobManager read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), LanceIndexJobManager.class); + } + + /** + * Rebuild the derived fence index and quota counters from the durable + * jobs after Gson image load. A fence-key collision between unresolved + * jobs (only possible on a corrupt image) keeps the smaller job id and + * logs a warning; replay itself can never produce one. + */ + @Override + public void gsonPostProcess() throws IOException { + fenceIndex.clear(); + List unresolvedJobs = new ArrayList<>(); + if (jobs == null) { + jobs = Maps.newConcurrentMap(); + } + for (LanceIndexJob job : jobs.values()) { + if (job == null || !job.isUnresolved()) { + continue; + } + if (!hasFenceIdentity(job)) { + LOG.warn("lance index job {} lacks fence identity (provider/locator/name);" + + " excluded from fence/quota rebuild", job.getJobId()); + continue; + } + unresolvedJobs.add(job); + Long existing = fenceIndex.get(job.fenceKey()); + if (existing != null) { + LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping {}", + existing, job.getJobId(), Math.min(existing, job.getJobId())); + } + if (existing == null || job.getJobId() < existing) { + fenceIndex.put(job.fenceKey(), job.getJobId()); + } + } + quota.rebuild(unresolvedJobs); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java new file mode 100644 index 00000000000000..ca4b6e4154762c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Three-level unresolved-job quota counters: per persisted table/locator + * identity, per catalog, and globally. An unresolved job is any job that still + * holds its same-name fence (PENDING/RUNNING, an unforced UNKNOWN, or a known + * terminal job whose required refresh is not DONE); active-plus-UNKNOWN counts + * stay bounded at every level. + * + *

This class only counts. Callers hold the manager write lock, so no + * internal synchronization exists. Config-gated limits are resolved by the + * admission layer and passed in as plain values; a non-positive limit disables + * that level's check. The counters are rebuilt from the durable jobs after + * replay/image load and are never persisted themselves. + */ +public class LanceIndexJobQuota { + private static final Logger LOG = LogManager.getLogger(LanceIndexJobQuota.class); + + private long globalCount; + private final Map catalogCounts = new HashMap<>(); + private final Map tableCounts = new HashMap<>(); + + /** + * Check every level whose limit is positive, then increment all three + * levels. Returns false (and increments nothing) when any enforced level + * is full: "current + 1 <= limit" must hold at every enforced level. + * + *

The manager admission path uses {@link #hasCapacity} plus + * {@link #charge} instead: the check and the charge deliberately straddle + * the edit-log write. This self-contained variant is reserved for direct + * use by the later admission slice and by tests. + */ + public boolean tryAcquire(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) { + if (!hasCapacity(job, tableLimit, catalogLimit, globalLimit)) { + return false; + } + charge(job); + return true; + } + + /** + * Pure check variant of {@link #tryAcquire}: true when incrementing would + * not exceed any positive limit. A non-positive limit disables that level. + */ + public boolean hasCapacity(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) { + Objects.requireNonNull(job, "job"); + if (globalLimit > 0 && globalCount + 1 > globalLimit) { + return false; + } + if (catalogLimit > 0 && getCatalogCount(job.getCatalogId()) + 1 > catalogLimit) { + return false; + } + return tableLimit <= 0 || getTableCount(job.getTableQuotaKey()) + 1 <= tableLimit; + } + + /** + * Increment all three levels unconditionally. Used when a durable record + * (already admitted, or replayed) is applied to memory. + */ + public void charge(LanceIndexJob job) { + globalCount++; + catalogCounts.merge(job.getCatalogId(), 1L, Long::sum); + tableCounts.merge(job.getTableQuotaKey(), 1L, Long::sum); + } + + /** + * Decrement all three levels. Underflow is clamped at zero and warned + * about: it indicates a bookkeeping bug, not a reason to fail replay. + */ + public void release(LanceIndexJob job) { + if (globalCount <= 0) { + LOG.warn("lance index job quota global underflow on release of job {}", job.getJobId()); + } else { + globalCount--; + } + decrement(catalogCounts, job.getCatalogId(), job.getJobId()); + decrement(tableCounts, job.getTableQuotaKey(), job.getJobId()); + } + + private static void decrement(Map counts, K key, long jobId) { + Long current = counts.get(key); + if (current == null || current <= 0) { + LOG.warn("lance index job quota underflow on release of job {} at key {}", jobId, key); + counts.remove(key); + return; + } + if (current == 1L) { + counts.remove(key); + } else { + counts.put(key, current - 1); + } + } + + /** + * Reset and recount from the unresolved durable jobs (replay / image load). + */ + public void rebuild(Iterable unresolvedJobs) { + globalCount = 0; + catalogCounts.clear(); + tableCounts.clear(); + for (LanceIndexJob job : unresolvedJobs) { + charge(job); + } + } + + public long getGlobalCount() { + return globalCount; + } + + public long getCatalogCount(long catalogId) { + return catalogCounts.getOrDefault(catalogId, 0L); + } + + public long getTableCount(TableQuotaKey key) { + return tableCounts.getOrDefault(key, 0L); + } + + /** + * Per persisted table/locator identity: (catalogId, normalizedLocator). + */ + public static final class TableQuotaKey { + private final long catalogId; + private final String normalizedLocator; + + public TableQuotaKey(long catalogId, String normalizedLocator) { + this.catalogId = catalogId; + this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator"); + } + + public long getCatalogId() { + return catalogId; + } + + public String getNormalizedLocator() { + return normalizedLocator; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TableQuotaKey)) { + return false; + } + TableQuotaKey that = (TableQuotaKey) o; + return catalogId == that.catalogId && normalizedLocator.equals(that.normalizedLocator); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, normalizedLocator); + } + + /** + * Deliberately omits the locator, like {@link LanceIndexFenceKey#toString()}. + */ + @Override + public String toString() { + return "TableQuotaKey{catalogId=" + catalogId + '}'; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index 75cbbe7b487abc..d2c1952211dc61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.InitDatabaseLog; import org.apache.doris.datasource.MetaIdMappingsLog; +import org.apache.doris.datasource.lance.job.LanceIndexJob; import org.apache.doris.ha.MasterInfo; import org.apache.doris.indexpolicy.DropIndexPolicyLog; import org.apache.doris.indexpolicy.IndexPolicy; @@ -1013,6 +1014,11 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_LANCE_INDEX_JOB_UPSERT: { + data = LanceIndexJob.read(in); + isRead = true; + break; + } case OperationType.OP_BEGIN_SNAPSHOT: { data = SnapshotState.read(in); isRead = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index bb01746a0ab826..9ccb4c5844528c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -64,6 +64,7 @@ import org.apache.doris.datasource.InitDatabaseLog; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.MetaIdMappingsLog; +import org.apache.doris.datasource.lance.job.LanceIndexJob; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.ha.MasterInfo; import org.apache.doris.indexpolicy.DropIndexPolicyLog; @@ -1442,6 +1443,11 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.getKeyManager().replayKeyOperation(info); break; } + case OperationType.OP_LANCE_INDEX_JOB_UPSERT: { + LanceIndexJob job = (LanceIndexJob) journal.getData(); + env.getLanceIndexJobManager().replayUpsertJob(job); + break; + } case OperationType.OP_BEGIN_SNAPSHOT: { // SnapshotState info = (SnapshotState) journal.getData(); // TODO: implement @@ -2353,6 +2359,10 @@ public void logDropIndexPolicy(DropIndexPolicyLog policy) { logEdit(OperationType.OP_DROP_INDEX_POLICY, policy); } + public void logLanceIndexJob(LanceIndexJob job) { + logEdit(OperationType.OP_LANCE_INDEX_JOB_UPSERT, job); + } + public void logCatalogLog(short id, CatalogLog log) { logEdit(id, log); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index ba2580f4408e87..e2fbc2131f7000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -422,6 +422,9 @@ public class OperationType { public static final short OP_CREATE_ROLE_MAPPING = 496; public static final short OP_DROP_ROLE_MAPPING = 497; + // lance index job 500 ~ 509 + public static final short OP_LANCE_INDEX_JOB_UPSERT = 500; + // For cloud. public static final short OP_UPDATE_CLOUD_REPLICA = 1000; @Deprecated diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java index 8f8425ec4a80cc..6ed7645d4610b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java @@ -286,6 +286,12 @@ public static MetaPersistMethod create(String name) throws NoSuchMethodException metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveKeyManagerStore", CountingDataOutputStream.class, long.class); break; + case "lanceIndexJobManager": + metaPersistMethod.readMethod = Env.class.getDeclaredMethod("loadLanceIndexJobManager", + DataInputStream.class, long.class); + metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveLanceIndexJobManager", + CountingDataOutputStream.class, long.class); + break; default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java index 68ee92c8e6d7bd..8f8f00dd64ea2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java @@ -44,7 +44,7 @@ public class PersistMetaModules { "globalFunction", "workloadGroups", "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "workloadSchedPolicy", "insertOverwrite", "plsql", "dictionaryManager", "indexPolicy", "KeyManagerStore", - "authenticationIntegrations", "roleMappings" + "authenticationIntegrations", "roleMappings", "lanceIndexJobManager" ); // The modules in `CloudEnv`. From b905cd8ff5c1fd9a346e228c8cbb980298decdcf Mon Sep 17 00:00:00 2001 From: u70b3 Date: Thu, 27 Aug 2026 12:21:45 +0000 Subject: [PATCH 3/4] [test](lance) Cover Lance index job lifecycle, replay and persistence PR3B unit tests (105 cases, pure UT, no FE service): - Normalization v1 incl. the Turkish dotted-I corner; locator forms and rejections (userinfo, empty scheme, relative path, no identity). - Section 6.3 classification matrix independently restated per cell; IF_CONDITION_NOOP confined to DROP IF EXISTS + NOT_FOUND. - State machine legality: UNKNOWN has no outgoing transitions, refresh transitions stay independent, revision CAS, blank invocation ids rejected at the dispatch boundary. - Fence/quota co-release timing (immediate on NOT_REQUIRED, on refresh DONE, never for FAILED/UNKNOWN), three-level quota boundaries, and rebuild equivalence after image load. - Section 7.3 replay matrix: PENDING re-dispatchable once; RUNNING swept to UNKNOWN at master transfer and never redispatchable; terminal jobs resume only refresh (REQUIRED and FAILED stay visible to the refresh driver); UNKNOWN rebuilds fence/quota/possible-live; force-released UNKNOWN frees the name; stale callbacks rejected on revision/invocation/epoch mismatch; replay idempotent with a monotonic revision guard; identity-less corrupt records tolerated without throwing, including follow-up upserts for the same job id. - Manager image write/read round-trip rebuilds derived fence/quota; JournalEntity round-trip covers the new op-500 dispatch; over-bounds text fields rejected at construction. --- .../job/LanceIndexDatasetLocatorTest.java | 128 ++++ .../job/LanceIndexJobManagerPersistTest.java | 363 ++++++++++++ .../job/LanceIndexJobManagerReplayTest.java | 431 ++++++++++++++ .../lance/job/LanceIndexJobQuotaTest.java | 211 +++++++ .../job/LanceIndexJobResultClassifyTest.java | 239 ++++++++ .../job/LanceIndexJobStateMachineTest.java | 558 ++++++++++++++++++ .../lance/job/LanceIndexJobTest.java | 280 +++++++++ .../job/LanceIndexNameNormalizerTest.java | 111 ++++ 8 files changed, 2321 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java new file mode 100644 index 00000000000000..64293dad43f0db --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit coverage for dataset-locator normalization v1, the fence-key identity for the + * target dataset: trim, lowercase scheme, userinfo rejection (credential-bearing URLs + * are never identity), rejection of locators with neither authority nor path, + * trailing-slash stripping with a kept root, and the absolute-path requirement for + * scheme-less locators. + */ +public class LanceIndexDatasetLocatorTest { + + @Test + public void lowercasesSchemeOnly() { + Assertions.assertEquals("s3://bucket/path", LanceIndexDatasetLocator.normalize("S3://bucket/path")); + Assertions.assertEquals("file:///data/x", LanceIndexDatasetLocator.normalize("FILE:///data/x")); + Assertions.assertEquals("hdfs://nn:8020/data", LanceIndexDatasetLocator.normalize("HDFS://nn:8020/data")); + } + + @Test + public void preservesAuthorityAndPathCase() { + // Bucket and path components are case-sensitive on the supported providers. + Assertions.assertEquals("s3://MyBucket/MyPath", LanceIndexDatasetLocator.normalize("s3://MyBucket/MyPath")); + } + + @Test + public void trimsSurroundingWhitespace() { + Assertions.assertEquals("/data/x", LanceIndexDatasetLocator.normalize(" /data/x \t\n")); + Assertions.assertEquals("s3://bucket/p", LanceIndexDatasetLocator.normalize(" s3://bucket/p ")); + } + + @Test + public void stripsTrailingSlashesFromPath() { + Assertions.assertEquals("s3://b/p", LanceIndexDatasetLocator.normalize("s3://b/p///")); + Assertions.assertEquals("/data/lance", LanceIndexDatasetLocator.normalize("/data/lance/")); + } + + @Test + public void keepsSchemeLessRoot() { + Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("/")); + Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("//")); + Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("///")); + } + + @Test + public void stripsTrailingSlashBehindAuthority() { + Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket/")); + Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket//")); + Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket")); + } + + @Test + public void acceptsFileUrlWithEmptyAuthority() { + Assertions.assertEquals("file:///data/x", LanceIndexDatasetLocator.normalize("file:///data/x")); + } + + @Test + public void keepsAuthorityPortAndQueryFreePathVerbatim() { + Assertions.assertEquals("hdfs://nn:8020/base/table.lance", + LanceIndexDatasetLocator.normalize("hdfs://nn:8020/base/table.lance")); + } + + @Test + public void allowsAtSignInPathButNotInAuthority() { + Assertions.assertEquals("s3://bucket/p@th", LanceIndexDatasetLocator.normalize("s3://bucket/p@th")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("s3://user@bucket/path")); + } + + @Test + public void rejectsCredentialBearingUserinfo() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("s3://user:secret@bucket/path")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("https://user:secret@example.com/ds")); + } + + @Test + public void rejectsSchemeLessRelativePath() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("data/x")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("./x")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("bucket/path")); + } + + @Test + public void rejectsNullEmptyAndBlank() { + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("")); + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(" ")); + } + + @Test + public void rejectsEmptyScheme() { + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("://path")); + } + + @Test + public void rejectsLocatorWithNeitherAuthorityNorPath() { + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("s3://")); + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("file://")); + // Trailing-slash stripping still leaves nothing behind the scheme. + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("s3:///")); + // An empty authority with a real path stays legal. + Assertions.assertEquals("file:///x", LanceIndexDatasetLocator.normalize("file:///x")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java new file mode 100644 index 00000000000000..b37dabd6d99bfb --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.common.DdlException; +import org.apache.doris.journal.JournalEntity; +import org.apache.doris.persist.OperationType; +import org.apache.doris.persist.gson.GsonUtils; + +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Persistence coverage for the Lance index job infrastructure: the manager image + * write/read round trip (including the fence-index and quota rebuild in + * gsonPostProcess), the journal mounting point for op code 500 + * ({@link OperationType#OP_LANCE_INDEX_JOB_UPSERT}) through + * {@link JournalEntity#write}/{@link JournalEntity#readFields}, the single-record Gson + * round trip field by field, and the bounded-text rejection at construction time. + */ +public class LanceIndexJobManagerPersistTest { + private static final long CATALOG_ID = 10L; + private static final String LOCATOR = "s3://bucket/dataset"; + private static final long BACKEND_ID = 1001L; + private static final long BE_EPOCH = 55L; + private static final String INVOCATION_ID = "invocation-1"; + + @Test + public void managerImageRoundtripRebuildsDerivedFenceAndQuota() throws Exception { + TestManager source = new TestManager(); + source.createJob(newCreateJob(1L, "IdxPending"), 100, 100, 100); + source.createJob(newCreateJob(2L, "IdxRunning"), 100, 100, 100); + source.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L); + source.createJob(newCreateJob(3L, "IdxCommitted"), 100, 100, 100); + source.markRunning(3L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L); + source.completeWithResult(3L, 1L, INVOCATION_ID, BE_EPOCH, + new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, + LanceIndexJobCompletionReason.NONE, "ok", false)); + source.createJob(newCreateJob(4L, "IdxUnknown"), 100, 100, 100); + source.markRunning(4L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L); + source.completeWithResult(4L, 1L, INVOCATION_ID, BE_EPOCH, + new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT, + LanceIndexJobCompletionReason.NONE, "lost", false)); + source.replayUpsertJob(forceReleasedUnknownJob(5L, "idxforced")); + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + source.write(new DataOutputStream(byteStream)); + LanceIndexJobManager loaded = + LanceIndexJobManager.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()))); + + Assertions.assertEquals(source.getJobCount(), loaded.getJobCount()); + for (long jobId = 1L; jobId <= 5L; jobId++) { + Assertions.assertEquals(GsonUtils.GSON.toJson(source.getJob(jobId)), + GsonUtils.GSON.toJson(loaded.getJob(jobId)), "job " + jobId); + } + + // The derived fence index survived: unresolved jobs still fence their names. + for (long jobId = 1L; jobId <= 4L; jobId++) { + Assertions.assertTrue(loaded.isFenceHeld(loaded.getJob(jobId).fenceKey()), "fence of job " + jobId); + } + Assertions.assertFalse(loaded.isFenceHeld(loaded.getJob(5L).fenceKey())); + Assertions.assertEquals(4L, loaded.getQuota().getGlobalCount()); + Assertions.assertEquals(4L, loaded.getQuota().getCatalogCount(CATALOG_ID)); + Assertions.assertEquals(4, loaded.getUnresolvedJobs().size()); + Assertions.assertEquals(1, loaded.getJobsNeedingRefresh().size()); + + // The fence is enforceable after the image load: a conflict is rejected by the fence + // CAS, which precedes any journal write. + Assertions.assertThrows(DdlException.class, + () -> loaded.createJob(newCreateJob(9L, "idxunknown"), 100, 100, 100)); + + // And a FORCE-released name is free again once the loaded image is rebuilt through replay. + TestManager rebuilt = new TestManager(); + for (long jobId = 1L; jobId <= 5L; jobId++) { + rebuilt.replayUpsertJob(loaded.getJob(jobId)); + } + rebuilt.createJob(newCreateJob(9L, "IdxForced"), 100, 100, 100); + Assertions.assertEquals(5L, rebuilt.getQuota().getGlobalCount()); + } + + @Test + public void emptyManagerRoundtripStaysEmpty() throws Exception { + TestManager source = new TestManager(); + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + source.write(new DataOutputStream(byteStream)); + LanceIndexJobManager loaded = + LanceIndexJobManager.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()))); + + Assertions.assertEquals(0, loaded.getJobCount()); + Assertions.assertEquals(0L, loaded.getQuota().getGlobalCount()); + Assertions.assertTrue(loaded.getUnresolvedJobs().isEmpty()); + } + + @Test + public void imageLoadKeepsTheSmallerJobIdOnAFenceCollision() { + // Only a corrupt image can hold two unresolved jobs on the same fence key; + // gsonPostProcess keeps the smaller job id (Gson auto-runs it after fromJson). + String jobJson = "\"jid\":%d,\"rev\":0,\"cid\":" + CATALOG_ID + + ",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY + "\",\"loc\":\"" + LOCATOR + + "\",\"din\":\"IdxA\",\"nin\":\"idxa\",\"ms\":\"PENDING\",\"rs\":\"NOT_REQUIRED\""; + String json = "{\"jobs\":{\"2\":{" + String.format(jobJson, 2L) + "}," + + "\"1\":{" + String.format(jobJson, 1L) + "}}}"; + LanceIndexJobManager loaded = GsonUtils.GSON.fromJson(json, LanceIndexJobManager.class); + + Assertions.assertNotNull(loaded.getJob(1L)); + Assertions.assertTrue(loaded.isFenceHeld(loaded.getJob(1L).fenceKey())); + // Both unresolved jobs still charge the quota. + Assertions.assertEquals(2L, loaded.getQuota().getGlobalCount()); + // The fence conflict message names the surviving smaller job id. + DdlException exception = Assertions.assertThrows(DdlException.class, + () -> loaded.createJob(newCreateJob(9L, "idxa"), 100, 100, 100)); + Assertions.assertTrue(exception.getMessage().contains("unresolved job 1")); + } + + @Test + public void schemaContractEqualityIsOrderSensitive() { + LanceIndexSchemaContract ordered = contract(); + LanceIndexSchemaContract sameOrder = contract(); + List reversed = new ArrayList<>(); + reversed.add(ordered.getFields().get(1)); + reversed.add(ordered.getFields().get(0)); + LanceIndexSchemaContract reordered = new LanceIndexSchemaContract(reversed); + + Assertions.assertEquals(ordered, sameOrder); + Assertions.assertEquals(ordered.hashCode(), sameOrder.hashCode()); + // The same fields in a different order are a different contract. + Assertions.assertNotEquals(ordered, reordered); + } + + @Test + public void journalEntityRoundtripUsesOpCode500() throws Exception { + LanceIndexJob job = fullyPopulatedJob(); + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(byteStream); + JournalEntity journalEntity = new JournalEntity(); + journalEntity.setData(job); + journalEntity.setOpCode(OperationType.OP_LANCE_INDEX_JOB_UPSERT); + journalEntity.write(output); + output.flush(); + + JournalEntity replayed = new JournalEntity(); + replayed.readFields(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()))); + + Assertions.assertEquals(OperationType.OP_LANCE_INDEX_JOB_UPSERT, replayed.getOpCode()); + Assertions.assertEquals(500, replayed.getOpCode()); + Assertions.assertTrue(replayed.getData() instanceof LanceIndexJob); + Assertions.assertEquals(GsonUtils.GSON.toJson(job), GsonUtils.GSON.toJson(replayed.getData())); + } + + @Test + public void journalEntityStreamPreservesRecordOrder() throws Exception { + List jobs = new ArrayList<>(); + jobs.add(fullyPopulatedJob()); + LanceIndexJob second = newCreateJob(77L, "IdxSecond"); + second.setMutationState(LanceIndexJobMutationState.PENDING); + jobs.add(second); + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + DataOutputStream output = new DataOutputStream(byteStream); + for (LanceIndexJob job : jobs) { + JournalEntity journalEntity = new JournalEntity(); + journalEntity.setData(job); + journalEntity.setOpCode(OperationType.OP_LANCE_INDEX_JOB_UPSERT); + journalEntity.write(output); + } + output.flush(); + + DataInputStream input = new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())); + for (LanceIndexJob expected : jobs) { + JournalEntity replayed = new JournalEntity(); + replayed.readFields(input); + Assertions.assertEquals(OperationType.OP_LANCE_INDEX_JOB_UPSERT, replayed.getOpCode()); + Assertions.assertEquals(GsonUtils.GSON.toJson(expected), GsonUtils.GSON.toJson(replayed.getData())); + } + Assertions.assertEquals(0, input.available()); + } + + @Test + public void jobStreamRoundtripPreservesAllFields() throws Exception { + LanceIndexJob job = fullyPopulatedJob(); + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + job.write(new DataOutputStream(byteStream)); + LanceIndexJob restored = + LanceIndexJob.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()))); + + assertSameJobFields(job, restored); + } + + @Test + public void jobGsonRoundtripPreservesAllFields() { + LanceIndexJob job = fullyPopulatedJob(); + LanceIndexJob restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(job), LanceIndexJob.class); + assertSameJobFields(job, restored); + } + + @Test + public void boundedTextFieldsRejectOverflow() { + String overMessage = StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES + 1); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, + LanceIndexJobCompletionReason.NONE, overMessage, false)); + // Multibyte characters count as UTF-8 bytes, not chars. + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, + LanceIndexJobCompletionReason.NONE, StringUtils.repeat("é", 513), false)); + new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, LanceIndexJobCompletionReason.NONE, + StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES), false); + + LanceIndexJob job = newCreateJob(1L, "IdxA"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES + 1))); + job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setForceNote(StringUtils.repeat("n", LanceIndexJob.MAX_FORCE_TEXT_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setForceWarning(StringUtils.repeat("w", LanceIndexJob.MAX_FORCE_TEXT_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setNormalizedIndexName( + StringUtils.repeat("n", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setDisplayIndexName( + StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> newCreateJob(2L, StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1))); + } + + @Test + public void resultRejectsNullResultCode() { + Assertions.assertThrows(NullPointerException.class, + () -> new LanceIndexJobResult(null, LanceIndexJobCompletionReason.NONE, "msg", false)); + } + + private static LanceIndexJob newCreateJob(long jobId, String displayName) { + return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, + displayName, LanceIndexNameNormalizer.normalize(displayName), + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", + null, 7L, null); + } + + private static LanceIndexSchemaContract contract() { + List fields = new ArrayList<>(); + fields.add(new LanceIndexSchemaContract.IndexedField(1L, "v", "fixed_size_list[float;192]", + false, 192, "float", false)); + fields.add(new LanceIndexSchemaContract.IndexedField(2L, "s", "decimal(10,2)", true, null, null, null)); + return new LanceIndexSchemaContract(fields); + } + + /** + * A job with every durable field populated, including result, schema contract, + * dispatch identity, and the FORCE audit fields. + */ + private static LanceIndexJob fullyPopulatedJob() { + LanceIndexJob job = new LanceIndexJob(42L, "creator", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxΩ", "idxω", + LanceIndexJobMutationType.REPLACE, true, false, "IVF_PQ", "v", + "{\"num_partitions\":\"256\"}", 99L, contract()); + job.setRevision(7L); + job.setCreateTimeMs(111L); + job.setUpdateTimeMs(222L); + job.setMutationState(LanceIndexJobMutationState.RUNNING); + job.setRefreshState(LanceIndexJobRefreshState.RUNNING); + job.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_IO, + LanceIndexJobCompletionReason.NONE, "io error", true)); + job.setBackendId(BACKEND_ID); + job.setBeProcessEpoch(BE_EPOCH); + job.setInvocationId(INVOCATION_ID); + job.setDeadlineMs(123456L); + job.setPossibleLiveOwned(true); + job.setTerminationProof(LanceIndexTerminationProof.NONE); + job.setForceActor("admin"); + job.setForceTimeMs(777L); + job.setForceNote("note"); + job.setForceWarning("warning"); + return job; + } + + private static LanceIndexJob forceReleasedUnknownJob(long jobId, String normalizedName) { + String json = "{\"jid\":" + jobId + ",\"cr\":\"tester\",\"rev\":2,\"cid\":" + CATALOG_ID + + ",\"dbn\":\"db1\",\"tbn\":\"tbl1\",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY + + "\",\"loc\":\"" + LOCATOR + "\",\"din\":\"" + normalizedName + "\",\"nin\":\"" + normalizedName + + "\",\"mt\":\"CREATE\",\"ms\":\"UNKNOWN\",\"rs\":\"NOT_REQUIRED\",\"fr\":true}"; + return GsonUtils.GSON.fromJson(json, LanceIndexJob.class); + } + + private static void assertSameJobFields(LanceIndexJob expected, LanceIndexJob actual) { + Assertions.assertEquals(expected.getJobId(), actual.getJobId()); + Assertions.assertEquals(expected.getCreator(), actual.getCreator()); + Assertions.assertEquals(expected.getRevision(), actual.getRevision()); + Assertions.assertEquals(expected.getCreateTimeMs(), actual.getCreateTimeMs()); + Assertions.assertEquals(expected.getUpdateTimeMs(), actual.getUpdateTimeMs()); + Assertions.assertEquals(expected.getCatalogId(), actual.getCatalogId()); + Assertions.assertEquals(expected.getDbName(), actual.getDbName()); + Assertions.assertEquals(expected.getTableName(), actual.getTableName()); + Assertions.assertEquals(expected.getProvider(), actual.getProvider()); + Assertions.assertEquals(expected.getNormalizedLocator(), actual.getNormalizedLocator()); + Assertions.assertEquals(expected.getDisplayIndexName(), actual.getDisplayIndexName()); + Assertions.assertEquals(expected.getNormalizedIndexName(), actual.getNormalizedIndexName()); + Assertions.assertEquals(expected.getMutationType(), actual.getMutationType()); + Assertions.assertEquals(expected.isIfNotExists(), actual.isIfNotExists()); + Assertions.assertEquals(expected.isIfExists(), actual.isIfExists()); + Assertions.assertEquals(expected.getIndexType(), actual.getIndexType()); + Assertions.assertEquals(expected.getColumnName(), actual.getColumnName()); + Assertions.assertEquals(expected.getPropertiesJson(), actual.getPropertiesJson()); + Assertions.assertEquals(expected.getAdmittedDatasetVersion(), actual.getAdmittedDatasetVersion()); + Assertions.assertEquals(expected.getSchemaContract(), actual.getSchemaContract()); + Assertions.assertEquals(expected.getMutationState(), actual.getMutationState()); + Assertions.assertEquals(expected.getRefreshState(), actual.getRefreshState()); + Assertions.assertEquals(expected.getResult().getResultCode(), actual.getResult().getResultCode()); + Assertions.assertEquals(expected.getResult().getCompletionReason(), actual.getResult().getCompletionReason()); + Assertions.assertEquals(expected.getResult().getSanitizedMessage(), actual.getResult().getSanitizedMessage()); + Assertions.assertEquals(expected.getResult().isExternalMetadataAdvanced(), + actual.getResult().isExternalMetadataAdvanced()); + Assertions.assertEquals(expected.getBackendId(), actual.getBackendId()); + Assertions.assertEquals(expected.getBeProcessEpoch(), actual.getBeProcessEpoch()); + Assertions.assertEquals(expected.getInvocationId(), actual.getInvocationId()); + Assertions.assertEquals(expected.getDeadlineMs(), actual.getDeadlineMs()); + Assertions.assertEquals(expected.isPossibleLiveOwned(), actual.isPossibleLiveOwned()); + Assertions.assertEquals(expected.getTerminationProof(), actual.getTerminationProof()); + Assertions.assertEquals(expected.isForceReleased(), actual.isForceReleased()); + Assertions.assertEquals(expected.getForceActor(), actual.getForceActor()); + Assertions.assertEquals(expected.getForceTimeMs(), actual.getForceTimeMs()); + Assertions.assertEquals(expected.getForceNote(), actual.getForceNote()); + Assertions.assertEquals(expected.getForceWarning(), actual.getForceWarning()); + Assertions.assertEquals(expected.fenceKey(), actual.fenceKey()); + Assertions.assertEquals(expected.getTableQuotaKey(), actual.getTableQuotaKey()); + } + + /** + * Edit-log seam: captures every durable record instead of writing the journal. + */ + private static class TestManager extends LanceIndexJobManager { + @Override + protected void writeEditLog(LanceIndexJob job) { + // No journal in a pure persistence unit test. + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java new file mode 100644 index 00000000000000..ecf7e3b07b9088 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java @@ -0,0 +1,431 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.common.DdlException; +import org.apache.doris.persist.gson.GsonUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Replay and master-transfer coverage for {@link LanceIndexJobManager}. The journal + * records of a scenario are captured from a source manager through the edit-log seam + * and replayed verbatim into a fresh target manager, exactly like a follower tailing + * the edit log. The pinned invariants: replay is a verbatim replace with a monotonic + * revision guard (no state transformation on followers); a replayed PENDING permits + * exactly one dispatch; only the master-election sweep turns a durable RUNNING into + * UNKNOWN, after which redispatch is permanently refused; an unresolved UNKNOWN keeps + * its fence and quota across replay while a FORCE-released UNKNOWN frees both; stale + * callbacks (revision / invocation id / BE epoch) never change state; and a rejected + * admission leaves no job, fence, quota charge, or journal record behind. + */ +public class LanceIndexJobManagerReplayTest { + private static final long CATALOG_ID = 10L; + private static final String LOCATOR = "s3://bucket/dataset"; + private static final long BACKEND_ID = 1001L; + private static final long BE_EPOCH = 55L; + private static final String INVOCATION_ID = "invocation-1"; + private static final long DEADLINE_MS = 9999L; + + @Test + public void replayedPendingPermitsExactlyOneDispatch() throws DdlException { + List records = runningRecords(1L, "IdxA"); + TestManager target = new TestManager(); + target.replayUpsertJob(records.get(0)); + + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(1L).getMutationState()); + Assertions.assertTrue(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(target.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, target.getJob(1L).getMutationState()); + } + + @Test + public void replayedRunningStaysRunningWithoutSweep() throws DdlException { + List records = runningRecords(1L, "IdxA"); + TestManager target = new TestManager(); + target.replayUpsertJob(records.get(0)); + target.replayUpsertJob(records.get(1)); + + LanceIndexJob stored = target.getJob(1L); + // A follower tailing a live master must not transform a fresh RUNNING record. + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState()); + Assertions.assertEquals(1L, stored.getRevision()); + Assertions.assertTrue(stored.holdsPossibleLiveSlot()); + Assertions.assertTrue(target.isFenceHeld(stored.fenceKey())); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + // Replay itself never writes the journal. + Assertions.assertTrue(target.editLog.isEmpty()); + } + + @Test + public void transferToMasterConvertsRunningToUnknownAndNeverRedispatches() throws DdlException { + List records = runningRecords(1L, "IdxA"); + TestManager target = new TestManager(); + target.replayUpsertJob(records.get(0)); + target.replayUpsertJob(records.get(1)); + LanceIndexFenceKey fenceKey = target.getJob(1L).fenceKey(); + + target.onTransferToMaster(); + + LanceIndexJob swept = target.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState()); + Assertions.assertEquals(2L, swept.getRevision()); + Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, swept.getResult().getResultCode()); + // Fence, quota, and the possible-live slot survive the sweep: only FORCE releases them. + Assertions.assertTrue(target.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + Assertions.assertTrue(swept.holdsPossibleLiveSlot()); + Assertions.assertTrue(target.getUnresolvedJobs().contains(swept)); + Assertions.assertEquals(1, target.editLog.size()); + + Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(target.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(1L).getMutationState()); + } + + @Test + public void transferToMasterDowngradesRunningRefreshToRequired() throws DdlException { + TestManager source = new TestManager(); + source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult()); + source.markRefreshRunning(1L, 2L); + + TestManager target = new TestManager(); + for (LanceIndexJob record : source.editLog) { + target.replayUpsertJob(record); + } + Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, target.getJob(1L).getRefreshState()); + + target.onTransferToMaster(); + + LanceIndexJob swept = target.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, swept.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, swept.getRefreshState()); + Assertions.assertEquals(4L, swept.getRevision()); + Assertions.assertTrue(target.getJobsNeedingRefresh().contains(swept)); + Assertions.assertTrue(target.isFenceHeld(swept.fenceKey())); + } + + @Test + public void replayedTerminalWithRefreshRequiredOnlyAllowsRefreshPath() throws DdlException { + TestManager source = new TestManager(); + source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult()); + + TestManager target = new TestManager(); + for (LanceIndexJob record : source.editLog) { + target.replayUpsertJob(record); + } + LanceIndexJob stored = target.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState()); + LanceIndexFenceKey fenceKey = stored.fenceKey(); + + // The mutation lifecycle is closed; only the refresh transitions remain. + Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(target.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, okResult())); + Assertions.assertTrue(target.getJobsNeedingRefresh().contains(stored)); + Assertions.assertTrue(target.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + + Assertions.assertTrue(target.markRefreshRunning(1L, 2L)); + Assertions.assertTrue(target.markRefreshDone(1L, 3L)); + Assertions.assertFalse(target.isFenceHeld(fenceKey)); + Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); + Assertions.assertTrue(target.getUnresolvedJobs().isEmpty()); + } + + @Test + public void replayedUnresolvedUnknownFencesTheSameName() throws DdlException { + TestManager source = new TestManager(); + source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT, + LanceIndexJobCompletionReason.NONE, "ambiguous", false)); + + TestManager target = new TestManager(); + for (LanceIndexJob record : source.editLog) { + target.replayUpsertJob(record); + } + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(1L).getMutationState()); + + // The unforced UNKNOWN still holds the same-name fence and the quota. + Assertions.assertThrows(DdlException.class, + () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100)); + Assertions.assertThrows(DdlException.class, + () -> target.createJob(newCreateJob(9L, "idxa"), 100, 100, 100)); + target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100); + Assertions.assertEquals(2L, target.getQuota().getGlobalCount()); + } + + @Test + public void replayedForceReleasedUnknownFreesNameAndQuota() { + TestManager target = new TestManager(); + target.replayUpsertJob(forceReleasedUnknownJob(7L, "idxforce")); + + Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); + Assertions.assertFalse(target.isFenceHeld(target.getJob(7L).fenceKey())); + Assertions.assertTrue(target.getUnresolvedJobs().isEmpty()); + + Assertions.assertDoesNotThrow(() -> target.createJob(newCreateJob(8L, "IdxForce"), 100, 100, 100)); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + Assertions.assertEquals(1, target.editLog.size()); + } + + @Test + public void staleCallbacksAreRejectedWithoutStateChange() throws DdlException { + List records = runningRecords(1L, "IdxA"); + TestManager target = new TestManager(); + target.replayUpsertJob(records.get(0)); + target.replayUpsertJob(records.get(1)); + + Assertions.assertFalse(target.completeWithResult(1L, 0L, INVOCATION_ID, BE_EPOCH, okResult())); + Assertions.assertFalse(target.completeWithResult(1L, 1L, "invocation-x", BE_EPOCH, okResult())); + Assertions.assertFalse(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH + 1, okResult())); + Assertions.assertFalse(target.completeWithResult(1L, 1L, INVOCATION_ID, null, okResult())); + Assertions.assertFalse(target.completeWithResult(404L, 1L, INVOCATION_ID, BE_EPOCH, okResult())); + + LanceIndexJob stored = target.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState()); + Assertions.assertEquals(1L, stored.getRevision()); + Assertions.assertNull(stored.getResult()); + Assertions.assertTrue(target.editLog.isEmpty()); + + Assertions.assertTrue(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult())); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, target.getJob(1L).getMutationState()); + } + + @Test + public void replayIsIdempotentForTheSameRecord() throws DdlException { + List records = runningRecords(1L, "IdxA"); + TestManager target = new TestManager(); + target.replayUpsertJob(records.get(0)); + target.replayUpsertJob(records.get(0)); + target.replayUpsertJob(records.get(1)); + target.replayUpsertJob(records.get(1)); + + Assertions.assertEquals(1, target.getJobCount()); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, target.getJob(1L).getMutationState()); + Assertions.assertEquals(1L, target.getJob(1L).getRevision()); + } + + @Test + public void lowerRevisionRecordNeverOverwritesHigherRevision() throws DdlException { + TestManager source = new TestManager(); + source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult()); + // Records: PENDING rev0, RUNNING rev1, COMMITTED+REQUIRED rev2. + + TestManager target = new TestManager(); + target.replayUpsertJob(source.editLog.get(2)); + target.replayUpsertJob(source.editLog.get(0)); + target.replayUpsertJob(source.editLog.get(1)); + + LanceIndexJob stored = target.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState()); + Assertions.assertEquals(2L, stored.getRevision()); + Assertions.assertTrue(target.isFenceHeld(stored.fenceKey())); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + Assertions.assertTrue(target.editLog.isEmpty()); + + // An equal revision replaces verbatim (idempotent re-delivery). + target.replayUpsertJob(source.editLog.get(2)); + Assertions.assertEquals(2L, target.getJob(1L).getRevision()); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + } + + @Test + public void replayToleratesNullAndIdentityLessRecords() { + TestManager target = new TestManager(); + target.replayUpsertJob(null); + Assertions.assertEquals(0, target.getJobCount()); + + // A corrupt record without fence identity stays queryable but out of the books. + LanceIndexJob sparse = GsonUtils.GSON.fromJson( + "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class); + target.replayUpsertJob(sparse); + Assertions.assertEquals(1, target.getJobCount()); + Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(5L).getMutationState()); + } + + @Test + public void replayingOverAnIdentityLessRecordNeverThrows() { + TestManager target = new TestManager(); + LanceIndexJob sparse = GsonUtils.GSON.fromJson( + "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class); + target.replayUpsertJob(sparse); + Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); + + // Re-delivering the same identity-less record must not key the release side on it. + Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(sparse)); + Assertions.assertEquals(1, target.getJobCount()); + Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); + + // Neither must a higher-revision identity-less upsert of the same job id. + LanceIndexJob unknown = GsonUtils.GSON.fromJson( + "{\"jid\":5,\"rev\":1,\"ms\":\"UNKNOWN\"}", LanceIndexJob.class); + Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(unknown)); + LanceIndexJob stored = target.getJob(5L); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState()); + Assertions.assertEquals(1L, stored.getRevision()); + Assertions.assertEquals(1, target.getJobCount()); + Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); + } + + @Test + public void transferToMasterSweepsOnlyTheRunningJobInAMixedPopulation() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxPending"), 100, 100, 100); + manager.createJob(newCreateJob(2L, "IdxCommitted"), 100, 100, 100); + manager.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH, okResult()); + manager.markRefreshRunning(2L, 2L); + manager.markRefreshDone(2L, 3L); + manager.createJob(newCreateJob(3L, "IdxRunning"), 100, 100, 100); + manager.markRunning(3L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + // Journal so far: create(1), create+run+complete+refreshRun+refreshDone(2), create+run(3). + Assertions.assertEquals(8, manager.editLog.size()); + + manager.onTransferToMaster(); + + // PENDING and the settled terminal job are replay-faithful: untouched. + LanceIndexJob pending = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, pending.getMutationState()); + Assertions.assertEquals(0L, pending.getRevision()); + LanceIndexJob committed = manager.getJob(2L); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.DONE, committed.getRefreshState()); + Assertions.assertEquals(4L, committed.getRevision()); + + // Only the durable RUNNING becomes UNKNOWN, through exactly one journal record. + LanceIndexJob swept = manager.getJob(3L); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState()); + Assertions.assertEquals(2L, swept.getRevision()); + Assertions.assertEquals(9, manager.editLog.size()); + } + + @Test + public void fenceRejectionLeavesNoJobNoQuotaAndNoJournalRecord() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + DdlException exception = Assertions.assertThrows(DdlException.class, + () -> manager.createJob(newCreateJob(2L, "idxa"), 100, 100, 100)); + // The rejection must not disclose the dataset locator to an unauthorized caller. + Assertions.assertFalse(exception.getMessage().contains("bucket")); + Assertions.assertFalse(exception.getMessage().contains(LOCATOR)); + + Assertions.assertEquals(1, manager.getJobCount()); + Assertions.assertEquals(1, manager.editLog.size()); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + // Only the original job's fence exists; nothing new was registered for the rejection. + Assertions.assertFalse(manager.isFenceHeld(newCreateJob(3L, "IdxB").fenceKey())); + } + + @Test + public void quotaRejectionLeavesNoJobNoFenceAndNoJournalRecord() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 1, 1, 1); + + Assertions.assertThrows(DdlException.class, + () -> manager.createJob(newCreateJob(2L, "IdxB"), 1, 1, 1)); + + Assertions.assertEquals(1, manager.getJobCount()); + Assertions.assertEquals(1, manager.editLog.size()); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + Assertions.assertFalse(manager.isFenceHeld(newCreateJob(2L, "IdxB").fenceKey())); + } + + @Test + public void duplicateJobIdIsRejectedBeforeAnythingElse() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + Assertions.assertThrows(DdlException.class, + () -> manager.createJob(newCreateJob(1L, "IdxB"), 100, 100, 100)); + Assertions.assertEquals(1, manager.getJobCount()); + Assertions.assertEquals(1, manager.editLog.size()); + Assertions.assertFalse(manager.isFenceHeld(newCreateJob(1L, "IdxB").fenceKey())); + } + + private static LanceIndexJob newCreateJob(long jobId, String displayName) { + return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, + displayName, LanceIndexNameNormalizer.normalize(displayName), + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", + null, 7L, null); + } + + private static LanceIndexJobResult okResult() { + return new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, + LanceIndexJobCompletionReason.NONE, "ok", false); + } + + /** + * Runs create + dispatch on a throwaway source manager and returns its journal + * records: [PENDING rev0, RUNNING rev1]. + */ + private static List runningRecords(long jobId, String displayName) throws DdlException { + TestManager source = new TestManager(); + source.createJob(newCreateJob(jobId, displayName), 100, 100, 100); + source.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + return source.editLog; + } + + /** + * Builds the durable form of a FORCE-released UNKNOWN job. The FORCE slice owns the + * transition itself; here the record only exists as replay input, so it is built + * from its JSON journal form. + */ + private static LanceIndexJob forceReleasedUnknownJob(long jobId, String normalizedName) { + String json = "{\"jid\":" + jobId + ",\"cr\":\"tester\",\"rev\":2,\"cid\":" + CATALOG_ID + + ",\"dbn\":\"db1\",\"tbn\":\"tbl1\",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY + + "\",\"loc\":\"" + LOCATOR + "\",\"din\":\"" + normalizedName + "\",\"nin\":\"" + normalizedName + + "\",\"mt\":\"CREATE\",\"ms\":\"UNKNOWN\",\"rs\":\"NOT_REQUIRED\",\"fr\":true," + + "\"fa\":\"admin\",\"ftm\":12345,\"fn\":\"operator note\"}"; + LanceIndexJob job = GsonUtils.GSON.fromJson(json, LanceIndexJob.class); + Assertions.assertTrue(job.isForceReleased()); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, job.getMutationState()); + return job; + } + + /** + * Edit-log seam: captures every durable record instead of writing the journal. + */ + private static class TestManager extends LanceIndexJobManager { + private final List editLog = new ArrayList<>(); + + @Override + protected void writeEditLog(LanceIndexJob job) { + editLog.add(job); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java new file mode 100644 index 00000000000000..c4527a21265a72 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Unit coverage for the three-level unresolved-job quota counters (table/locator, + * catalog, global): the "current + 1 <= limit" boundary at each level, disabled + * levels for non-positive limits, release recovery with underflow clamping, and rebuild + * equivalence with live counting. The "which jobs count" semantics are + * {@link LanceIndexJob#isUnresolved()}; the rebuild-side composition is pinned here too. + */ +public class LanceIndexJobQuotaTest { + private static final long CATALOG_ID = 10L; + private static final String LOCATOR_A = "s3://bucket/table-a"; + private static final String LOCATOR_B = "s3://bucket/table-b"; + + @Test + public void tryAcquireChargesAllThreeLevels() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + LanceIndexJob job = newJob(1L, CATALOG_ID, LOCATOR_A); + + Assertions.assertTrue(quota.tryAcquire(job, 5, 5, 5)); + Assertions.assertEquals(1L, quota.getGlobalCount()); + Assertions.assertEquals(1L, quota.getCatalogCount(CATALOG_ID)); + Assertions.assertEquals(1L, quota.getTableCount(job.getTableQuotaKey())); + } + + @Test + public void tableLimitRejectsTheNextJobExactlyAtLimit() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 2, 0, 0)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 2, 0, 0)); + + LanceIndexJob third = newJob(3L, CATALOG_ID, LOCATOR_A); + Assertions.assertFalse(quota.tryAcquire(third, 2, 0, 0)); + // A rejected acquire charges nothing at any level. + Assertions.assertEquals(2L, quota.getGlobalCount()); + Assertions.assertEquals(2L, quota.getTableCount(third.getTableQuotaKey())); + + // The limit is per table/locator identity: another table still has room. + Assertions.assertTrue(quota.tryAcquire(newJob(4L, CATALOG_ID, LOCATOR_B), 2, 0, 0)); + } + + @Test + public void catalogLimitRejectsAcrossTables() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 0, 2, 0)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_B), 0, 2, 0)); + Assertions.assertFalse(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 0, 2, 0)); + + // Another catalog is a separate level. + Assertions.assertTrue(quota.tryAcquire(newJob(4L, 20L, LOCATOR_A), 0, 2, 0)); + } + + @Test + public void globalLimitRejectsAcrossCatalogs() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 0, 0, 2)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, 20L, LOCATOR_A), 0, 0, 2)); + Assertions.assertFalse(quota.tryAcquire(newJob(3L, 30L, LOCATOR_B), 0, 0, 2)); + Assertions.assertEquals(2L, quota.getGlobalCount()); + } + + @Test + public void nonPositiveLimitDisablesThatLevel() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + for (int i = 0; i < 10; i++) { + Assertions.assertTrue(quota.tryAcquire(newJob(i, CATALOG_ID, LOCATOR_A), 0, 0, 0)); + } + LanceIndexJobQuota negativeLimits = new LanceIndexJobQuota(); + for (int i = 0; i < 10; i++) { + Assertions.assertTrue(negativeLimits.tryAcquire(newJob(i, CATALOG_ID, LOCATOR_A), -1, -1, -1)); + } + Assertions.assertEquals(10L, quota.getGlobalCount()); + Assertions.assertEquals(10L, negativeLimits.getGlobalCount()); + } + + @Test + public void releaseRecoversCapacityAtAllLevels() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + LanceIndexJob first = newJob(1L, CATALOG_ID, LOCATOR_A); + Assertions.assertTrue(quota.tryAcquire(first, 1, 1, 1)); + Assertions.assertFalse(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 1, 1, 1)); + + quota.release(first); + Assertions.assertEquals(0L, quota.getGlobalCount()); + Assertions.assertEquals(0L, quota.getCatalogCount(CATALOG_ID)); + Assertions.assertEquals(0L, quota.getTableCount(first.getTableQuotaKey())); + + Assertions.assertTrue(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 1, 1, 1)); + } + + @Test + public void releaseUnderflowClampsAtZero() { + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + // Releasing a job that was never charged must not fail and must not go negative. + quota.release(newJob(1L, CATALOG_ID, LOCATOR_A)); + Assertions.assertEquals(0L, quota.getGlobalCount()); + Assertions.assertEquals(0L, quota.getCatalogCount(CATALOG_ID)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 1, 1, 1)); + } + + @Test + public void rebuildMatchesIncrementalCounting() { + List unresolved = new ArrayList<>(); + unresolved.add(newJob(1L, CATALOG_ID, LOCATOR_A)); + unresolved.add(newJob(2L, CATALOG_ID, LOCATOR_A)); + unresolved.add(newJob(3L, CATALOG_ID, LOCATOR_B)); + unresolved.add(newJob(4L, 20L, LOCATOR_A)); + + LanceIndexJobQuota incremental = new LanceIndexJobQuota(); + for (LanceIndexJob job : unresolved) { + Assertions.assertTrue(incremental.tryAcquire(job, 0, 0, 0)); + } + LanceIndexJobQuota rebuilt = new LanceIndexJobQuota(); + rebuilt.rebuild(unresolved); + + Assertions.assertEquals(incremental.getGlobalCount(), rebuilt.getGlobalCount()); + Assertions.assertEquals(incremental.getCatalogCount(CATALOG_ID), rebuilt.getCatalogCount(CATALOG_ID)); + Assertions.assertEquals(incremental.getCatalogCount(20L), rebuilt.getCatalogCount(20L)); + Assertions.assertEquals(incremental.getTableCount(unresolved.get(0).getTableQuotaKey()), + rebuilt.getTableCount(unresolved.get(0).getTableQuotaKey())); + Assertions.assertEquals(incremental.getTableCount(unresolved.get(2).getTableQuotaKey()), + rebuilt.getTableCount(unresolved.get(2).getTableQuotaKey())); + + rebuilt.rebuild(Collections.emptyList()); + Assertions.assertEquals(0L, rebuilt.getGlobalCount()); + Assertions.assertEquals(0L, rebuilt.getCatalogCount(CATALOG_ID)); + } + + @Test + public void rebuildCountsUnresolvedJobsOnly() { + List jobs = new ArrayList<>(); + // Active and unforced-unknown jobs count. + jobs.add(jobInState(1L, LanceIndexJobMutationState.PENDING, LanceIndexJobRefreshState.NOT_REQUIRED, false)); + jobs.add(jobInState(2L, LanceIndexJobMutationState.RUNNING, LanceIndexJobRefreshState.NOT_REQUIRED, false)); + jobs.add(jobInState(3L, LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, false)); + jobs.add(jobInState(4L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.REQUIRED, false)); + jobs.add(jobInState(5L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.RUNNING, false)); + jobs.add(jobInState(6L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.FAILED, false)); + jobs.add(jobInState(7L, LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.FAILED, false)); + // Resolved jobs do not count: forced UNKNOWN and terminal jobs with a settled refresh. + jobs.add(jobInState(8L, LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, true)); + jobs.add(jobInState(9L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.DONE, false)); + jobs.add(jobInState(10L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.NOT_REQUIRED, false)); + jobs.add(jobInState(11L, LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.NOT_REQUIRED, + false)); + + List unresolved = new ArrayList<>(); + for (LanceIndexJob job : jobs) { + if (job.isUnresolved()) { + unresolved.add(job); + } + } + Assertions.assertEquals(7, unresolved.size()); + + LanceIndexJobQuota quota = new LanceIndexJobQuota(); + quota.rebuild(unresolved); + Assertions.assertEquals(7L, quota.getGlobalCount()); + Assertions.assertEquals(7L, quota.getCatalogCount(CATALOG_ID)); + Assertions.assertEquals(7L, quota.getTableCount(unresolved.get(0).getTableQuotaKey())); + } + + @Test + public void tableQuotaKeyIdentityAndLocatorHiding() { + LanceIndexJobQuota.TableQuotaKey key = new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A); + Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A), key); + Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A).hashCode(), key.hashCode()); + Assertions.assertNotEquals(new LanceIndexJobQuota.TableQuotaKey(20L, LOCATOR_A), key); + Assertions.assertNotEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_B), key); + Assertions.assertFalse(key.toString().contains(LOCATOR_A)); + } + + private static LanceIndexJob newJob(long jobId, long catalogId, String locator) { + return new LanceIndexJob(jobId, "tester", catalogId, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, locator, + "idx" + jobId, "idx" + jobId, + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 1L, null); + } + + private static LanceIndexJob jobInState(long jobId, LanceIndexJobMutationState mutationState, + LanceIndexJobRefreshState refreshState, boolean forceReleased) { + LanceIndexJob job = newJob(jobId, CATALOG_ID, LOCATOR_A); + job.setMutationState(mutationState); + job.setRefreshState(refreshState); + job.setForceReleased(forceReleased); + return job; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java new file mode 100644 index 00000000000000..cbceb49caf1f05 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Exhaustive coverage of the provider-result classification table: every saved typed + * result code is classified into the durable (mutationState, refreshState, + * completionReason) triple for the full cross product of 13 codes x 3 mutation types + * x ifExists x externalMetadataAdvanced (156 combinations). The expected values are an + * independent restatement of the design table, so a row flipped on either side fails. + * + *

Key invariants pinned here: PRE_INVOCATION_* always prove NOT_COMMITTED and owe a + * refresh only when trusted revalidation observed external metadata advancement; + * NATIVE_NOT_FOUND has no attribution for CREATE/REPLACE (UNKNOWN) but is a clean + * NOT_COMMITTED for DROP; IF_CONDITION_NOOP exists only for DROP IF EXISTS + NOT_FOUND; + * every other ambiguous post-invocation outcome is UNKNOWN + NOT_REQUIRED. + */ +public class LanceIndexJobResultClassifyTest { + + @Test + public void fullClassificationProductMatchesDesignTable() { + int combos = 0; + for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) { + for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) { + for (boolean ifExists : new boolean[]{false, true}) { + for (boolean advanced : new boolean[]{false, true}) { + LanceIndexJobResultCode.Classification classification = + LanceIndexJobResultCode.classify(type, code, ifExists, advanced); + String context = "code=" + code + ", type=" + type + + ", ifExists=" + ifExists + ", advanced=" + advanced; + Assertions.assertEquals(expectedMutationState(type, code), + classification.getMutationState(), context); + Assertions.assertEquals(expectedRefreshState(type, code, advanced), + classification.getRefreshState(), context); + Assertions.assertEquals(expectedCompletionReason(type, code, ifExists), + classification.getCompletionReason(), context); + combos++; + } + } + } + } + Assertions.assertEquals(13 * 3 * 2 * 2, combos); + } + + @Test + public void ifConditionNoopExistsOnlyForDropIfExistsNotFound() { + int noopCombos = 0; + for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) { + for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) { + for (boolean ifExists : new boolean[]{false, true}) { + for (boolean advanced : new boolean[]{false, true}) { + LanceIndexJobResultCode.Classification classification = + LanceIndexJobResultCode.classify(type, code, ifExists, advanced); + if (classification.getCompletionReason() == LanceIndexJobCompletionReason.IF_CONDITION_NOOP) { + noopCombos++; + Assertions.assertEquals(LanceIndexJobMutationType.DROP, type); + Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_NOT_FOUND, code); + Assertions.assertTrue(ifExists); + } + } + } + } + } + // Exactly the two advanced-flag variants of (DROP, NATIVE_NOT_FOUND, ifExists). + Assertions.assertEquals(2, noopCombos); + } + + @Test + public void preInvocationRefreshOwesOnlyWhenExternalMetadataAdvanced() { + for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) { + if (!code.isPreInvocation()) { + continue; + } + for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) { + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, + LanceIndexJobResultCode.classify(type, code, false, false).getRefreshState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, + LanceIndexJobResultCode.classify(type, code, false, true).getRefreshState()); + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, + LanceIndexJobResultCode.classify(type, code, true, true).getMutationState()); + } + } + } + + @Test + public void nativeOkCommitsAndOwesRefreshForEveryMutationType() { + for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) { + LanceIndexJobResultCode.Classification classification = + LanceIndexJobResultCode.classify(type, LanceIndexJobResultCode.NATIVE_OK, false, false); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, classification.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, classification.getRefreshState()); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason()); + } + } + + @Test + public void commitConflictIsNotCommittedWithRefreshOwed() { + for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) { + LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify( + type, LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT, false, false); + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, classification.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, classification.getRefreshState()); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason()); + } + } + + @Test + public void nativeNotFoundAttributionDependsOnMutationType() { + for (LanceIndexJobMutationType type : new LanceIndexJobMutationType[]{ + LanceIndexJobMutationType.CREATE, LanceIndexJobMutationType.REPLACE}) { + LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify( + type, LanceIndexJobResultCode.NATIVE_NOT_FOUND, false, false); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState()); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason()); + } + LanceIndexJobResultCode.Classification dropClassification = LanceIndexJobResultCode.classify( + LanceIndexJobMutationType.DROP, LanceIndexJobResultCode.NATIVE_NOT_FOUND, false, false); + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, dropClassification.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, dropClassification.getRefreshState()); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, dropClassification.getCompletionReason()); + } + + @Test + public void ambiguousPostInvocationOutcomesAreUnknownWithoutRefresh() { + for (LanceIndexJobResultCode code : new LanceIndexJobResultCode[]{ + LanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT, LanceIndexJobResultCode.NATIVE_NOT_SUPPORTED, + LanceIndexJobResultCode.NATIVE_INDEX, LanceIndexJobResultCode.NATIVE_IO, + LanceIndexJobResultCode.NATIVE_INTERNAL, LanceIndexJobResultCode.NO_TRUSTED_RESULT}) { + for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) { + LanceIndexJobResultCode.Classification classification = + LanceIndexJobResultCode.classify(type, code, true, true); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState(), + "code=" + code + ", type=" + type); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState(), + "code=" + code + ", type=" + type); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason(), + "code=" + code + ", type=" + type); + } + } + } + + @Test + public void nullResultCodeFallsBackToNoTrustedResult() { + LanceIndexJobResultCode.Classification classification = + LanceIndexJobResultCode.classify(LanceIndexJobMutationType.CREATE, null, false, false); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState()); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason()); + } + + @Test + public void isPreInvocationCoversExactlyTheFourTrustedRejections() { + int preInvocationCount = 0; + for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) { + if (code.isPreInvocation()) { + preInvocationCount++; + } + } + Assertions.assertEquals(4, preInvocationCount); + Assertions.assertEquals(13, LanceIndexJobResultCode.values().length); + } + + /** + * Independent restatement of the design table, mutation-state column. + */ + private static LanceIndexJobMutationState expectedMutationState( + LanceIndexJobMutationType type, LanceIndexJobResultCode code) { + if (isPreInvocationSpec(code)) { + return LanceIndexJobMutationState.NOT_COMMITTED; + } + switch (code) { + case NATIVE_OK: + return LanceIndexJobMutationState.COMMITTED; + case NATIVE_COMMIT_CONFLICT: + return LanceIndexJobMutationState.NOT_COMMITTED; + case NATIVE_NOT_FOUND: + return type == LanceIndexJobMutationType.DROP + ? LanceIndexJobMutationState.NOT_COMMITTED : LanceIndexJobMutationState.UNKNOWN; + default: + return LanceIndexJobMutationState.UNKNOWN; + } + } + + /** + * Independent restatement of the design table, refresh-obligation column. + */ + private static LanceIndexJobRefreshState expectedRefreshState( + LanceIndexJobMutationType type, LanceIndexJobResultCode code, boolean externalMetadataAdvanced) { + if (isPreInvocationSpec(code)) { + return externalMetadataAdvanced + ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED; + } + switch (code) { + case NATIVE_OK: + case NATIVE_COMMIT_CONFLICT: + return LanceIndexJobRefreshState.REQUIRED; + case NATIVE_NOT_FOUND: + return type == LanceIndexJobMutationType.DROP + ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED; + default: + return LanceIndexJobRefreshState.NOT_REQUIRED; + } + } + + /** + * Independent restatement of the design table, completion-reason column. + */ + private static LanceIndexJobCompletionReason expectedCompletionReason( + LanceIndexJobMutationType type, LanceIndexJobResultCode code, boolean ifExists) { + return type == LanceIndexJobMutationType.DROP && code == LanceIndexJobResultCode.NATIVE_NOT_FOUND && ifExists + ? LanceIndexJobCompletionReason.IF_CONDITION_NOOP : LanceIndexJobCompletionReason.NONE; + } + + private static boolean isPreInvocationSpec(LanceIndexJobResultCode code) { + return code == LanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION + || code == LanceIndexJobResultCode.PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT + || code == LanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED + || code == LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java new file mode 100644 index 00000000000000..deb848b3048e5c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java @@ -0,0 +1,558 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.common.DdlException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Lifecycle state-machine coverage driven through the master write paths of + * {@link LanceIndexJobManager} with the edit-log seam captured in memory. Covers the + * legal mutation chain PENDING -> RUNNING -> (COMMITTED | NOT_COMMITTED | + * UNKNOWN), the impossibility of leaving UNKNOWN, the independent refresh chain + * REQUIRED -> RUNNING -> DONE / FAILED with FAILED -> RUNNING retry, the + * revision compare-and-set on every transition, and the fence/quota release timing + * (immediately on a terminal outcome with refresh NOT_REQUIRED, only at refresh DONE + * otherwise, never for UNKNOWN without FORCE). + */ +public class LanceIndexJobStateMachineTest { + private static final long CATALOG_ID = 10L; + private static final String LOCATOR = "s3://bucket/dataset"; + private static final long BACKEND_ID = 1001L; + private static final long BE_EPOCH = 55L; + private static final String INVOCATION_ID = "invocation-1"; + private static final long DEADLINE_MS = 9999L; + + @Test + public void createInitializesPendingRecord() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState()); + Assertions.assertEquals(0L, stored.getRevision()); + Assertions.assertFalse(stored.isPossibleLiveOwned()); + Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof()); + Assertions.assertTrue(stored.getCreateTimeMs() > 0L); + Assertions.assertTrue(manager.isFenceHeld(stored.fenceKey())); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + Assertions.assertEquals(1L, manager.getQuota().getCatalogCount(CATALOG_ID)); + Assertions.assertEquals(1L, manager.getQuota().getTableCount(stored.getTableQuotaKey())); + Assertions.assertEquals(1, manager.getUnresolvedJobs().size()); + Assertions.assertEquals(1, manager.editLog.size()); + } + + @Test + public void pendingToRunningToCommitted() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + + LanceIndexJob running = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, running.getMutationState()); + Assertions.assertEquals(1L, running.getRevision()); + Assertions.assertEquals(BACKEND_ID, running.getBackendId().longValue()); + Assertions.assertEquals(BE_EPOCH, running.getBeProcessEpoch().longValue()); + Assertions.assertEquals(INVOCATION_ID, running.getInvocationId()); + Assertions.assertEquals(DEADLINE_MS, running.getDeadlineMs().longValue()); + Assertions.assertTrue(running.holdsPossibleLiveSlot()); + + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + LanceIndexJob committed = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, committed.getRefreshState()); + Assertions.assertEquals(2L, committed.getRevision()); + Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_OK, committed.getResult().getResultCode()); + Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, committed.getResult().getCompletionReason()); + Assertions.assertTrue(manager.getJobsNeedingRefresh().contains(committed)); + } + + @Test + public void pendingToRunningToNotCommitted() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED))); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState()); + Assertions.assertEquals(2L, stored.getRevision()); + } + + @Test + public void pendingToRunningToUnknown() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NO_TRUSTED_RESULT))); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState()); + Assertions.assertEquals(2L, stored.getRevision()); + } + + @Test + public void dropIfExistsNotFoundCompletesWithIfConditionNoop() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newDropJob(1L, "IdxA", true), 100, 100, 100); + Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_NOT_FOUND))); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState()); + Assertions.assertEquals(LanceIndexJobCompletionReason.IF_CONDITION_NOOP, + stored.getResult().getCompletionReason()); + } + + @Test + public void markRunningRejectsSecondDispatch() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + + Assertions.assertFalse(manager.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState()); + Assertions.assertEquals(1L, manager.getJob(1L).getRevision()); + } + + @Test + public void markRunningRejectsWrongRevisionAndUnknownJob() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + Assertions.assertFalse(manager.markRunning(1L, 5L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(manager.markRunning(404L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState()); + Assertions.assertEquals(1, manager.editLog.size()); + } + + @Test + public void completeRejectsNonRunningJob() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + Assertions.assertFalse(manager.completeWithResult(1L, 0L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState()); + + createAndRun(manager, 2L, "IdxB"); + Assertions.assertTrue(manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertFalse(manager.completeWithResult(2L, 2L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, manager.getJob(2L).getMutationState()); + } + + @Test + public void unknownHasNoOutgoingTransitions() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NO_TRUSTED_RESULT))); + int loggedRecords = manager.editLog.size(); + + Assertions.assertFalse(manager.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertFalse(manager.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertFalse(manager.markRefreshRunning(1L, 2L)); + Assertions.assertFalse(manager.markRefreshDone(1L, 2L)); + Assertions.assertFalse(manager.markRefreshFailed(1L, 2L)); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState()); + Assertions.assertEquals(2L, stored.getRevision()); + Assertions.assertEquals(loggedRecords, manager.editLog.size()); + } + + @Test + public void refreshLifecycleReleasesFenceAndQuotaAtDone() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + + LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + + Assertions.assertTrue(manager.markRefreshRunning(1L, 2L)); + Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, manager.getJob(1L).getRefreshState()); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + + Assertions.assertTrue(manager.markRefreshDone(1L, 3L)); + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobRefreshState.DONE, stored.getRefreshState()); + Assertions.assertEquals(4L, stored.getRevision()); + Assertions.assertFalse(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); + Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty()); + Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty()); + } + + @Test + public void refreshFailureKeepsFenceAndRetriesThroughRunning() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); + + Assertions.assertTrue(manager.markRefreshRunning(1L, 2L)); + Assertions.assertTrue(manager.markRefreshFailed(1L, 3L)); + LanceIndexJob failed = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, failed.getRefreshState()); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + Assertions.assertTrue(manager.getUnresolvedJobs().contains(failed)); + + // FAILED -> RUNNING is the retry entry through the idempotent refresh path. + Assertions.assertTrue(manager.markRefreshRunning(1L, 4L)); + Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, manager.getJob(1L).getRefreshState()); + Assertions.assertTrue(manager.markRefreshDone(1L, 5L)); + Assertions.assertFalse(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); + } + + @Test + public void refreshTransitionsRejectedFromNotRequired() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED))); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(1L).getRefreshState()); + + Assertions.assertFalse(manager.markRefreshRunning(1L, 2L)); + Assertions.assertFalse(manager.markRefreshDone(1L, 2L)); + Assertions.assertFalse(manager.markRefreshFailed(1L, 2L)); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(1L).getRefreshState()); + Assertions.assertEquals(2L, manager.getJob(1L).getRevision()); + } + + @Test + public void refreshTransitionsRejectedFromDone() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertTrue(manager.markRefreshRunning(1L, 2L)); + Assertions.assertTrue(manager.markRefreshDone(1L, 3L)); + + Assertions.assertFalse(manager.markRefreshRunning(1L, 4L)); + Assertions.assertFalse(manager.markRefreshDone(1L, 4L)); + Assertions.assertFalse(manager.markRefreshFailed(1L, 4L)); + Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState()); + } + + @Test + public void refreshTransitionIsRevisionGuarded() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + + Assertions.assertFalse(manager.markRefreshRunning(1L, 99L)); + Assertions.assertFalse(manager.markRefreshRunning(404L, 2L)); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState()); + Assertions.assertEquals(2L, manager.getJob(1L).getRevision()); + } + + @Test + public void fenceAndQuotaReleasedImmediatelyWhenRefreshNotRequired() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION))); + + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, manager.getJob(1L).getMutationState()); + Assertions.assertFalse(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); + Assertions.assertEquals(0L, manager.getQuota().getCatalogCount(CATALOG_ID)); + Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty()); + } + + @Test + public void fenceAndQuotaSurviveUnknown() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); + + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_IO))); + + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, manager.getJob(1L).getMutationState()); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + Assertions.assertEquals(1, manager.getUnresolvedJobs().size()); + Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty()); + } + + @Test + public void terminationProofReleasesSlotOnly() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); + Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot()); + + Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, LanceIndexTerminationProof.NONE)); + Assertions.assertFalse(manager.recordTerminationProof(1L, 99L, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, LanceIndexTerminationProof.CHILD_REAPED)); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState()); + Assertions.assertEquals(LanceIndexTerminationProof.CHILD_REAPED, stored.getTerminationProof()); + Assertions.assertFalse(stored.holdsPossibleLiveSlot()); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + + // A slot may be proven exactly once. + Assertions.assertFalse(manager.recordTerminationProof(1L, 2L, LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE)); + } + + @Test + public void everyAcceptedTransitionWritesExactlyOneEditLogRecord() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS); + manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, result(LanceIndexJobResultCode.NATIVE_OK)); + manager.markRefreshRunning(1L, 2L); + manager.markRefreshDone(1L, 3L); + Assertions.assertEquals(5, manager.editLog.size()); + + // Rejected transitions never reach the journal. + Assertions.assertFalse(manager.markRefreshDone(1L, 4L)); + Assertions.assertFalse(manager.markRunning(1L, 4L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertEquals(5, manager.editLog.size()); + } + + @Test + public void markRunningRejectsBlankInvocationId() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + // A null/blank invocation identity would match a null field under Objects.equals + // in completeWithResult and silently defeat the stale-callback guard. + Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, null, DEADLINE_MS)); + Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, "", DEADLINE_MS)); + Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, " \t\n", DEADLINE_MS)); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState()); + Assertions.assertEquals(0L, stored.getRevision()); + Assertions.assertNull(stored.getInvocationId()); + Assertions.assertEquals(1, manager.editLog.size()); + + // A well-formed dispatch is still accepted afterwards. + Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState()); + } + + @Test + public void failedRefreshJobStaysVisibleToTheRefreshDriver() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertTrue(manager.markRefreshRunning(1L, 2L)); + Assertions.assertTrue(manager.markRefreshFailed(1L, 3L)); + + // FAILED still owes the idempotent retry: the driver must see the job. + Assertions.assertTrue(manager.getJobsNeedingRefresh().contains(manager.getJob(1L))); + + Assertions.assertTrue(manager.markRefreshRunning(1L, 4L)); + Assertions.assertTrue(manager.markRefreshDone(1L, 5L)); + Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty()); + + // Terminal jobs with refresh DONE or NOT_REQUIRED never show up. + createAndRun(manager, 2L, "IdxB"); + Assertions.assertTrue(manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED))); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(2L).getRefreshState()); + Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty()); + } + + @Test + public void createJobResetsLifecycleFieldsAndPublishesAPrivateCopy() throws DdlException { + LanceIndexJob dirty = newCreateJob(1L, "IdxA"); + dirty.setRevision(9L); + dirty.setMutationState(LanceIndexJobMutationState.RUNNING); + dirty.setRefreshState(LanceIndexJobRefreshState.RUNNING); + dirty.setResult(result(LanceIndexJobResultCode.NATIVE_OK)); + dirty.setBackendId(BACKEND_ID); + dirty.setBeProcessEpoch(BE_EPOCH); + dirty.setInvocationId(INVOCATION_ID); + dirty.setDeadlineMs(DEADLINE_MS); + dirty.setPossibleLiveOwned(true); + dirty.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED); + dirty.setForceReleased(true); + dirty.setForceActor("admin"); + dirty.setForceTimeMs(7L); + dirty.setForceNote("note"); + dirty.setForceWarning("warning"); + + TestManager manager = new TestManager(); + manager.createJob(dirty, 100, 100, 100); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertNotSame(dirty, stored); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState()); + Assertions.assertEquals(0L, stored.getRevision()); + Assertions.assertNull(stored.getResult()); + Assertions.assertNull(stored.getBackendId()); + Assertions.assertNull(stored.getBeProcessEpoch()); + Assertions.assertNull(stored.getInvocationId()); + Assertions.assertNull(stored.getDeadlineMs()); + Assertions.assertFalse(stored.isPossibleLiveOwned()); + Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof()); + Assertions.assertFalse(stored.isForceReleased()); + Assertions.assertNull(stored.getForceActor()); + Assertions.assertNull(stored.getForceTimeMs()); + Assertions.assertNull(stored.getForceNote()); + Assertions.assertNull(stored.getForceWarning()); + + // Mutating the caller's object after admission touches nothing inside the manager. + dirty.setMutationState(LanceIndexJobMutationState.UNKNOWN); + dirty.setNormalizedIndexName("idxb"); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState()); + Assertions.assertTrue(manager.isFenceHeld(newCreateJob(2L, "idxa").fenceKey())); + Assertions.assertFalse(manager.isFenceHeld(newCreateJob(3L, "IdxB").fenceKey())); + } + + @Test + public void refreshDoneOrFailedRequiresAPrecedingRefreshRunning() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState()); + + // No direct REQUIRED -> DONE / FAILED: a refresh must actually run first. + Assertions.assertFalse(manager.markRefreshDone(1L, 2L)); + Assertions.assertFalse(manager.markRefreshFailed(1L, 2L)); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState()); + Assertions.assertEquals(2L, manager.getJob(1L).getRevision()); + Assertions.assertEquals(3, manager.editLog.size()); + } + + @Test + public void terminationProofNeedsASlotAndStillLandsAfterTheTerminalResult() throws DdlException { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + + // PENDING owns no possible-live slot: a proof has nothing to release. + Assertions.assertFalse(manager.recordTerminationProof(1L, 0L, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertEquals(0L, manager.getJob(1L).getRevision()); + Assertions.assertEquals(1, manager.editLog.size()); + + Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + // The terminal outcome does not release the slot; the proof still lands afterwards. + LanceIndexJob committed = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState()); + Assertions.assertTrue(committed.holdsPossibleLiveSlot()); + LanceIndexFenceKey fenceKey = committed.fenceKey(); + + Assertions.assertTrue(manager.recordTerminationProof(1L, 2L, LanceIndexTerminationProof.CHILD_REAPED)); + LanceIndexJob proven = manager.getJob(1L); + Assertions.assertFalse(proven.holdsPossibleLiveSlot()); + // Fence and quota still follow the refresh rule, not the proof. + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + + Assertions.assertTrue(manager.markRefreshRunning(1L, 3L)); + Assertions.assertTrue(manager.markRefreshDone(1L, 4L)); + Assertions.assertFalse(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); + } + + @Test + public void notCommittedWithRefreshRequiredReleasesFenceAtRefreshDone() throws DdlException { + TestManager manager = new TestManager(); + createAndRun(manager, 1L, "IdxA"); + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT, + LanceIndexJobCompletionReason.NONE, "commit conflict", false))); + + LanceIndexJob stored = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState()); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState()); + LanceIndexFenceKey fenceKey = stored.fenceKey(); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + Assertions.assertTrue(manager.getJobsNeedingRefresh().contains(stored)); + + Assertions.assertTrue(manager.markRefreshRunning(1L, 2L)); + Assertions.assertTrue(manager.markRefreshDone(1L, 3L)); + Assertions.assertFalse(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); + Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty()); + } + + private static LanceIndexJob newCreateJob(long jobId, String displayName) { + return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, + displayName, LanceIndexNameNormalizer.normalize(displayName), + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", + null, 7L, null); + } + + private static LanceIndexJob newDropJob(long jobId, String displayName, boolean ifExists) { + return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, + displayName, LanceIndexNameNormalizer.normalize(displayName), + LanceIndexJobMutationType.DROP, false, ifExists, null, "v", + null, 7L, null); + } + + private static LanceIndexJobResult result(LanceIndexJobResultCode code) { + return new LanceIndexJobResult(code, LanceIndexJobCompletionReason.NONE, "sanitized message", false); + } + + private static LanceIndexJob createAndRun(TestManager manager, long jobId, String displayName) throws DdlException { + manager.createJob(newCreateJob(jobId, displayName), 100, 100, 100); + Assertions.assertTrue(manager.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + return manager.getJob(jobId); + } + + /** + * Edit-log seam: captures every durable record instead of writing the journal. + */ + private static class TestManager extends LanceIndexJobManager { + private final List editLog = new ArrayList<>(); + + @Override + protected void writeEditLog(LanceIndexJob job) { + editLog.add(job); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java new file mode 100644 index 00000000000000..dd5bd9b10dea55 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.persist.gson.GsonUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Record-level coverage for {@link LanceIndexJob}: the derived fence and quota keys, + * the full {@link LanceIndexJob#isUnresolved()} matrix that keeps fence and quota + * alive together, the possible-live slot semantics (released only by a matching + * termination proof or a durable FORCE_RELEASE, never by a deadline), corrupt-record + * fallbacks toward the safe direction, and the deep-copy constructor. + */ +public class LanceIndexJobTest { + private static final long CATALOG_ID = 10L; + private static final String LOCATOR = "s3://bucket/dataset"; + private static final long BACKEND_ID = 1001L; + private static final long BE_EPOCH = 55L; + private static final String INVOCATION_ID = "invocation-1"; + + @Test + public void fenceKeyCarriesIdentityAndHidesLocator() { + LanceIndexJob job = newCreateJob(1L, "IdxA"); + LanceIndexFenceKey key = job.fenceKey(); + + Assertions.assertEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY, + LOCATOR, "idxa"), key); + Assertions.assertEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY, + LOCATOR, "idxa").hashCode(), key.hashCode()); + Assertions.assertNotEquals(new LanceIndexFenceKey(20L, LanceIndexFenceKey.PROVIDER_DIRECTORY, + LOCATOR, "idxa"), key); + Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, "REST", LOCATOR, "idxa"), key); + Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY, + "s3://bucket/other", "idxa"), key); + Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY, + LOCATOR, "idxb"), key); + + // Fence-conflict messages may surface to users without target privileges. + Assertions.assertFalse(key.toString().contains(LOCATOR)); + Assertions.assertTrue(key.toString().contains("idxa")); + } + + @Test + public void fenceKeyIsStableAcrossJobsWithTheSameIdentity() { + LanceIndexJob first = newCreateJob(1L, "IdxA"); + LanceIndexJob second = newCreateJob(2L, "idxa"); + // Display case differs, normalized identity does not. + Assertions.assertEquals(first.fenceKey(), second.fenceKey()); + Assertions.assertEquals(first.fenceKey().hashCode(), second.fenceKey().hashCode()); + } + + @Test + public void tableQuotaKeyDerivesFromCatalogAndLocator() { + LanceIndexJob job = newCreateJob(1L, "IdxA"); + Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR), job.getTableQuotaKey()); + } + + @Test + public void activeStatesAreAlwaysUnresolved() { + for (LanceIndexJobRefreshState refresh : LanceIndexJobRefreshState.values()) { + Assertions.assertTrue(jobInState(LanceIndexJobMutationState.PENDING, refresh, false).isUnresolved()); + Assertions.assertTrue(jobInState(LanceIndexJobMutationState.RUNNING, refresh, false).isUnresolved()); + } + } + + @Test + public void knownTerminalStatesFollowTheRefreshState() { + for (LanceIndexJobMutationState terminal : new LanceIndexJobMutationState[]{ + LanceIndexJobMutationState.COMMITTED, LanceIndexJobMutationState.NOT_COMMITTED}) { + Assertions.assertFalse(jobInState(terminal, LanceIndexJobRefreshState.NOT_REQUIRED, false).isUnresolved()); + Assertions.assertFalse(jobInState(terminal, LanceIndexJobRefreshState.DONE, false).isUnresolved()); + Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.REQUIRED, false).isUnresolved()); + Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.RUNNING, false).isUnresolved()); + // A failed refresh still holds the fence: the retry goes through the idempotent path. + Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.FAILED, false).isUnresolved()); + } + } + + @Test + public void unknownIsUnresolvedUntilForceReleased() { + for (LanceIndexJobRefreshState refresh : LanceIndexJobRefreshState.values()) { + Assertions.assertTrue(jobInState(LanceIndexJobMutationState.UNKNOWN, refresh, false).isUnresolved()); + Assertions.assertFalse(jobInState(LanceIndexJobMutationState.UNKNOWN, refresh, true).isUnresolved()); + } + } + + @Test + public void nullStatesFallBackToTheSafeDirection() { + // A corrupt record missing its states must keep the fence (treated as UNKNOWN) + // and must never become redispatchable PENDING. + LanceIndexJob job = GsonUtils.GSON.fromJson( + "{\"jid\":1,\"ms\":null,\"rs\":null}", LanceIndexJob.class); + Assertions.assertNull(job.getMutationState()); + Assertions.assertNull(job.getRefreshState()); + Assertions.assertTrue(job.isUnresolved()); + + LanceIndexJob forced = GsonUtils.GSON.fromJson( + "{\"jid\":1,\"ms\":null,\"rs\":null,\"fr\":true}", LanceIndexJob.class); + Assertions.assertFalse(forced.isUnresolved()); + + // A terminal record missing its refresh state owes one (treated as REQUIRED). + LanceIndexJob committedNoRefresh = GsonUtils.GSON.fromJson( + "{\"jid\":1,\"ms\":\"COMMITTED\",\"rs\":null}", LanceIndexJob.class); + Assertions.assertTrue(committedNoRefresh.isUnresolved()); + } + + @Test + public void missingRefreshStateKeyFallsBackToRequired() { + // A corrupt terminal record without the "rs" key at all keeps the fence: the + // field initial value is REQUIRED, the same safe direction as the null fallback. + LanceIndexJob missingKey = GsonUtils.GSON.fromJson( + "{\"jid\":1,\"ms\":\"COMMITTED\"}", LanceIndexJob.class); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, missingKey.getRefreshState()); + Assertions.assertTrue(missingKey.isUnresolved()); + + // A legal terminal record always carries the key explicitly. + LanceIndexJob legal = GsonUtils.GSON.fromJson( + "{\"jid\":1,\"ms\":\"COMMITTED\",\"rs\":\"NOT_REQUIRED\"}", LanceIndexJob.class); + Assertions.assertFalse(legal.isUnresolved()); + } + + @Test + public void possibleLiveSlotMatrix() { + LanceIndexJob job = new LanceIndexJob(); + Assertions.assertFalse(job.holdsPossibleLiveSlot()); + + job.setPossibleLiveOwned(true); + Assertions.assertTrue(job.holdsPossibleLiveSlot()); + + job.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED); + Assertions.assertFalse(job.holdsPossibleLiveSlot()); + + job.setTerminationProof(LanceIndexTerminationProof.NONE); + job.setForceReleased(true); + Assertions.assertFalse(job.holdsPossibleLiveSlot()); + + // A corrupt record missing the proof is treated as NONE (slot still owned). + LanceIndexJob nullProof = GsonUtils.GSON.fromJson( + "{\"jid\":1,\"plo\":true,\"tp\":null}", LanceIndexJob.class); + Assertions.assertNull(nullProof.getTerminationProof()); + Assertions.assertTrue(nullProof.holdsPossibleLiveSlot()); + } + + @Test + public void terminationProofClearsSlotButKeepsFenceAndOutcome() throws Exception { + TestManager manager = new TestManager(); + manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); + manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L); + LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); + + Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE)); + LanceIndexJob proven = manager.getJob(1L); + Assertions.assertFalse(proven.holdsPossibleLiveSlot()); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, proven.getMutationState()); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); + + // The ambiguous result still lands afterwards: UNKNOWN keeps the fence, and the + // already-recorded proof keeps the slot released. + Assertions.assertTrue(manager.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, + new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT, + LanceIndexJobCompletionReason.NONE, "ambiguous", false))); + LanceIndexJob unknown = manager.getJob(1L); + Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, unknown.getMutationState()); + Assertions.assertFalse(unknown.holdsPossibleLiveSlot()); + Assertions.assertTrue(unknown.isUnresolved()); + Assertions.assertTrue(manager.isFenceHeld(fenceKey)); + } + + @Test + public void copyConstructorDuplicatesEveryFieldIndependently() { + LanceIndexJob original = newCreateJob(1L, "IdxA"); + original.setRevision(3L); + original.setMutationState(LanceIndexJobMutationState.RUNNING); + original.setRefreshState(LanceIndexJobRefreshState.RUNNING); + original.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, + LanceIndexJobCompletionReason.NONE, "ok", false)); + original.setBackendId(BACKEND_ID); + original.setBeProcessEpoch(BE_EPOCH); + original.setInvocationId(INVOCATION_ID); + original.setDeadlineMs(123L); + original.setPossibleLiveOwned(true); + original.setForceActor("admin"); + original.setForceTimeMs(9L); + original.setForceNote("note"); + original.setForceWarning("warning"); + + LanceIndexJob copy = new LanceIndexJob(original); + Assertions.assertEquals(GsonUtils.GSON.toJson(original), GsonUtils.GSON.toJson(copy)); + + copy.setRevision(99L); + copy.setMutationState(LanceIndexJobMutationState.UNKNOWN); + copy.setPossibleLiveOwned(false); + Assertions.assertEquals(3L, original.getRevision()); + Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, original.getMutationState()); + Assertions.assertTrue(original.isPossibleLiveOwned()); + } + + @Test + public void admissionConstructorRejectsNullIdentity() { + Assertions.assertThrows(NullPointerException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + null, LOCATOR, "IdxA", "idxa", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(NullPointerException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, null, "IdxA", "idxa", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(NullPointerException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", "idxa", + null, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, null, "idxa", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", null, + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + } + + @Test + public void toStringNamesTheIndexButHidesTheLocator() { + LanceIndexJob job = newCreateJob(1L, "IdxA"); + Assertions.assertTrue(job.toString().contains("IdxA")); + Assertions.assertFalse(job.toString().contains("bucket")); + Assertions.assertFalse(job.toString().contains(LOCATOR)); + } + + private static LanceIndexJob newCreateJob(long jobId, String displayName) { + return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, + displayName, LanceIndexNameNormalizer.normalize(displayName), + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", + null, 7L, null); + } + + private static LanceIndexJob jobInState(LanceIndexJobMutationState mutationState, + LanceIndexJobRefreshState refreshState, boolean forceReleased) { + LanceIndexJob job = new LanceIndexJob(); + job.setMutationState(mutationState); + job.setRefreshState(refreshState); + job.setForceReleased(forceReleased); + return job; + } + + /** + * Edit-log seam: captures every durable record instead of writing the journal. + */ + private static class TestManager extends LanceIndexJobManager { + private final List editLog = new ArrayList<>(); + + @Override + protected void writeEditLog(LanceIndexJob job) { + editLog.add(job); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java new file mode 100644 index 00000000000000..d2ad765697cf52 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Locale; + +/** + * Unit coverage for index-name normalization v1: the fence-key identity is exactly + * {@code toLowerCase(Locale.ROOT)}, so the tests pin the locale-sensitive corners + * (the Turkish dotted-I family) in addition to plain ASCII case folding and the + * persisted-name byte bound. + */ +public class LanceIndexNameNormalizerTest { + + @Test + public void normalizeLowercasesAscii() { + Assertions.assertEquals("my_index", LanceIndexNameNormalizer.normalize("My_Index")); + Assertions.assertEquals("idx", LanceIndexNameNormalizer.normalize("IDX")); + Assertions.assertEquals("idx", LanceIndexNameNormalizer.normalize("idx")); + Assertions.assertEquals("", LanceIndexNameNormalizer.normalize("")); + } + + @Test + public void normalizeKeepsDigitsAndUnderscores() { + Assertions.assertEquals("idx_2024_v2", LanceIndexNameNormalizer.normalize("Idx_2024_V2")); + } + + @Test + public void normalizeUsesRootLocaleForConditionalMappings() { + // İ (capital I with dot above) folds to i + combining dot above under the ROOT/default + // mapping, never to the Turkish locale's dotless ı. This pins normalization v1 as + // environment-independent. + String dottedCapitalI = "İ"; + String normalized = LanceIndexNameNormalizer.normalize(dottedCapitalI); + Assertions.assertEquals("i̇", normalized); + Assertions.assertEquals(dottedCapitalI.toLowerCase(Locale.ROOT), normalized); + Assertions.assertNotEquals(dottedCapitalI.toLowerCase(new Locale("tr")), normalized); + } + + @Test + public void normalizeHandlesUnicodeLettersAndLeavesCjkUntouched() { + Assertions.assertEquals("äöü", LanceIndexNameNormalizer.normalize("ÄÖÜ")); + Assertions.assertEquals("索引", LanceIndexNameNormalizer.normalize("索引")); + Assertions.assertEquals("ß", LanceIndexNameNormalizer.normalize("ß")); + } + + @Test + public void normalizeRejectsNull() { + Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexNameNormalizer.normalize(null)); + } + + @Test + public void isCaseOnlyDuplicateTrueOnlyForPureCaseDifference() { + Assertions.assertTrue(LanceIndexNameNormalizer.isCaseOnlyDuplicate("MyIdx", "myidx")); + Assertions.assertTrue(LanceIndexNameNormalizer.isCaseOnlyDuplicate("MYIDX", "myidx")); + Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate("myidx", "myidx")); + Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate("idxA", "idxB")); + Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate(null, "idx")); + Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate("idx", null)); + } + + @Test + public void validateDisplayNameAcceptsBoundarySizes() { + LanceIndexNameNormalizer.validateDisplayName("i"); + LanceIndexNameNormalizer.validateDisplayName( + StringUtils.repeat("a", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES)); + // 512 two-byte characters are exactly at the byte bound. + String exactMultibyte = StringUtils.repeat("é", 512); + Assertions.assertEquals(1024, exactMultibyte.getBytes(StandardCharsets.UTF_8).length); + LanceIndexNameNormalizer.validateDisplayName(exactMultibyte); + } + + @Test + public void validateDisplayNameRejectsNullEmptyAndOversize() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexNameNormalizer.validateDisplayName(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexNameNormalizer.validateDisplayName("")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexNameNormalizer.validateDisplayName( + StringUtils.repeat("a", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1))); + // 513 two-byte characters exceed the byte bound even though the char count is small. + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexNameNormalizer.validateDisplayName(StringUtils.repeat("é", 513))); + } + + @Test + public void boundIsPinnedAt1024Utf8Bytes() { + Assertions.assertEquals(1024, LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES); + } +} From 36ed4ef5f8d0e93f0ee1cb293116597278499ef0 Mon Sep 17 00:00:00 2001 From: kid Date: Fri, 28 Aug 2026 23:39:56 +0800 Subject: [PATCH 4/4] [fix](lance) Harden durable index job invariants --- .../lance/job/LanceIndexDatasetLocator.java | 84 ++++++--- .../datasource/lance/job/LanceIndexJob.java | 101 ++++++++-- .../lance/job/LanceIndexJobManager.java | 175 ++++++++++++------ .../lance/job/LanceIndexJobQuota.java | 16 +- .../lance/job/LanceIndexSchemaContract.java | 64 ++++++- .../job/LanceIndexDatasetLocatorTest.java | 55 ++++++ .../job/LanceIndexJobManagerPersistTest.java | 73 ++++++++ .../job/LanceIndexJobManagerReplayTest.java | 73 +++++++- .../lance/job/LanceIndexJobQuotaTest.java | 45 +++-- .../job/LanceIndexJobStateMachineTest.java | 102 +++++++++- .../lance/job/LanceIndexJobTest.java | 27 ++- .../lance/job/LanceIndexJobWiringTest.java | 78 ++++++++ 12 files changed, 752 insertions(+), 141 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java index bed9a2dff04970..bed1ae696d8aa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java @@ -17,6 +17,9 @@ package org.apache.doris.datasource.lance.job; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.Locale; /** @@ -46,6 +49,8 @@ */ public final class LanceIndexDatasetLocator { private static final String SCHEME_SEPARATOR = "://"; + /** Finite bound for the stable locator persisted in every durable job record. */ + public static final int MAX_LOCATOR_BYTES = 4096; private LanceIndexDatasetLocator() { } @@ -53,9 +58,10 @@ private LanceIndexDatasetLocator() { /** * Normalize a raw dataset locator into its durable identity form. * - * @throws IllegalArgumentException if the locator is null/empty, carries - * userinfo, has an empty scheme, has neither an authority nor a - * path, or is a scheme-less relative path + * @throws IllegalArgumentException if the locator is null/empty/oversized, + * is not a valid hierarchical URI or absolute path, carries + * userinfo/query/fragment data, has neither an authority nor a path, + * or is a scheme-less relative path */ public static String normalize(String rawLocator) { if (rawLocator == null) { @@ -65,36 +71,68 @@ public static String normalize(String rawLocator) { if (locator.isEmpty()) { throw new IllegalArgumentException("dataset locator must not be empty"); } - int separator = locator.indexOf(SCHEME_SEPARATOR); - if (separator < 0) { - if (!locator.startsWith("/")) { - throw new IllegalArgumentException( - "dataset locator without a scheme must be an absolute path: " + abbreviate(locator)); - } - return stripTrailingSlashes(locator, 1); + if (locator.getBytes(StandardCharsets.UTF_8).length > MAX_LOCATOR_BYTES) { + throw new IllegalArgumentException( + "dataset locator exceeds " + MAX_LOCATOR_BYTES + " UTF-8 bytes"); } - String scheme = locator.substring(0, separator); - if (scheme.isEmpty()) { - throw new IllegalArgumentException("dataset locator has an empty scheme: " + abbreviate(locator)); + + // Preserve the established spelling of the scheme-less filesystem root. + // java.net.URI rejects "//" as a network-path reference without an + // authority, while it is a valid absolute filesystem path here. + if (containsOnlySlashes(locator)) { + return "/"; } - String rest = locator.substring(separator + SCHEME_SEPARATOR.length()); - int pathStart = rest.indexOf('/'); - String authority = pathStart < 0 ? rest : rest.substring(0, pathStart); - if (authority.contains("@")) { - // Never persist or key on a credential-bearing URL. + + URI uri; + try { + uri = new URI(locator); + } catch (URISyntaxException e) { + // Do not include the raw locator: it may contain credentials. + throw new IllegalArgumentException("dataset locator is not a valid URI or absolute path"); + } + if (uri.getRawUserInfo() != null) { throw new IllegalArgumentException( "credential-bearing dataset locators are never identity (userinfo is not allowed)"); } - String path = pathStart < 0 ? "" : stripTrailingSlashes(rest.substring(pathStart), 0); + if (uri.getRawQuery() != null) { + // Presigned object-store URLs and SAS URLs carry credentials here. + throw new IllegalArgumentException("dataset locator query parameters are not allowed"); + } + if (uri.getRawFragment() != null) { + throw new IllegalArgumentException("dataset locator fragments are not allowed"); + } + + String scheme = uri.getScheme(); + if (scheme == null) { + if (!locator.startsWith("/")) { + throw new IllegalArgumentException("dataset locator without a scheme must be an absolute path"); + } + return stripTrailingSlashes(locator, 1); + } + if (uri.isOpaque() || !locator.regionMatches(scheme.length(), SCHEME_SEPARATOR, 0, + SCHEME_SEPARATOR.length())) { + throw new IllegalArgumentException("dataset locator scheme must use hierarchical '://' syntax"); + } + String authority = uri.getRawAuthority() == null ? "" : uri.getRawAuthority(); + String rawPath = uri.getRawPath() == null ? "" : uri.getRawPath(); + String path = stripTrailingSlashes(rawPath, 0); if (authority.isEmpty() && path.isEmpty()) { // "s3://" / "file://" carry no identity at all; "file:///x" (empty // authority, non-empty path) is legal and does not reach this. - throw new IllegalArgumentException( - "dataset locator has neither an authority nor a path: " + abbreviate(locator)); + throw new IllegalArgumentException("dataset locator has neither an authority nor a path"); } return scheme.toLowerCase(Locale.ROOT) + SCHEME_SEPARATOR + authority + path; } + private static boolean containsOnlySlashes(String value) { + for (int index = 0; index < value.length(); index++) { + if (value.charAt(index) != '/') { + return false; + } + } + return true; + } + private static String stripTrailingSlashes(String value, int minLength) { int end = value.length(); while (end > minLength && value.charAt(end - 1) == '/') { @@ -102,8 +140,4 @@ private static String stripTrailingSlashes(String value, int minLength) { } return value.substring(0, end); } - - private static String abbreviate(String locator) { - return locator.length() <= 64 ? locator : locator.substring(0, 64) + "..."; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java index 52bb24ee2bc4e4..59386acbcdb94c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java @@ -59,6 +59,10 @@ public class LanceIndexJob implements Writable { public static final int MAX_PROPERTIES_JSON_BYTES = 4096; /** Bound on the FORCE_RELEASE operator note and the late-commit warning text. */ public static final int MAX_FORCE_TEXT_BYTES = 1024; + /** Bound on persisted creator, target-name, mutation, and FORCE-actor text. */ + public static final int MAX_DURABLE_TEXT_BYTES = 1024; + /** Dispatch identities are UUID-like tokens, not arbitrary worker output. */ + public static final int MAX_INVOCATION_ID_BYTES = 256; // ------------------------------------------------------------------ // Identity @@ -166,6 +170,14 @@ public class LanceIndexJob implements Writable { @SerializedName(value = "bpe") private Long beProcessEpoch; + /** + * Immutable revision established by PENDING -> RUNNING. Result and + * termination-proof callbacks use this dispatch identity rather than racing + * on the record's global revision. Null only before dispatch or in old data. + */ + @SerializedName(value = "drv") + private Long dispatchRevision; + /** Immutable per-dispatch UUID; callbacks must present the matching identity. */ @SerializedName(value = "iid") private String invocationId; @@ -217,10 +229,10 @@ public LanceIndexJob(long jobId, String creator, long catalogId, String dbName, String columnName, String propertiesJson, long admittedDatasetVersion, LanceIndexSchemaContract schemaContract) { this.jobId = jobId; - this.creator = creator; + this.creator = checkRequiredBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator"); this.catalogId = catalogId; - this.dbName = dbName; - this.tableName = tableName; + this.dbName = checkRequiredBytes(dbName, MAX_DURABLE_TEXT_BYTES, "dbName"); + this.tableName = checkRequiredBytes(tableName, MAX_DURABLE_TEXT_BYTES, "tableName"); this.provider = Objects.requireNonNull(provider, "provider"); this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator"); setDisplayIndexName(displayIndexName); @@ -228,11 +240,12 @@ public LanceIndexJob(long jobId, String creator, long catalogId, String dbName, this.mutationType = Objects.requireNonNull(mutationType, "mutationType"); this.ifNotExists = ifNotExists; this.ifExists = ifExists; - this.indexType = indexType; - this.columnName = columnName; + setIndexType(indexType); + setColumnName(columnName); setPropertiesJson(propertiesJson); this.admittedDatasetVersion = admittedDatasetVersion; this.schemaContract = schemaContract; + validateForAdmission(); } /** @@ -269,6 +282,7 @@ public LanceIndexJob(LanceIndexJob other) { this.result = other.result; this.backendId = other.backendId; this.beProcessEpoch = other.beProcessEpoch; + this.dispatchRevision = other.dispatchRevision; this.invocationId = other.invocationId; this.deadlineMs = other.deadlineMs; this.possibleLiveOwned = other.possibleLiveOwned; @@ -336,6 +350,52 @@ public boolean holdsPossibleLiveSlot() { && !forceReleased; } + /** + * Validate the invariant-bearing and bounded fields before this record is + * admitted. This is intentionally separate from Gson replay, which must + * remain tolerant of old or corrupt records in the safe direction. + */ + public void validateForAdmission() { + checkRequiredBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator"); + checkRequiredBytes(dbName, MAX_DURABLE_TEXT_BYTES, "dbName"); + checkRequiredBytes(tableName, MAX_DURABLE_TEXT_BYTES, "tableName"); + if (provider == null) { + throw new IllegalArgumentException("lance index job provider must not be null"); + } + if (!LanceIndexFenceKey.PROVIDER_DIRECTORY.equals(provider)) { + throw new IllegalArgumentException("lance index job provider must be DIRECTORY"); + } + if (normalizedLocator == null) { + throw new IllegalArgumentException("normalized dataset locator must not be null"); + } + String canonicalLocator = LanceIndexDatasetLocator.normalize(normalizedLocator); + if (!canonicalLocator.equals(normalizedLocator)) { + throw new IllegalArgumentException("dataset locator is not in canonical identity form"); + } + LanceIndexNameNormalizer.validateDisplayName(displayIndexName); + validateNormalizedIndexName(normalizedIndexName); + String expectedNormalizedName = LanceIndexNameNormalizer.normalize(displayIndexName); + if (!expectedNormalizedName.equals(normalizedIndexName)) { + throw new IllegalArgumentException("normalized index name does not match the display name"); + } + if (mutationType == null) { + throw new IllegalArgumentException("mutation type must not be null"); + } + checkBytes(indexType, MAX_DURABLE_TEXT_BYTES, "indexType"); + checkBytes(columnName, MAX_DURABLE_TEXT_BYTES, "columnName"); + checkBytes(propertiesJson, MAX_PROPERTIES_JSON_BYTES, "propertiesJson"); + checkBytes(invocationId, MAX_INVOCATION_ID_BYTES, "invocationId"); + checkBytes(forceActor, MAX_DURABLE_TEXT_BYTES, "forceActor"); + checkBytes(forceNote, MAX_FORCE_TEXT_BYTES, "forceNote"); + checkBytes(forceWarning, MAX_FORCE_TEXT_BYTES, "forceWarning"); + if (result != null) { + checkBytes(result.getSanitizedMessage(), LanceIndexJobResult.MAX_MESSAGE_BYTES, "sanitizedMessage"); + } + if (schemaContract != null) { + schemaContract.validateForAdmission(); + } + } + // ------------------------------------------------------------------ // Accessors. Setters for bounded text fields re-validate the bound. // ------------------------------------------------------------------ @@ -353,7 +413,7 @@ public String getCreator() { } public void setCreator(String creator) { - this.creator = creator; + this.creator = checkBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator"); } public long getRevision() { @@ -414,6 +474,11 @@ public String getNormalizedIndexName() { } public final void setNormalizedIndexName(String normalizedIndexName) { + validateNormalizedIndexName(normalizedIndexName); + this.normalizedIndexName = normalizedIndexName; + } + + private static void validateNormalizedIndexName(String normalizedIndexName) { if (normalizedIndexName == null || normalizedIndexName.isEmpty()) { throw new IllegalArgumentException("normalized index name must not be null or empty"); } @@ -422,7 +487,6 @@ public final void setNormalizedIndexName(String normalizedIndexName) { throw new IllegalArgumentException( "normalized index name exceeds " + LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + " UTF-8 bytes"); } - this.normalizedIndexName = normalizedIndexName; } public LanceIndexJobMutationType getMutationType() { @@ -442,7 +506,7 @@ public String getIndexType() { } public void setIndexType(String indexType) { - this.indexType = indexType; + this.indexType = checkBytes(indexType, MAX_DURABLE_TEXT_BYTES, "indexType"); } public String getColumnName() { @@ -450,7 +514,7 @@ public String getColumnName() { } public void setColumnName(String columnName) { - this.columnName = columnName; + this.columnName = checkBytes(columnName, MAX_DURABLE_TEXT_BYTES, "columnName"); } public String getPropertiesJson() { @@ -509,12 +573,20 @@ public void setBeProcessEpoch(Long beProcessEpoch) { this.beProcessEpoch = beProcessEpoch; } + public Long getDispatchRevision() { + return dispatchRevision; + } + + public void setDispatchRevision(Long dispatchRevision) { + this.dispatchRevision = dispatchRevision; + } + public String getInvocationId() { return invocationId; } public void setInvocationId(String invocationId) { - this.invocationId = invocationId; + this.invocationId = checkBytes(invocationId, MAX_INVOCATION_ID_BYTES, "invocationId"); } public Long getDeadlineMs() { @@ -554,7 +626,7 @@ public String getForceActor() { } public void setForceActor(String forceActor) { - this.forceActor = forceActor; + this.forceActor = checkBytes(forceActor, MAX_DURABLE_TEXT_BYTES, "forceActor"); } public Long getForceTimeMs() { @@ -588,6 +660,13 @@ private static String checkBytes(String value, int maxBytes, String fieldName) { return value; } + private static String checkRequiredBytes(String value, int maxBytes, String fieldName) { + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException(fieldName + " must not be null or empty"); + } + return checkBytes(value, maxBytes, fieldName); + } + // ------------------------------------------------------------------ // Serialization (Gson stream style, see DropIndexPolicyLog) // ------------------------------------------------------------------ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java index 4eae18093a492d..8ece27bb62f338 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java @@ -37,7 +37,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.NavigableSet; import java.util.Objects; +import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -83,11 +85,12 @@ public class LanceIndexJobManager implements Writable, GsonPostProcessable { private ConcurrentMap jobs = Maps.newConcurrentMap(); /** - * Derived: fence key -> jobId, holds the unresolved jobs that carry fence - * identity (identity-less corrupt records are kept out of the books). Rebuilt - * after replay/image load. + * Derived: fence key -> all job ids holding that fence. Legal admission creates + * exactly one owner; retaining every owner for a corrupt collision keeps the + * fence fail-closed when one of those jobs later settles. Identity-less corrupt + * records are kept out of the books. Rebuilt after replay/image load. */ - private final Map fenceIndex = Maps.newHashMap(); + private final Map> fenceIndex = Maps.newHashMap(); /** Derived: three-level unresolved counters, rebuilt from the unresolved jobs. */ private final LanceIndexJobQuota quota = new LanceIndexJobQuota(); @@ -137,14 +140,22 @@ public void createJob(LanceIndexJob job, long tableLimit, long catalogLimit, lon Objects.requireNonNull(job, "job"); writeLock(); try { + try { + job.validateForAdmission(); + } catch (IllegalArgumentException e) { + throw new DdlException("invalid lance index job: " + e.getMessage(), e); + } + if (tableLimit <= 0 || catalogLimit <= 0 || globalLimit <= 0) { + throw new DdlException("lance index job quota limits must all be positive"); + } if (jobs.containsKey(job.getJobId())) { throw new DdlException("lance index job id already exists: " + job.getJobId()); } - Long fencingJobId = fenceIndex.get(job.fenceKey()); - if (fencingJobId != null) { + NavigableSet fencingJobIds = fenceIndex.get(job.fenceKey()); + if (fencingJobIds != null && !fencingJobIds.isEmpty()) { // Never disclose the locator in the rejection (the caller may lack target privilege). throw new DdlException("lance index '" + job.getDisplayIndexName() - + "' is fenced by unresolved job " + fencingJobId + + "' is fenced by unresolved job " + fencingJobIds.first() + "; resolve that job (FORCE_RELEASE) before reusing the name"); } // Pure admission check; the charge itself happens in applyToMemory together with the fence, @@ -167,6 +178,7 @@ public void createJob(LanceIndexJob job, long tableLimit, long catalogLimit, lon admitted.setBackendId(null); admitted.setBeProcessEpoch(null); admitted.setInvocationId(null); + admitted.setDispatchRevision(null); admitted.setDeadlineMs(null); admitted.setPossibleLiveOwned(false); admitted.setTerminationProof(LanceIndexTerminationProof.NONE); @@ -208,6 +220,16 @@ public boolean markRunning(long jobId, long expectedRevision, long backendId, lo jobId, expectedRevision, current); return false; } + try { + current.validateForAdmission(); + } catch (IllegalArgumentException e) { + LOG.warn("reject markRunning for invalid lance index job {}: {}", jobId, e.getMessage()); + return false; + } + if (!hasFenceIdentity(current)) { + LOG.warn("reject markRunning for lance index job {} without a valid fence identity", jobId); + return false; + } LanceIndexJob updated = new LanceIndexJob(current); updated.setMutationState(LanceIndexJobMutationState.RUNNING); updated.setBackendId(backendId); @@ -215,7 +237,9 @@ public boolean markRunning(long jobId, long expectedRevision, long backendId, lo updated.setInvocationId(invocationId); updated.setDeadlineMs(deadlineMs); updated.setPossibleLiveOwned(true); - updated.setRevision(current.getRevision() + 1); + long dispatchRevision = current.getRevision() + 1; + updated.setRevision(dispatchRevision); + updated.setDispatchRevision(dispatchRevision); updated.setUpdateTimeMs(System.currentTimeMillis()); writeEditLog(updated); applyToMemory(updated); @@ -227,7 +251,7 @@ public boolean markRunning(long jobId, long expectedRevision, long backendId, lo /** * RUNNING -> terminal, from a worker/supervisor result. A callback must - * match the durable dispatch identity exactly (job revision, invocation + * match the durable dispatch identity exactly (immutable dispatch revision, invocation * id, and BE process epoch); a stale callback only logs a warning and * changes nothing. The typed result is classified into (mutation state, * refresh obligation, completion reason); message text is never inspected. @@ -240,15 +264,16 @@ public boolean markRunning(long jobId, long expectedRevision, long backendId, lo * * @return false (with a warning) when the callback is stale or the job is not RUNNING */ - public boolean completeWithResult(long jobId, long expectedRevision, String invocationId, Long beProcessEpoch, + public boolean completeWithResult(long jobId, long expectedDispatchRevision, String invocationId, + Long beProcessEpoch, LanceIndexJobResult result) { Objects.requireNonNull(result, "result"); writeLock(); try { LanceIndexJob current = jobs.get(jobId); - if (current == null || current.getRevision() != expectedRevision) { - LOG.warn("reject stale lance index job callback for job {}: expected revision {}, current {}", - jobId, expectedRevision, current); + if (current == null || dispatchRevisionOf(current) != expectedDispatchRevision) { + LOG.warn("reject stale lance index job callback for job {}: expected dispatch revision {}, current {}", + jobId, expectedDispatchRevision, current); return false; } if (current.getMutationState() != LanceIndexJobMutationState.RUNNING) { @@ -266,6 +291,11 @@ public boolean completeWithResult(long jobId, long expectedRevision, String invo current.getMutationType(), result.getResultCode(), current.isIfExists(), result.isExternalMetadataAdvanced()); LanceIndexJob updated = new LanceIndexJob(current); + if (updated.getDispatchRevision() == null) { + // Backfill old RUNNING records before the global revision advances so + // an independent termination proof can still identify this dispatch. + updated.setDispatchRevision(expectedDispatchRevision); + } updated.setMutationState(classification.getMutationState()); updated.setRefreshState(classification.getRefreshState()); updated.setResult(new LanceIndexJobResult(result.getResultCode(), classification.getCompletionReason(), @@ -313,6 +343,11 @@ private boolean transitionRefresh(long jobId, long expectedRevision, LanceIndexJ target, jobId, expectedRevision, current); return false; } + if (current.getMutationState() == null || !current.getMutationState().isTerminal()) { + LOG.warn("reject refresh transition to {} for non-terminal lance index job {} in mutation state {}", + target, jobId, current.getMutationState()); + return false; + } LanceIndexJobRefreshState from = current.getRefreshState(); boolean legal = (target == LanceIndexJobRefreshState.RUNNING && (from == LanceIndexJobRefreshState.REQUIRED || from == LanceIndexJobRefreshState.FAILED)) @@ -337,17 +372,27 @@ private boolean transitionRefresh(long jobId, long expectedRevision, LanceIndexJ /** * Record a matching termination proof for a job that still owns a - * possible-live slot. This releases only the slot: it never changes the - * mutation state and never releases the fence or quota. + * possible-live slot. Backend, BE process epoch, invocation id, and immutable + * dispatch revision must all match. This releases only the slot: it never + * changes the mutation state and never releases the fence or quota. */ - public boolean recordTerminationProof(long jobId, long expectedRevision, LanceIndexTerminationProof proof) { + public boolean recordTerminationProof(long jobId, long expectedDispatchRevision, long backendId, + long beProcessEpoch, String invocationId, LanceIndexTerminationProof proof) { Objects.requireNonNull(proof, "proof"); writeLock(); try { LanceIndexJob current = jobs.get(jobId); - if (current == null || current.getRevision() != expectedRevision) { - LOG.warn("reject termination proof for lance index job {}: expected revision {}, current {}", - jobId, expectedRevision, current); + if (current == null || dispatchRevisionOf(current) != expectedDispatchRevision) { + LOG.warn("reject termination proof for lance index job {}: expected dispatch revision {}, current {}", + jobId, expectedDispatchRevision, current); + return false; + } + if (StringUtils.isBlank(invocationId) + || !Objects.equals(current.getBackendId(), backendId) + || !Objects.equals(current.getBeProcessEpoch(), beProcessEpoch) + || !Objects.equals(current.getInvocationId(), invocationId)) { + LOG.warn("reject stale termination proof for lance index job {}:" + + " dispatch identity mismatch, current {}", jobId, current); return false; } if (proof == LanceIndexTerminationProof.NONE || !current.isPossibleLiveOwned() @@ -357,6 +402,9 @@ public boolean recordTerminationProof(long jobId, long expectedRevision, LanceIn return false; } LanceIndexJob updated = new LanceIndexJob(current); + if (updated.getDispatchRevision() == null) { + updated.setDispatchRevision(expectedDispatchRevision); + } updated.setTerminationProof(proof); updated.setPossibleLiveOwned(false); updated.setRevision(current.getRevision() + 1); @@ -389,7 +437,7 @@ public void onTransferToMaster() { } for (LanceIndexJob job : snapshot) { if (job.getMutationState() == LanceIndexJobMutationState.RUNNING) { - boolean completed = completeWithResult(job.getJobId(), job.getRevision(), job.getInvocationId(), + boolean completed = completeWithResult(job.getJobId(), dispatchRevisionOf(job), job.getInvocationId(), job.getBeProcessEpoch(), new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT, LanceIndexJobCompletionReason.NONE, @@ -442,13 +490,14 @@ public void replayUpsertJob(LanceIndexJob job) { } writeLock(); try { - LanceIndexJob existing = jobs.get(job.getJobId()); - if (existing != null && job.getRevision() < existing.getRevision()) { + LanceIndexJob replayed = new LanceIndexJob(job); + LanceIndexJob existing = jobs.get(replayed.getJobId()); + if (existing != null && replayed.getRevision() < existing.getRevision()) { LOG.warn("ignore stale lance index job record for job {}: replayed revision {} < current {}", - job.getJobId(), job.getRevision(), existing.getRevision()); + replayed.getJobId(), replayed.getRevision(), existing.getRevision()); return; } - applyToMemory(job); + applyToMemory(replayed); } finally { writeUnlock(); } @@ -459,38 +508,53 @@ public void replayUpsertJob(LanceIndexJob job) { * the replaced record. Fence and quota always move together: a record * holds both while {@link LanceIndexJob#isUnresolved()}, provided it * carries fence identity (a corrupt identity-less record stays queryable - * but out of the books on both the charge and the release side). Caller - * holds the write lock. + * but out of the books on both the charge and the release side). A private + * copy is always stored so neither a replay input nor an edit-log seam + * reference can mutate the published record. Caller holds the write lock. */ private void applyToMemory(LanceIndexJob job) { - LanceIndexJob old = jobs.put(job.getJobId(), job); + LanceIndexJob stored = new LanceIndexJob(job); + LanceIndexJob old = jobs.put(stored.getJobId(), stored); // Release only what the identity guard below booked: an identity-less corrupt // record was stored without fence/quota accounting, so keying on it would throw. if (old != null && old.isUnresolved() && hasFenceIdentity(old)) { - fenceIndex.remove(old.fenceKey(), old.getJobId()); + removeFenceOwner(old); quota.release(old); } - if (job.isUnresolved()) { - if (hasFenceIdentity(job)) { - Long displaced = fenceIndex.put(job.fenceKey(), job.getJobId()); - if (displaced != null && displaced.longValue() != job.getJobId()) { - // Only a corrupt journal can collide here; keep the smaller job id, - // the same rule as gsonPostProcess. - LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping {}", - displaced, job.getJobId(), Math.min(displaced, job.getJobId())); - if (displaced.longValue() < job.getJobId()) { - fenceIndex.put(job.fenceKey(), displaced); - } + if (stored.isUnresolved()) { + if (hasFenceIdentity(stored)) { + NavigableSet owners = fenceIndex.computeIfAbsent(stored.fenceKey(), ignored -> new TreeSet<>()); + if (!owners.isEmpty() && !owners.contains(stored.getJobId())) { + LOG.warn("fence key collision between unresolved lance index jobs {} and {};" + + " keeping fence owner {}", owners.first(), stored.getJobId(), + Math.min(owners.first(), stored.getJobId())); } - quota.charge(job); + owners.add(stored.getJobId()); + quota.charge(stored); } else { // Corrupt record tolerance: keep it queryable but out of the fence/quota books. LOG.warn("lance index job {} lacks fence identity (provider/locator/name);" - + " stored without fence/quota accounting", job.getJobId()); + + " stored without fence/quota accounting", stored.getJobId()); } } } + private void removeFenceOwner(LanceIndexJob job) { + LanceIndexFenceKey fenceKey = job.fenceKey(); + NavigableSet owners = fenceIndex.get(fenceKey); + if (owners == null) { + return; + } + owners.remove(job.getJobId()); + if (owners.isEmpty()) { + fenceIndex.remove(fenceKey); + } + } + + private static long dispatchRevisionOf(LanceIndexJob job) { + return job.getDispatchRevision() == null ? job.getRevision() : job.getDispatchRevision(); + } + private static boolean hasFenceIdentity(LanceIndexJob job) { return job.getProvider() != null && job.getNormalizedLocator() != null && job.getNormalizedIndexName() != null; @@ -503,7 +567,8 @@ private static boolean hasFenceIdentity(LanceIndexJob job) { public LanceIndexJob getJob(long jobId) { readLock(); try { - return jobs.get(jobId); + LanceIndexJob job = jobs.get(jobId); + return job == null ? null : new LanceIndexJob(job); } finally { readUnlock(); } @@ -518,8 +583,8 @@ public List getUnresolvedJobs() { try { List result = new ArrayList<>(); for (LanceIndexJob job : jobs.values()) { - if (job != null && job.isUnresolved()) { - result.add(job); + if (job != null && job.isUnresolved() && hasFenceIdentity(job)) { + result.add(new LanceIndexJob(job)); } } return result; @@ -541,9 +606,10 @@ public List getJobsNeedingRefresh() { List result = new ArrayList<>(); for (LanceIndexJob job : jobs.values()) { if (job != null && job.getMutationState() != null && job.getMutationState().isTerminal() + && hasFenceIdentity(job) && (job.getRefreshState() == LanceIndexJobRefreshState.REQUIRED || job.getRefreshState() == LanceIndexJobRefreshState.FAILED)) { - result.add(job); + result.add(new LanceIndexJob(job)); } } return result; @@ -562,7 +628,7 @@ public boolean isFenceHeld(LanceIndexFenceKey fenceKey) { } @VisibleForTesting - public LanceIndexJobQuota getQuota() { + LanceIndexJobQuota getQuota() { return quota; } @@ -595,8 +661,9 @@ public static LanceIndexJobManager read(DataInput in) throws IOException { /** * Rebuild the derived fence index and quota counters from the durable * jobs after Gson image load. A fence-key collision between unresolved - * jobs (only possible on a corrupt image) keeps the smaller job id and - * logs a warning; replay itself can never produce one. + * jobs (only possible on a corrupt image) retains every owner so the fence + * remains held until all colliding jobs settle; conflict reporting still + * uses the smaller job id. */ @Override public void gsonPostProcess() throws IOException { @@ -615,14 +682,12 @@ public void gsonPostProcess() throws IOException { continue; } unresolvedJobs.add(job); - Long existing = fenceIndex.get(job.fenceKey()); - if (existing != null) { - LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping {}", - existing, job.getJobId(), Math.min(existing, job.getJobId())); - } - if (existing == null || job.getJobId() < existing) { - fenceIndex.put(job.fenceKey(), job.getJobId()); + NavigableSet owners = fenceIndex.computeIfAbsent(job.fenceKey(), ignored -> new TreeSet<>()); + if (!owners.isEmpty() && !owners.contains(job.getJobId())) { + LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping fence owner {}", + owners.first(), job.getJobId(), Math.min(owners.first(), job.getJobId())); } + owners.add(job.getJobId()); } quota.rebuild(unresolvedJobs); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java index ca4b6e4154762c..5634dc6f9355a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java @@ -33,8 +33,7 @@ * *

This class only counts. Callers hold the manager write lock, so no * internal synchronization exists. Config-gated limits are resolved by the - * admission layer and passed in as plain values; a non-positive limit disables - * that level's check. The counters are rebuilt from the durable jobs after + * admission layer and passed in as positive finite values. The counters are rebuilt from the durable jobs after * replay/image load and are never persisted themselves. */ public class LanceIndexJobQuota { @@ -47,7 +46,7 @@ public class LanceIndexJobQuota { /** * Check every level whose limit is positive, then increment all three * levels. Returns false (and increments nothing) when any enforced level - * is full: "current + 1 <= limit" must hold at every enforced level. + * is full: "current < limit" must hold at every level. * *

The manager admission path uses {@link #hasCapacity} plus * {@link #charge} instead: the check and the charge deliberately straddle @@ -64,17 +63,20 @@ public boolean tryAcquire(LanceIndexJob job, long tableLimit, long catalogLimit, /** * Pure check variant of {@link #tryAcquire}: true when incrementing would - * not exceed any positive limit. A non-positive limit disables that level. + * not exceed any positive finite limit. A non-positive limit is rejected. */ public boolean hasCapacity(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) { Objects.requireNonNull(job, "job"); - if (globalLimit > 0 && globalCount + 1 > globalLimit) { + if (tableLimit <= 0 || catalogLimit <= 0 || globalLimit <= 0) { return false; } - if (catalogLimit > 0 && getCatalogCount(job.getCatalogId()) + 1 > catalogLimit) { + if (globalCount >= globalLimit) { return false; } - return tableLimit <= 0 || getTableCount(job.getTableQuotaKey()) + 1 <= tableLimit; + if (getCatalogCount(job.getCatalogId()) >= catalogLimit) { + return false; + } + return getTableCount(job.getTableQuotaKey()) < tableLimit; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java index e38175e91c2658..1175de7223da46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java @@ -19,6 +19,7 @@ import com.google.gson.annotations.SerializedName; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -37,6 +38,10 @@ public class LanceIndexSchemaContract { /** The only schema contract version defined. */ public static final int SCHEMA_CONTRACT_VERSION_V1 = 1; + /** A finite representation bound; 4.2 admission may impose a smaller type-specific arity. */ + public static final int MAX_INDEXED_FIELDS = 64; + /** Bound for every persisted schema-contract string. */ + public static final int MAX_FIELD_STRING_BYTES = 1024; @SerializedName(value = "scv") private int schemaContractVersion = SCHEMA_CONTRACT_VERSION_V1; @@ -52,7 +57,52 @@ public LanceIndexSchemaContract() { public LanceIndexSchemaContract(List fields) { this.schemaContractVersion = SCHEMA_CONTRACT_VERSION_V1; - this.fields = Collections.unmodifiableList(new ArrayList<>(Objects.requireNonNull(fields, "fields"))); + this.fields = validatedFields(fields); + } + + /** + * Revalidate a possibly Gson-created contract before it enters a new durable + * admission record. Replay remains tolerant, but newly journaled records must + * use the supported version and bounded representation. + */ + public void validateForAdmission() { + if (schemaContractVersion != SCHEMA_CONTRACT_VERSION_V1) { + throw new IllegalArgumentException("unsupported schema contract version: " + schemaContractVersion); + } + validatedFields(fields); + } + + private static List validatedFields(List source) { + if (source == null) { + throw new IllegalArgumentException("schema contract fields must not be null"); + } + if (source.size() > MAX_INDEXED_FIELDS) { + throw new IllegalArgumentException( + "schema contract exceeds " + MAX_INDEXED_FIELDS + " indexed fields"); + } + List copy = new ArrayList<>(source.size()); + for (IndexedField field : source) { + if (field == null) { + throw new IllegalArgumentException("schema contract indexed field must not be null"); + } + field.validateForAdmission(); + copy.add(field); + } + return Collections.unmodifiableList(copy); + } + + private static String checkString(String value, boolean nullable, String fieldName) { + if (value == null) { + if (nullable) { + return null; + } + throw new IllegalArgumentException(fieldName + " must not be null"); + } + if (value.getBytes(StandardCharsets.UTF_8).length > MAX_FIELD_STRING_BYTES) { + throw new IllegalArgumentException( + fieldName + " exceeds " + MAX_FIELD_STRING_BYTES + " UTF-8 bytes"); + } + return value; } public int getSchemaContractVersion() { @@ -122,14 +172,20 @@ public IndexedField() { public IndexedField(long fieldId, String normalizedName, String normalizedType, boolean nullable, Integer fixedSizeListDimension, String vectorElementType, Boolean vectorElementNullable) { this.fieldId = fieldId; - this.normalizedName = Objects.requireNonNull(normalizedName, "normalizedName"); - this.normalizedType = Objects.requireNonNull(normalizedType, "normalizedType"); + this.normalizedName = checkString(normalizedName, false, "normalizedName"); + this.normalizedType = checkString(normalizedType, false, "normalizedType"); this.nullable = nullable; this.fixedSizeListDimension = fixedSizeListDimension; - this.vectorElementType = vectorElementType; + this.vectorElementType = checkString(vectorElementType, true, "vectorElementType"); this.vectorElementNullable = vectorElementNullable; } + private void validateForAdmission() { + checkString(normalizedName, false, "normalizedName"); + checkString(normalizedType, false, "normalizedType"); + checkString(vectorElementType, true, "vectorElementType"); + } + public long getFieldId() { return fieldId; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java index 64293dad43f0db..2e46203b815959 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.lance.job; +import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -94,6 +95,60 @@ public void rejectsCredentialBearingUserinfo() { () -> LanceIndexDatasetLocator.normalize("https://user:secret@example.com/ds")); } + @Test + public void rejectsQueriesAndFragmentsThatCouldCarryCredentials() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize( + "https://bucket.example/ds?X-Amz-Credential=AKIA_TEST&X-Amz-Signature=secret")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("https://account.blob.core.windows.net/ds?sig=secret")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("s3://bucket/ds#credential-fragment")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("/data/ds?token=secret")); + } + + @Test + public void malformedLocatorErrorsDoNotEchoPotentialSecrets() { + String malformed = "s 3://user:top-secret@bucket/path"; + IllegalArgumentException malformedError = Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize(malformed)); + Assertions.assertFalse(malformedError.getMessage().contains(malformed)); + Assertions.assertFalse(malformedError.getMessage().contains("top-secret")); + + String relative = "access-key:top-secret@bucket/path"; + IllegalArgumentException relativeError = Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize(relative)); + Assertions.assertFalse(relativeError.getMessage().contains(relative)); + Assertions.assertFalse(relativeError.getMessage().contains("top-secret")); + } + + @Test + public void rejectsMalformedOrNonHierarchicalSchemes() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("1s3://bucket/path")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("s3:/bucket/path")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("s3:bucket/path")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize("https://[malformed/path")); + } + + @Test + public void locatorBoundCountsUtf8Bytes() { + String prefix = "s3://bucket/"; + String atLimit = prefix + StringUtils.repeat( + "a", LanceIndexDatasetLocator.MAX_LOCATOR_BYTES - prefix.length()); + Assertions.assertEquals(atLimit, LanceIndexDatasetLocator.normalize(atLimit)); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize(atLimit + "a")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> LanceIndexDatasetLocator.normalize(prefix + StringUtils.repeat("é", + (LanceIndexDatasetLocator.MAX_LOCATOR_BYTES - prefix.length()) / 2 + 1))); + } + @Test public void rejectsSchemeLessRelativePath() { Assertions.assertThrows(IllegalArgumentException.class, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java index b37dabd6d99bfb..c63733355c9cd8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java @@ -150,6 +150,54 @@ public void schemaContractEqualityIsOrderSensitive() { Assertions.assertNotEquals(ordered, reordered); } + @Test + public void schemaContractIsBoundedNullFreeAndImmutable() { + LanceIndexSchemaContract.IndexedField field = new LanceIndexSchemaContract.IndexedField( + 1L, "v", "fixed_size_list[float;192]", false, 192, "float", false); + List atLimit = new ArrayList<>(); + for (int index = 0; index < LanceIndexSchemaContract.MAX_INDEXED_FIELDS; index++) { + atLimit.add(field); + } + LanceIndexSchemaContract contractAtLimit = new LanceIndexSchemaContract(atLimit); + Assertions.assertEquals(LanceIndexSchemaContract.MAX_INDEXED_FIELDS, contractAtLimit.getFields().size()); + + atLimit.add(field); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexSchemaContract(atLimit)); + List withNull = new ArrayList<>(); + withNull.add(null); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexSchemaContract(withNull)); + + List source = new ArrayList<>(); + source.add(field); + LanceIndexSchemaContract copied = new LanceIndexSchemaContract(source); + source.clear(); + Assertions.assertEquals(1, copied.getFields().size()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> copied.getFields().add(field)); + } + + @Test + public void schemaContractStringBoundsCountUtf8Bytes() { + String atLimit = StringUtils.repeat("é", LanceIndexSchemaContract.MAX_FIELD_STRING_BYTES / 2); + new LanceIndexSchemaContract.IndexedField(1L, atLimit, atLimit, false, null, atLimit, null); + + String overLimit = atLimit + "é"; + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexSchemaContract.IndexedField( + 1L, overLimit, "type", false, null, null, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexSchemaContract.IndexedField( + 1L, "name", overLimit, false, null, null, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexSchemaContract.IndexedField( + 1L, "name", "type", false, null, overLimit, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexSchemaContract.IndexedField( + 1L, null, "type", false, null, null, null)); + } + @Test public void journalEntityRoundtripUsesOpCode500() throws Exception { LanceIndexJob job = fullyPopulatedJob(); @@ -232,6 +280,16 @@ public void boundedTextFieldsRejectOverflow() { StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES), false); LanceIndexJob job = newCreateJob(1L, "IdxA"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setCreator(StringUtils.repeat("c", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setIndexType(StringUtils.repeat("t", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setColumnName(StringUtils.repeat("c", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setInvocationId(StringUtils.repeat("i", LanceIndexJob.MAX_INVOCATION_ID_BYTES + 1))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> job.setForceActor(StringUtils.repeat("a", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1))); Assertions.assertThrows(IllegalArgumentException.class, () -> job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES + 1))); job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES)); @@ -247,6 +305,19 @@ public void boundedTextFieldsRejectOverflow() { StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1))); Assertions.assertThrows(IllegalArgumentException.class, () -> newCreateJob(2L, StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1))); + + // Optional durable text remains nullable; required creator identity does not. + job.setCreator(null); + Assertions.assertThrows(IllegalArgumentException.class, job::validateForAdmission); + job.setCreator("tester"); + job.setIndexType(null); + job.setColumnName(null); + job.setInvocationId(null); + job.setForceActor(null); + job.setForceNote(null); + job.setForceWarning(null); + job.setPropertiesJson(null); + job.validateForAdmission(); } @Test @@ -289,6 +360,7 @@ private static LanceIndexJob fullyPopulatedJob() { LanceIndexJobCompletionReason.NONE, "io error", true)); job.setBackendId(BACKEND_ID); job.setBeProcessEpoch(BE_EPOCH); + job.setDispatchRevision(7L); job.setInvocationId(INVOCATION_ID); job.setDeadlineMs(123456L); job.setPossibleLiveOwned(true); @@ -338,6 +410,7 @@ private static void assertSameJobFields(LanceIndexJob expected, LanceIndexJob ac actual.getResult().isExternalMetadataAdvanced()); Assertions.assertEquals(expected.getBackendId(), actual.getBackendId()); Assertions.assertEquals(expected.getBeProcessEpoch(), actual.getBeProcessEpoch()); + Assertions.assertEquals(expected.getDispatchRevision(), actual.getDispatchRevision()); Assertions.assertEquals(expected.getInvocationId(), actual.getInvocationId()); Assertions.assertEquals(expected.getDeadlineMs(), actual.getDeadlineMs()); Assertions.assertEquals(expected.isPossibleLiveOwned(), actual.isPossibleLiveOwned()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java index ecf7e3b07b9088..68d51a134e9057 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java @@ -95,7 +95,7 @@ public void transferToMasterConvertsRunningToUnknownAndNeverRedispatches() throw Assertions.assertTrue(target.isFenceHeld(fenceKey)); Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); Assertions.assertTrue(swept.holdsPossibleLiveSlot()); - Assertions.assertTrue(target.getUnresolvedJobs().contains(swept)); + Assertions.assertTrue(containsJob(target.getUnresolvedJobs(), swept.getJobId())); Assertions.assertEquals(1, target.editLog.size()); Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); @@ -124,7 +124,7 @@ public void transferToMasterDowngradesRunningRefreshToRequired() throws DdlExcep Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, swept.getMutationState()); Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, swept.getRefreshState()); Assertions.assertEquals(4L, swept.getRevision()); - Assertions.assertTrue(target.getJobsNeedingRefresh().contains(swept)); + Assertions.assertTrue(containsJob(target.getJobsNeedingRefresh(), swept.getJobId())); Assertions.assertTrue(target.isFenceHeld(swept.fenceKey())); } @@ -147,7 +147,7 @@ public void replayedTerminalWithRefreshRequiredOnlyAllowsRefreshPath() throws Dd // The mutation lifecycle is closed; only the refresh transitions remain. Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); Assertions.assertFalse(target.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, okResult())); - Assertions.assertTrue(target.getJobsNeedingRefresh().contains(stored)); + Assertions.assertTrue(containsJob(target.getJobsNeedingRefresh(), stored.getJobId())); Assertions.assertTrue(target.isFenceHeld(fenceKey)); Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); @@ -274,6 +274,62 @@ public void replayToleratesNullAndIdentityLessRecords() { Assertions.assertEquals(1, target.getJobCount()); Assertions.assertEquals(0L, target.getQuota().getGlobalCount()); Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(5L).getMutationState()); + Assertions.assertTrue(target.getUnresolvedJobs().isEmpty()); + Assertions.assertFalse(target.markRunning(5L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + } + + @Test + public void resolvingTheSmallestCollidingJobLeavesTheOtherFenceOwner() throws DdlException { + TestManager target = new TestManager(); + target.replayUpsertJob(pendingRecord(1L, "IdxA")); + target.replayUpsertJob(pendingRecord(2L, "IdxA")); + LanceIndexFenceKey fenceKey = target.getJob(1L).fenceKey(); + + Assertions.assertEquals(2L, target.getQuota().getGlobalCount()); + DdlException initialConflict = Assertions.assertThrows(DdlException.class, + () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100)); + Assertions.assertTrue(initialConflict.getMessage().contains("unresolved job 1")); + + Assertions.assertTrue(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS)); + Assertions.assertTrue(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + new LanceIndexJobResult(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED, + LanceIndexJobCompletionReason.NONE, "rejected before invocation", false))); + + Assertions.assertTrue(target.isFenceHeld(fenceKey)); + Assertions.assertEquals(1L, target.getQuota().getGlobalCount()); + DdlException remainingConflict = Assertions.assertThrows(DdlException.class, + () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100)); + Assertions.assertTrue(remainingConflict.getMessage().contains("unresolved job 2")); + } + + @Test + public void replayAndQueryBoundariesReturnDefensiveCopies() throws DdlException { + TestManager target = new TestManager(); + LanceIndexJob replayed = pendingRecord(1L, "IdxA"); + target.replayUpsertJob(replayed); + + replayed.setMutationState(LanceIndexJobMutationState.COMMITTED); + replayed.setRefreshState(LanceIndexJobRefreshState.DONE); + replayed.setRevision(99L); + Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(1L).getMutationState()); + Assertions.assertEquals(0L, target.getJob(1L).getRevision()); + + LanceIndexJob queried = target.getJob(1L); + queried.setForceReleased(true); + queried.setRevision(88L); + Assertions.assertFalse(target.getJob(1L).isForceReleased()); + Assertions.assertEquals(0L, target.getJob(1L).getRevision()); + + LanceIndexJob listed = target.getUnresolvedJobs().get(0); + listed.setForceReleased(true); + Assertions.assertFalse(target.getJob(1L).isForceReleased()); + + target.createJob(newCreateJob(2L, "IdxB"), 100, 100, 100); + Assertions.assertTrue(target.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, "invocation-2", DEADLINE_MS)); + Assertions.assertTrue(target.completeWithResult(2L, 1L, "invocation-2", BE_EPOCH, okResult())); + LanceIndexJob refreshCandidate = target.getJobsNeedingRefresh().get(0); + refreshCandidate.setRefreshState(LanceIndexJobRefreshState.DONE); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, target.getJob(2L).getRefreshState()); } @Test @@ -384,6 +440,17 @@ private static LanceIndexJob newCreateJob(long jobId, String displayName) { null, 7L, null); } + private static LanceIndexJob pendingRecord(long jobId, String displayName) { + LanceIndexJob job = newCreateJob(jobId, displayName); + job.setMutationState(LanceIndexJobMutationState.PENDING); + job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED); + return job; + } + + private static boolean containsJob(List jobs, long jobId) { + return jobs.stream().anyMatch(job -> job.getJobId() == jobId); + } + private static LanceIndexJobResult okResult() { return new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, LanceIndexJobCompletionReason.NONE, "ok", false); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java index c4527a21265a72..9511f7f1bc06a2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java @@ -26,8 +26,8 @@ /** * Unit coverage for the three-level unresolved-job quota counters (table/locator, - * catalog, global): the "current + 1 <= limit" boundary at each level, disabled - * levels for non-positive limits, release recovery with underflow clamping, and rebuild + * catalog, global): the "current < limit" boundary at each level, rejection of + * non-positive limits, release recovery with underflow clamping, and rebuild * equivalence with live counting. The "which jobs count" semantics are * {@link LanceIndexJob#isUnresolved()}; the rebuild-side composition is pinned here too. */ @@ -50,51 +50,48 @@ public void tryAcquireChargesAllThreeLevels() { @Test public void tableLimitRejectsTheNextJobExactlyAtLimit() { LanceIndexJobQuota quota = new LanceIndexJobQuota(); - Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 2, 0, 0)); - Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 2, 0, 0)); + Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 2, 100, 100)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 2, 100, 100)); LanceIndexJob third = newJob(3L, CATALOG_ID, LOCATOR_A); - Assertions.assertFalse(quota.tryAcquire(third, 2, 0, 0)); + Assertions.assertFalse(quota.tryAcquire(third, 2, 100, 100)); // A rejected acquire charges nothing at any level. Assertions.assertEquals(2L, quota.getGlobalCount()); Assertions.assertEquals(2L, quota.getTableCount(third.getTableQuotaKey())); // The limit is per table/locator identity: another table still has room. - Assertions.assertTrue(quota.tryAcquire(newJob(4L, CATALOG_ID, LOCATOR_B), 2, 0, 0)); + Assertions.assertTrue(quota.tryAcquire(newJob(4L, CATALOG_ID, LOCATOR_B), 2, 100, 100)); } @Test public void catalogLimitRejectsAcrossTables() { LanceIndexJobQuota quota = new LanceIndexJobQuota(); - Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 0, 2, 0)); - Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_B), 0, 2, 0)); - Assertions.assertFalse(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 0, 2, 0)); + Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 100, 2, 100)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_B), 100, 2, 100)); + Assertions.assertFalse(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 100, 2, 100)); // Another catalog is a separate level. - Assertions.assertTrue(quota.tryAcquire(newJob(4L, 20L, LOCATOR_A), 0, 2, 0)); + Assertions.assertTrue(quota.tryAcquire(newJob(4L, 20L, LOCATOR_A), 100, 2, 100)); } @Test public void globalLimitRejectsAcrossCatalogs() { LanceIndexJobQuota quota = new LanceIndexJobQuota(); - Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 0, 0, 2)); - Assertions.assertTrue(quota.tryAcquire(newJob(2L, 20L, LOCATOR_A), 0, 0, 2)); - Assertions.assertFalse(quota.tryAcquire(newJob(3L, 30L, LOCATOR_B), 0, 0, 2)); + Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 100, 100, 2)); + Assertions.assertTrue(quota.tryAcquire(newJob(2L, 20L, LOCATOR_A), 100, 100, 2)); + Assertions.assertFalse(quota.tryAcquire(newJob(3L, 30L, LOCATOR_B), 100, 100, 2)); Assertions.assertEquals(2L, quota.getGlobalCount()); } @Test - public void nonPositiveLimitDisablesThatLevel() { + public void nonPositiveLimitIsRejectedWithoutACharge() { LanceIndexJobQuota quota = new LanceIndexJobQuota(); - for (int i = 0; i < 10; i++) { - Assertions.assertTrue(quota.tryAcquire(newJob(i, CATALOG_ID, LOCATOR_A), 0, 0, 0)); - } - LanceIndexJobQuota negativeLimits = new LanceIndexJobQuota(); - for (int i = 0; i < 10; i++) { - Assertions.assertTrue(negativeLimits.tryAcquire(newJob(i, CATALOG_ID, LOCATOR_A), -1, -1, -1)); - } - Assertions.assertEquals(10L, quota.getGlobalCount()); - Assertions.assertEquals(10L, negativeLimits.getGlobalCount()); + LanceIndexJob job = newJob(1L, CATALOG_ID, LOCATOR_A); + Assertions.assertFalse(quota.tryAcquire(job, 0, 1, 1)); + Assertions.assertFalse(quota.tryAcquire(job, 1, 0, 1)); + Assertions.assertFalse(quota.tryAcquire(job, 1, 1, 0)); + Assertions.assertFalse(quota.tryAcquire(job, -1, 1, 1)); + Assertions.assertEquals(0L, quota.getGlobalCount()); } @Test @@ -132,7 +129,7 @@ public void rebuildMatchesIncrementalCounting() { LanceIndexJobQuota incremental = new LanceIndexJobQuota(); for (LanceIndexJob job : unresolved) { - Assertions.assertTrue(incremental.tryAcquire(job, 0, 0, 0)); + Assertions.assertTrue(incremental.tryAcquire(job, 100, 100, 100)); } LanceIndexJobQuota rebuilt = new LanceIndexJobQuota(); rebuilt.rebuild(unresolved); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java index deb848b3048e5c..54a5945eb79b16 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java @@ -86,7 +86,7 @@ public void pendingToRunningToCommitted() throws DdlException { Assertions.assertEquals(2L, committed.getRevision()); Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_OK, committed.getResult().getResultCode()); Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, committed.getResult().getCompletionReason()); - Assertions.assertTrue(manager.getJobsNeedingRefresh().contains(committed)); + Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), committed.getJobId())); } @Test @@ -229,7 +229,7 @@ public void refreshFailureKeepsFenceAndRetriesThroughRunning() throws DdlExcepti Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, failed.getRefreshState()); Assertions.assertTrue(manager.isFenceHeld(fenceKey)); Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); - Assertions.assertTrue(manager.getUnresolvedJobs().contains(failed)); + Assertions.assertTrue(containsJob(manager.getUnresolvedJobs(), failed.getJobId())); // FAILED -> RUNNING is the retry entry through the idempotent refresh path. Assertions.assertTrue(manager.markRefreshRunning(1L, 4L)); @@ -322,9 +322,12 @@ public void terminationProofReleasesSlotOnly() throws DdlException { LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot()); - Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, LanceIndexTerminationProof.NONE)); - Assertions.assertFalse(manager.recordTerminationProof(1L, 99L, LanceIndexTerminationProof.CHILD_REAPED)); - Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.NONE)); + Assertions.assertFalse(manager.recordTerminationProof(1L, 99L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); LanceIndexJob stored = manager.getJob(1L); Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState()); @@ -334,7 +337,8 @@ public void terminationProofReleasesSlotOnly() throws DdlException { Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); // A slot may be proven exactly once. - Assertions.assertFalse(manager.recordTerminationProof(1L, 2L, LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE)); + Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE)); } @Test @@ -385,7 +389,7 @@ public void failedRefreshJobStaysVisibleToTheRefreshDriver() throws DdlException Assertions.assertTrue(manager.markRefreshFailed(1L, 3L)); // FAILED still owes the idempotent retry: the driver must see the job. - Assertions.assertTrue(manager.getJobsNeedingRefresh().contains(manager.getJob(1L))); + Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), 1L)); Assertions.assertTrue(manager.markRefreshRunning(1L, 4L)); Assertions.assertTrue(manager.markRefreshDone(1L, 5L)); @@ -469,7 +473,8 @@ public void terminationProofNeedsASlotAndStillLandsAfterTheTerminalResult() thro manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100); // PENDING owns no possible-live slot: a proof has nothing to release. - Assertions.assertFalse(manager.recordTerminationProof(1L, 0L, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertFalse(manager.recordTerminationProof(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); Assertions.assertEquals(0L, manager.getJob(1L).getRevision()); Assertions.assertEquals(1, manager.editLog.size()); @@ -482,7 +487,8 @@ public void terminationProofNeedsASlotAndStillLandsAfterTheTerminalResult() thro Assertions.assertTrue(committed.holdsPossibleLiveSlot()); LanceIndexFenceKey fenceKey = committed.fenceKey(); - Assertions.assertTrue(manager.recordTerminationProof(1L, 2L, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); LanceIndexJob proven = manager.getJob(1L); Assertions.assertFalse(proven.holdsPossibleLiveSlot()); // Fence and quota still follow the refresh rule, not the proof. @@ -495,6 +501,78 @@ public void terminationProofNeedsASlotAndStillLandsAfterTheTerminalResult() thro Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); } + @Test + public void terminationProofAndResultUseTheImmutableDispatchRevisionInEitherOrder() throws DdlException { + TestManager proofFirst = new TestManager(); + createAndRun(proofFirst, 1L, "IdxA"); + Assertions.assertEquals(1L, proofFirst.getJob(1L).getDispatchRevision()); + + Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID + 1, BE_EPOCH, + INVOCATION_ID, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH + 1, + INVOCATION_ID, LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, + "wrong-invocation", LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertEquals(1L, proofFirst.getJob(1L).getRevision()); + + Assertions.assertTrue(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertEquals(2L, proofFirst.getJob(1L).getRevision()); + Assertions.assertEquals(1L, proofFirst.getJob(1L).getDispatchRevision()); + Assertions.assertTrue(proofFirst.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, + proofFirst.getJob(1L).getMutationState()); + + TestManager resultFirst = new TestManager(); + createAndRun(resultFirst, 2L, "IdxB"); + Assertions.assertTrue(resultFirst.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertTrue(resultFirst.recordTerminationProof(2L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertFalse(resultFirst.getJob(2L).holdsPossibleLiveSlot()); + Assertions.assertEquals(1L, resultFirst.getJob(2L).getDispatchRevision()); + + TestManager legacyResultFirst = new TestManager(); + createAndRun(legacyResultFirst, 3L, "IdxC"); + LanceIndexJob legacyRunning = legacyResultFirst.getJob(3L); + legacyRunning.setDispatchRevision(null); + legacyResultFirst.replayUpsertJob(legacyRunning); + Assertions.assertTrue(legacyResultFirst.completeWithResult(3L, 1L, INVOCATION_ID, BE_EPOCH, + result(LanceIndexJobResultCode.NATIVE_OK))); + Assertions.assertEquals(1L, legacyResultFirst.getJob(3L).getDispatchRevision()); + Assertions.assertTrue(legacyResultFirst.recordTerminationProof(3L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.CHILD_REAPED)); + Assertions.assertFalse(legacyResultFirst.getJob(3L).holdsPossibleLiveSlot()); + } + + @Test + public void refreshTransitionsRejectNonTerminalMutationState() { + TestManager manager = new TestManager(); + LanceIndexJob corrupt = newCreateJob(1L, "IdxA"); + corrupt.setMutationState(LanceIndexJobMutationState.PENDING); + corrupt.setRefreshState(LanceIndexJobRefreshState.REQUIRED); + manager.replayUpsertJob(corrupt); + + Assertions.assertFalse(manager.markRefreshRunning(1L, 0L)); + Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState()); + Assertions.assertTrue(manager.editLog.isEmpty()); + } + + @Test + public void admissionRejectsNonPositiveQuotaLimitsBeforeDurableWrite() { + TestManager manager = new TestManager(); + Assertions.assertThrows(DdlException.class, + () -> manager.createJob(newCreateJob(1L, "IdxA"), 0, 1, 1)); + Assertions.assertThrows(DdlException.class, + () -> manager.createJob(newCreateJob(1L, "IdxA"), 1, 0, 1)); + Assertions.assertThrows(DdlException.class, + () -> manager.createJob(newCreateJob(1L, "IdxA"), 1, 1, -1)); + Assertions.assertEquals(0, manager.getJobCount()); + Assertions.assertEquals(0L, manager.getQuota().getGlobalCount()); + Assertions.assertTrue(manager.editLog.isEmpty()); + } + @Test public void notCommittedWithRefreshRequiredReleasesFenceAtRefreshDone() throws DdlException { TestManager manager = new TestManager(); @@ -509,7 +587,7 @@ public void notCommittedWithRefreshRequiredReleasesFenceAtRefreshDone() throws D LanceIndexFenceKey fenceKey = stored.fenceKey(); Assertions.assertTrue(manager.isFenceHeld(fenceKey)); Assertions.assertEquals(1L, manager.getQuota().getGlobalCount()); - Assertions.assertTrue(manager.getJobsNeedingRefresh().contains(stored)); + Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), stored.getJobId())); Assertions.assertTrue(manager.markRefreshRunning(1L, 2L)); Assertions.assertTrue(manager.markRefreshDone(1L, 3L)); @@ -526,6 +604,10 @@ private static LanceIndexJob newCreateJob(long jobId, String displayName) { null, 7L, null); } + private static boolean containsJob(List jobs, long jobId) { + return jobs.stream().anyMatch(job -> job.getJobId() == jobId); + } + private static LanceIndexJob newDropJob(long jobId, String displayName, boolean ifExists) { return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1", LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java index dd5bd9b10dea55..1f61b184d196af 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java @@ -169,7 +169,8 @@ public void terminationProofClearsSlotButKeepsFenceAndOutcome() throws Exception manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L); LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey(); - Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE)); + Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, + LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE)); LanceIndexJob proven = manager.getJob(1L); Assertions.assertFalse(proven.holdsPossibleLiveSlot()); Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, proven.getMutationState()); @@ -178,7 +179,7 @@ public void terminationProofClearsSlotButKeepsFenceAndOutcome() throws Exception // The ambiguous result still lands afterwards: UNKNOWN keeps the fence, and the // already-recorded proof keeps the slot released. - Assertions.assertTrue(manager.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, + Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT, LanceIndexJobCompletionReason.NONE, "ambiguous", false))); LanceIndexJob unknown = manager.getJob(1L); @@ -198,6 +199,7 @@ public void copyConstructorDuplicatesEveryFieldIndependently() { LanceIndexJobCompletionReason.NONE, "ok", false)); original.setBackendId(BACKEND_ID); original.setBeProcessEpoch(BE_EPOCH); + original.setDispatchRevision(1L); original.setInvocationId(INVOCATION_ID); original.setDeadlineMs(123L); original.setPossibleLiveOwned(true); @@ -241,6 +243,27 @@ public void admissionConstructorRejectsNullIdentity() { LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); } + @Test + public void admissionRejectsPseudoCanonicalFenceIdentity() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + "directory", LOCATOR, "IdxA", "idxa", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, "S3://bucket/dataset/", "IdxA", "idxa", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", "IdxA", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1", + LanceIndexFenceKey.PROVIDER_DIRECTORY, + "https://bucket.example/ds?X-Amz-Signature=secret", "IdxA", "idxa", + LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null)); + } + @Test public void toStringNamesTheIndexButHidesTheLocator() { LanceIndexJob job = newCreateJob(1L, "IdxA"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java new file mode 100644 index 00000000000000..ecbd2b42799bb1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.job; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.io.CountingDataOutputStream; +import org.apache.doris.persist.OperationType; +import org.apache.doris.persist.meta.MetaPersistMethod; +import org.apache.doris.persist.meta.PersistMetaModules; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.DataInputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class LanceIndexJobWiringTest { + private static final short LANCE_INDEX_JOB_OPCODE = 500; + private static final String LANCE_INDEX_JOB_MODULE = "lanceIndexJobManager"; + + @Test + public void lanceIndexJobOpcodeIsUniquelyAssigned() throws IllegalAccessException { + List fieldsUsingOpcode = new ArrayList<>(); + for (Field field : OperationType.class.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) + && field.getType() == short.class + && field.getShort(null) == LANCE_INDEX_JOB_OPCODE) { + fieldsUsingOpcode.add(field.getName()); + } + } + + Assertions.assertEquals( + Collections.singletonList("OP_LANCE_INDEX_JOB_UPSERT"), + fieldsUsingOpcode, + "operation code 500 must remain uniquely assigned to Lance index job upserts"); + Assertions.assertEquals(LANCE_INDEX_JOB_OPCODE, OperationType.OP_LANCE_INDEX_JOB_UPSERT); + } + + @Test + public void lanceIndexJobManagerIsTheLastBaseImageModuleWithEnvBindings() throws Exception { + Assertions.assertEquals( + LANCE_INDEX_JOB_MODULE, + PersistMetaModules.MODULE_NAMES.get(PersistMetaModules.MODULE_NAMES.size() - 1), + "new image modules must be appended without reordering existing base modules"); + + MetaPersistMethod persistMethod = PersistMetaModules.MODULES_MAP.get(LANCE_INDEX_JOB_MODULE); + Assertions.assertNotNull(persistMethod); + + Method expectedReadMethod = Env.class.getDeclaredMethod( + "loadLanceIndexJobManager", DataInputStream.class, long.class); + Method expectedWriteMethod = Env.class.getDeclaredMethod( + "saveLanceIndexJobManager", CountingDataOutputStream.class, long.class); + Assertions.assertEquals(expectedReadMethod, persistMethod.readMethod); + Assertions.assertEquals(expectedWriteMethod, persistMethod.writeMethod); + Assertions.assertEquals(long.class, persistMethod.readMethod.getReturnType()); + Assertions.assertEquals(long.class, persistMethod.writeMethod.getReturnType()); + } +}