diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 43235d9ac51160..b1c17cec6aa786 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -109,6 +109,7 @@
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.datasource.iceberg.IcebergSysExternalTable;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
+import org.apache.doris.datasource.lance.job.LanceIndexJobManager;
import org.apache.doris.datasource.paimon.PaimonExternalTable;
import org.apache.doris.datasource.paimon.PaimonSysExternalTable;
import org.apache.doris.deploy.DeployManager;
@@ -568,6 +569,8 @@ public class Env {
private InsertOverwriteManager insertOverwriteManager;
+ private LanceIndexJobManager lanceIndexJobManager;
+
private DNSCache dnsCache;
private final NereidsSqlCacheManager sqlCacheManager;
@@ -853,6 +856,7 @@ public Env(boolean isCheckpointCatalog) {
this.mtmvService = new MTMVService();
this.eventProcessor = new EventProcessor(mtmvService);
this.insertOverwriteManager = new InsertOverwriteManager();
+ this.lanceIndexJobManager = new LanceIndexJobManager();
this.dnsCache = new DNSCache();
this.sqlCacheManager = new NereidsSqlCacheManager();
this.sortedPartitionsCacheManager = new NereidsSortedPartitionsCacheManager();
@@ -978,6 +982,10 @@ public InsertOverwriteManager getInsertOverwriteManager() {
return insertOverwriteManager;
}
+ public LanceIndexJobManager getLanceIndexJobManager() {
+ return lanceIndexJobManager;
+ }
+
public TabletScheduler getTabletScheduler() {
return tabletScheduler;
}
@@ -1812,6 +1820,11 @@ private void transferToMaster() {
insertOverwriteManager.allTaskFail();
+ // A durable RUNNING Lance index job at this point may have lost its result with the
+ // old master: sweep it to UNKNOWN (and refresh RUNNING back to REQUIRED) before any
+ // master-only dispatcher could start.
+ lanceIndexJobManager.onTransferToMaster();
+
toMasterProgress = "start daemon threads";
// coz current fe was not master fe and didn't get all fes' alive session report before, which cause
@@ -2657,6 +2670,18 @@ public long saveDictionaryManager(CountingDataOutputStream out, long checksum) t
return checksum;
}
+ public long loadLanceIndexJobManager(DataInputStream in, long checksum) throws IOException {
+ this.lanceIndexJobManager = LanceIndexJobManager.read(in);
+ LOG.info("finished replay lance index job manager from image");
+ return checksum;
+ }
+
+ public long saveLanceIndexJobManager(CountingDataOutputStream out, long checksum) throws IOException {
+ this.lanceIndexJobManager.write(out);
+ LOG.info("finished save lance index job manager to image");
+ return checksum;
+ }
+
// Only called by checkpoint thread
// return the latest image file's absolute path
public String saveImage() throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java
new file mode 100644
index 00000000000000..bed1ae696d8aa2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocator.java
@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+/**
+ * Dataset locator normalization v1 for the durable fence key. The rules, in
+ * order:
+ *
+ *
+ * - trim surrounding whitespace;
+ * - if a {@code scheme://} prefix is present, lowercase the scheme
+ * (aligned with the {@code LanceStorageProvider.schemeOf} precedent);
+ * - a URL whose authority carries userinfo is rejected: credential-bearing
+ * URLs are never identity;
+ * - a locator with a scheme but neither an authority nor a path (for
+ * example {@code "s3://"}) carries no identity and is rejected; an empty
+ * authority with a non-empty path ({@code "file:///x"}) stays legal;
+ * - trailing {@code '/'} characters are removed, keeping the root
+ * (a scheme-less {@code "/"} stays {@code "/"});
+ * - without a scheme the locator must be an absolute path (start with
+ * {@code '/'}), otherwise it is rejected.
+ *
+ *
+ * The authority (host/bucket) and path keep their original case: bucket and
+ * path components are case-sensitive on the providers Doris supports, and
+ * normalization v1 deliberately does not define cross-alias equivalence. URI
+ * aliases and external writers replacing the dataset at the same URI are
+ * outside Doris serialization.
+ */
+public final class LanceIndexDatasetLocator {
+ private static final String SCHEME_SEPARATOR = "://";
+ /** Finite bound for the stable locator persisted in every durable job record. */
+ public static final int MAX_LOCATOR_BYTES = 4096;
+
+ private LanceIndexDatasetLocator() {
+ }
+
+ /**
+ * Normalize a raw dataset locator into its durable identity form.
+ *
+ * @throws IllegalArgumentException if the locator is null/empty/oversized,
+ * is not a valid hierarchical URI or absolute path, carries
+ * userinfo/query/fragment data, has neither an authority nor a path,
+ * or is a scheme-less relative path
+ */
+ public static String normalize(String rawLocator) {
+ if (rawLocator == null) {
+ throw new IllegalArgumentException("dataset locator must not be null");
+ }
+ String locator = rawLocator.trim();
+ if (locator.isEmpty()) {
+ throw new IllegalArgumentException("dataset locator must not be empty");
+ }
+ if (locator.getBytes(StandardCharsets.UTF_8).length > MAX_LOCATOR_BYTES) {
+ throw new IllegalArgumentException(
+ "dataset locator exceeds " + MAX_LOCATOR_BYTES + " UTF-8 bytes");
+ }
+
+ // Preserve the established spelling of the scheme-less filesystem root.
+ // java.net.URI rejects "//" as a network-path reference without an
+ // authority, while it is a valid absolute filesystem path here.
+ if (containsOnlySlashes(locator)) {
+ return "/";
+ }
+
+ URI uri;
+ try {
+ uri = new URI(locator);
+ } catch (URISyntaxException e) {
+ // Do not include the raw locator: it may contain credentials.
+ throw new IllegalArgumentException("dataset locator is not a valid URI or absolute path");
+ }
+ if (uri.getRawUserInfo() != null) {
+ throw new IllegalArgumentException(
+ "credential-bearing dataset locators are never identity (userinfo is not allowed)");
+ }
+ if (uri.getRawQuery() != null) {
+ // Presigned object-store URLs and SAS URLs carry credentials here.
+ throw new IllegalArgumentException("dataset locator query parameters are not allowed");
+ }
+ if (uri.getRawFragment() != null) {
+ throw new IllegalArgumentException("dataset locator fragments are not allowed");
+ }
+
+ String scheme = uri.getScheme();
+ if (scheme == null) {
+ if (!locator.startsWith("/")) {
+ throw new IllegalArgumentException("dataset locator without a scheme must be an absolute path");
+ }
+ return stripTrailingSlashes(locator, 1);
+ }
+ if (uri.isOpaque() || !locator.regionMatches(scheme.length(), SCHEME_SEPARATOR, 0,
+ SCHEME_SEPARATOR.length())) {
+ throw new IllegalArgumentException("dataset locator scheme must use hierarchical '://' syntax");
+ }
+ String authority = uri.getRawAuthority() == null ? "" : uri.getRawAuthority();
+ String rawPath = uri.getRawPath() == null ? "" : uri.getRawPath();
+ String path = stripTrailingSlashes(rawPath, 0);
+ if (authority.isEmpty() && path.isEmpty()) {
+ // "s3://" / "file://" carry no identity at all; "file:///x" (empty
+ // authority, non-empty path) is legal and does not reach this.
+ throw new IllegalArgumentException("dataset locator has neither an authority nor a path");
+ }
+ return scheme.toLowerCase(Locale.ROOT) + SCHEME_SEPARATOR + authority + path;
+ }
+
+ private static boolean containsOnlySlashes(String value) {
+ for (int index = 0; index < value.length(); index++) {
+ if (value.charAt(index) != '/') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String stripTrailingSlashes(String value, int minLength) {
+ int end = value.length();
+ while (end > minLength && value.charAt(end - 1) == '/') {
+ end--;
+ }
+ return value.substring(0, end);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java
new file mode 100644
index 00000000000000..19cc9efe80ad45
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexFenceKey.java
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import java.util.Objects;
+
+/**
+ * The durable same-name target/fence key:
+ *
+ * ( persisted catalog identity, provider = DIRECTORY,
+ * normalized stable dataset locator, persisted normalized logical-index-name bytes )
+ *
+ * The display name is not part of the key; it is persisted on the job itself.
+ * This class is a derived in-memory index key and is not persisted directly.
+ */
+public final class LanceIndexFenceKey {
+ /** Provider of every job in this delivery slice, mapped from a filesystem (Directory) Lance catalog. */
+ public static final String PROVIDER_DIRECTORY = "DIRECTORY";
+
+ private final long catalogId;
+ private final String provider;
+ private final String normalizedLocator;
+ private final String normalizedIndexName;
+
+ public LanceIndexFenceKey(long catalogId, String provider, String normalizedLocator, String normalizedIndexName) {
+ this.catalogId = catalogId;
+ this.provider = Objects.requireNonNull(provider, "provider");
+ this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator");
+ this.normalizedIndexName = Objects.requireNonNull(normalizedIndexName, "normalizedIndexName");
+ }
+
+ public long getCatalogId() {
+ return catalogId;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ public String getNormalizedLocator() {
+ return normalizedLocator;
+ }
+
+ public String getNormalizedIndexName() {
+ return normalizedIndexName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof LanceIndexFenceKey)) {
+ return false;
+ }
+ LanceIndexFenceKey that = (LanceIndexFenceKey) o;
+ return catalogId == that.catalogId
+ && provider.equals(that.provider)
+ && normalizedLocator.equals(that.normalizedLocator)
+ && normalizedIndexName.equals(that.normalizedIndexName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(catalogId, provider, normalizedLocator, normalizedIndexName);
+ }
+
+ /**
+ * Deliberately omits the locator: fence-conflict messages may surface to
+ * users without target privileges and must not disclose it.
+ */
+ @Override
+ public String toString() {
+ return "LanceIndexFenceKey{catalogId=" + catalogId + ", provider=" + provider
+ + ", normalizedIndexName=" + normalizedIndexName + '}';
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java
new file mode 100644
index 00000000000000..59386acbcdb94c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJob.java
@@ -0,0 +1,695 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+/**
+ * The minimal durable Lance index job record. It contains only what the
+ * lifecycle invariants require: job identity/creator/revision and bounded
+ * timestamps; the persisted target identity and normalized same-name fence
+ * key; the mutation intent; the admitted dataset version and schema-contract
+ * representation; the mutation outcome, independent refresh state, typed
+ * result, and a bounded sanitized message; the dispatch identity (selected BE,
+ * BE process epoch, immutable invocation ID, deadline), possible-live
+ * ownership, and any matching termination proof; and the FORCE audit fields
+ * (populated only by the FORCE_RELEASE slice; here they are carried for
+ * durability and replay).
+ *
+ * The record never carries secrets, raw/opaque provider data, or unbounded
+ * values: name/message/note/properties fields are length-checked at
+ * construction/mutation time. Credentials are resolved at execution time from
+ * the current catalog and are never persisted here.
+ *
+ *
Serialization is the standard Gson stream: {@code Text.writeString} of
+ * the JSON form. Every durable field carries a short {@link SerializedName};
+ * fields missing in old data keep the safe in-class defaults, so replay stays
+ * tolerant of additive evolution. An instance is effectively immutable while
+ * published in the manager: every durable transition is applied to a private
+ * copy that is logged and then swapped in, so the object written to the edit
+ * log is never mutated afterwards.
+ */
+public class LanceIndexJob implements Writable {
+ /** Bound on the user/build properties JSON snapshot. */
+ public static final int MAX_PROPERTIES_JSON_BYTES = 4096;
+ /** Bound on the FORCE_RELEASE operator note and the late-commit warning text. */
+ public static final int MAX_FORCE_TEXT_BYTES = 1024;
+ /** Bound on persisted creator, target-name, mutation, and FORCE-actor text. */
+ public static final int MAX_DURABLE_TEXT_BYTES = 1024;
+ /** Dispatch identities are UUID-like tokens, not arbitrary worker output. */
+ public static final int MAX_INVOCATION_ID_BYTES = 256;
+
+ // ------------------------------------------------------------------
+ // Identity
+ // ------------------------------------------------------------------
+ @SerializedName(value = "jid")
+ private long jobId;
+
+ @SerializedName(value = "cr")
+ private String creator;
+
+ /** Bumped by +1 on every durable transition; callbacks and replay are revision-checked. */
+ @SerializedName(value = "rev")
+ private long revision;
+
+ @SerializedName(value = "ctm")
+ private long createTimeMs;
+
+ @SerializedName(value = "utm")
+ private long updateTimeMs;
+
+ // ------------------------------------------------------------------
+ // Target / fence identity
+ // ------------------------------------------------------------------
+ @SerializedName(value = "cid")
+ private long catalogId;
+
+ /** Local database name, kept for privilege and FORCE resolution when the catalog still resolves. */
+ @SerializedName(value = "dbn")
+ private String dbName;
+
+ /** Local table name, kept for privilege and FORCE resolution when the catalog still resolves. */
+ @SerializedName(value = "tbn")
+ private String tableName;
+
+ @SerializedName(value = "prv")
+ private String provider = LanceIndexFenceKey.PROVIDER_DIRECTORY;
+
+ @SerializedName(value = "loc")
+ private String normalizedLocator;
+
+ @SerializedName(value = "din")
+ private String displayIndexName;
+
+ @SerializedName(value = "nin")
+ private String normalizedIndexName;
+
+ // ------------------------------------------------------------------
+ // Mutation intent
+ // ------------------------------------------------------------------
+ @SerializedName(value = "mt")
+ private LanceIndexJobMutationType mutationType = LanceIndexJobMutationType.CREATE;
+
+ @SerializedName(value = "ine")
+ private boolean ifNotExists;
+
+ @SerializedName(value = "ie")
+ private boolean ifExists;
+
+ /** Logical Lance algorithm (IVF_PQ / BTREE / BITMAP); required for CREATE/REPLACE, nullable for DROP. */
+ @SerializedName(value = "it")
+ private String indexType;
+
+ @SerializedName(value = "cn")
+ private String columnName;
+
+ @SerializedName(value = "pj")
+ private String propertiesJson;
+
+ // ------------------------------------------------------------------
+ // Admission snapshot
+ // ------------------------------------------------------------------
+ @SerializedName(value = "adv")
+ private long admittedDatasetVersion;
+
+ /** Nullable only for DROP, which does not revalidate an indexed-field contract. */
+ @SerializedName(value = "sc")
+ private LanceIndexSchemaContract schemaContract;
+
+ // ------------------------------------------------------------------
+ // Dual state
+ // ------------------------------------------------------------------
+ @SerializedName(value = "ms")
+ private LanceIndexJobMutationState mutationState = LanceIndexJobMutationState.UNKNOWN;
+
+ /**
+ * Safe default for a corrupt record missing the key: REQUIRED holds the fence on a
+ * terminal record, mirroring the null fallback in {@link #isUnresolved()}. A legal
+ * record always carries the state explicitly (admission sets NOT_REQUIRED).
+ */
+ @SerializedName(value = "rs")
+ private LanceIndexJobRefreshState refreshState = LanceIndexJobRefreshState.REQUIRED;
+
+ // ------------------------------------------------------------------
+ // Result
+ // ------------------------------------------------------------------
+ @SerializedName(value = "res")
+ private LanceIndexJobResult result;
+
+ // ------------------------------------------------------------------
+ // Dispatch
+ // ------------------------------------------------------------------
+ @SerializedName(value = "bid")
+ private Long backendId;
+
+ @SerializedName(value = "bpe")
+ private Long beProcessEpoch;
+
+ /**
+ * Immutable revision established by PENDING -> RUNNING. Result and
+ * termination-proof callbacks use this dispatch identity rather than racing
+ * on the record's global revision. Null only before dispatch or in old data.
+ */
+ @SerializedName(value = "drv")
+ private Long dispatchRevision;
+
+ /** Immutable per-dispatch UUID; callbacks must present the matching identity. */
+ @SerializedName(value = "iid")
+ private String invocationId;
+
+ /** Bounds wait/runtime only; never proves termination and never releases a possible-live slot. */
+ @SerializedName(value = "dlm")
+ private Long deadlineMs;
+
+ @SerializedName(value = "plo")
+ private boolean possibleLiveOwned;
+
+ @SerializedName(value = "tp")
+ private LanceIndexTerminationProof terminationProof = LanceIndexTerminationProof.NONE;
+
+ // ------------------------------------------------------------------
+ // FORCE_RELEASE audit (populated only by the FORCE slice; durable + replayable here)
+ // ------------------------------------------------------------------
+ @SerializedName(value = "fr")
+ private boolean forceReleased;
+
+ @SerializedName(value = "fa")
+ private String forceActor;
+
+ @SerializedName(value = "ftm")
+ private Long forceTimeMs;
+
+ @SerializedName(value = "fn")
+ private String forceNote;
+
+ @SerializedName(value = "fw")
+ private String forceWarning;
+
+ /**
+ * No-arg constructor for Gson replay only; missing fields keep the safe
+ * defaults declared above (UNKNOWN mutation state holds the fence, never
+ * the redispatchable PENDING; REQUIRED refresh state owes a refresh rather
+ * than silently releasing the fence).
+ */
+ public LanceIndexJob() {
+ }
+
+ /**
+ * Admission constructor: identity, intent, and the admission snapshot. The
+ * manager initializes the lifecycle fields when the job is admitted.
+ */
+ public LanceIndexJob(long jobId, String creator, long catalogId, String dbName, String tableName,
+ String provider, String normalizedLocator, String displayIndexName, String normalizedIndexName,
+ LanceIndexJobMutationType mutationType, boolean ifNotExists, boolean ifExists, String indexType,
+ String columnName, String propertiesJson, long admittedDatasetVersion,
+ LanceIndexSchemaContract schemaContract) {
+ this.jobId = jobId;
+ this.creator = checkRequiredBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator");
+ this.catalogId = catalogId;
+ this.dbName = checkRequiredBytes(dbName, MAX_DURABLE_TEXT_BYTES, "dbName");
+ this.tableName = checkRequiredBytes(tableName, MAX_DURABLE_TEXT_BYTES, "tableName");
+ this.provider = Objects.requireNonNull(provider, "provider");
+ this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator");
+ setDisplayIndexName(displayIndexName);
+ setNormalizedIndexName(normalizedIndexName);
+ this.mutationType = Objects.requireNonNull(mutationType, "mutationType");
+ this.ifNotExists = ifNotExists;
+ this.ifExists = ifExists;
+ setIndexType(indexType);
+ setColumnName(columnName);
+ setPropertiesJson(propertiesJson);
+ this.admittedDatasetVersion = admittedDatasetVersion;
+ this.schemaContract = schemaContract;
+ validateForAdmission();
+ }
+
+ /**
+ * Copy used by the manager to stage a durable transition: the copy is
+ * mutated, written to the edit log, and then swapped in verbatim, so a
+ * published instance is never mutated after being logged. Every field is
+ * carried over; the {@code result} and {@code schemaContract} references
+ * are shared, which is safe because both are immutable values that are
+ * only ever replaced wholesale, never mutated in place.
+ */
+ public LanceIndexJob(LanceIndexJob other) {
+ this.jobId = other.jobId;
+ this.creator = other.creator;
+ this.revision = other.revision;
+ this.createTimeMs = other.createTimeMs;
+ this.updateTimeMs = other.updateTimeMs;
+ this.catalogId = other.catalogId;
+ this.dbName = other.dbName;
+ this.tableName = other.tableName;
+ this.provider = other.provider;
+ this.normalizedLocator = other.normalizedLocator;
+ this.displayIndexName = other.displayIndexName;
+ this.normalizedIndexName = other.normalizedIndexName;
+ this.mutationType = other.mutationType;
+ this.ifNotExists = other.ifNotExists;
+ this.ifExists = other.ifExists;
+ this.indexType = other.indexType;
+ this.columnName = other.columnName;
+ this.propertiesJson = other.propertiesJson;
+ this.admittedDatasetVersion = other.admittedDatasetVersion;
+ this.schemaContract = other.schemaContract;
+ this.mutationState = other.mutationState;
+ this.refreshState = other.refreshState;
+ this.result = other.result;
+ this.backendId = other.backendId;
+ this.beProcessEpoch = other.beProcessEpoch;
+ this.dispatchRevision = other.dispatchRevision;
+ this.invocationId = other.invocationId;
+ this.deadlineMs = other.deadlineMs;
+ this.possibleLiveOwned = other.possibleLiveOwned;
+ this.terminationProof = other.terminationProof;
+ this.forceReleased = other.forceReleased;
+ this.forceActor = other.forceActor;
+ this.forceTimeMs = other.forceTimeMs;
+ this.forceNote = other.forceNote;
+ this.forceWarning = other.forceWarning;
+ }
+
+ // ------------------------------------------------------------------
+ // Derived helpers
+ // ------------------------------------------------------------------
+
+ /**
+ * The durable same-name fence key of this job.
+ */
+ public LanceIndexFenceKey fenceKey() {
+ return new LanceIndexFenceKey(catalogId, provider, normalizedLocator, normalizedIndexName);
+ }
+
+ /**
+ * The per persisted table/locator quota identity.
+ */
+ public LanceIndexJobQuota.TableQuotaKey getTableQuotaKey() {
+ return new LanceIndexJobQuota.TableQuotaKey(catalogId, normalizedLocator);
+ }
+
+ /**
+ * Whether this job still holds its same-name fence and unresolved quota.
+ * Fence and quota are released together: PENDING/RUNNING always hold; a
+ * known terminal job holds until its required refresh is DONE (FAILED
+ * still holds, refresh may retry); UNKNOWN holds until a durable
+ * FORCE_RELEASE. A null state from a corrupt record is treated as UNKNOWN,
+ * the safe direction (fence retained, never redispatched).
+ */
+ public boolean isUnresolved() {
+ LanceIndexJobMutationState ms = mutationState == null ? LanceIndexJobMutationState.UNKNOWN : mutationState;
+ switch (ms) {
+ case PENDING:
+ case RUNNING:
+ return true;
+ case UNKNOWN:
+ return !forceReleased;
+ case COMMITTED:
+ case NOT_COMMITTED:
+ default:
+ LanceIndexJobRefreshState rs =
+ refreshState == null ? LanceIndexJobRefreshState.REQUIRED : refreshState;
+ return rs == LanceIndexJobRefreshState.REQUIRED
+ || rs == LanceIndexJobRefreshState.RUNNING
+ || rs == LanceIndexJobRefreshState.FAILED;
+ }
+ }
+
+ /**
+ * Whether this job still owns a possible-live worker slot: released only
+ * by a matching termination proof or a durable FORCE_RELEASE, never by a
+ * deadline.
+ */
+ public boolean holdsPossibleLiveSlot() {
+ return possibleLiveOwned
+ && (terminationProof == null || terminationProof == LanceIndexTerminationProof.NONE)
+ && !forceReleased;
+ }
+
+ /**
+ * Validate the invariant-bearing and bounded fields before this record is
+ * admitted. This is intentionally separate from Gson replay, which must
+ * remain tolerant of old or corrupt records in the safe direction.
+ */
+ public void validateForAdmission() {
+ checkRequiredBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator");
+ checkRequiredBytes(dbName, MAX_DURABLE_TEXT_BYTES, "dbName");
+ checkRequiredBytes(tableName, MAX_DURABLE_TEXT_BYTES, "tableName");
+ if (provider == null) {
+ throw new IllegalArgumentException("lance index job provider must not be null");
+ }
+ if (!LanceIndexFenceKey.PROVIDER_DIRECTORY.equals(provider)) {
+ throw new IllegalArgumentException("lance index job provider must be DIRECTORY");
+ }
+ if (normalizedLocator == null) {
+ throw new IllegalArgumentException("normalized dataset locator must not be null");
+ }
+ String canonicalLocator = LanceIndexDatasetLocator.normalize(normalizedLocator);
+ if (!canonicalLocator.equals(normalizedLocator)) {
+ throw new IllegalArgumentException("dataset locator is not in canonical identity form");
+ }
+ LanceIndexNameNormalizer.validateDisplayName(displayIndexName);
+ validateNormalizedIndexName(normalizedIndexName);
+ String expectedNormalizedName = LanceIndexNameNormalizer.normalize(displayIndexName);
+ if (!expectedNormalizedName.equals(normalizedIndexName)) {
+ throw new IllegalArgumentException("normalized index name does not match the display name");
+ }
+ if (mutationType == null) {
+ throw new IllegalArgumentException("mutation type must not be null");
+ }
+ checkBytes(indexType, MAX_DURABLE_TEXT_BYTES, "indexType");
+ checkBytes(columnName, MAX_DURABLE_TEXT_BYTES, "columnName");
+ checkBytes(propertiesJson, MAX_PROPERTIES_JSON_BYTES, "propertiesJson");
+ checkBytes(invocationId, MAX_INVOCATION_ID_BYTES, "invocationId");
+ checkBytes(forceActor, MAX_DURABLE_TEXT_BYTES, "forceActor");
+ checkBytes(forceNote, MAX_FORCE_TEXT_BYTES, "forceNote");
+ checkBytes(forceWarning, MAX_FORCE_TEXT_BYTES, "forceWarning");
+ if (result != null) {
+ checkBytes(result.getSanitizedMessage(), LanceIndexJobResult.MAX_MESSAGE_BYTES, "sanitizedMessage");
+ }
+ if (schemaContract != null) {
+ schemaContract.validateForAdmission();
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Accessors. Setters for bounded text fields re-validate the bound.
+ // ------------------------------------------------------------------
+
+ public long getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(long jobId) {
+ this.jobId = jobId;
+ }
+
+ public String getCreator() {
+ return creator;
+ }
+
+ public void setCreator(String creator) {
+ this.creator = checkBytes(creator, MAX_DURABLE_TEXT_BYTES, "creator");
+ }
+
+ public long getRevision() {
+ return revision;
+ }
+
+ public void setRevision(long revision) {
+ this.revision = revision;
+ }
+
+ public long getCreateTimeMs() {
+ return createTimeMs;
+ }
+
+ public void setCreateTimeMs(long createTimeMs) {
+ this.createTimeMs = createTimeMs;
+ }
+
+ public long getUpdateTimeMs() {
+ return updateTimeMs;
+ }
+
+ public void setUpdateTimeMs(long updateTimeMs) {
+ this.updateTimeMs = updateTimeMs;
+ }
+
+ public long getCatalogId() {
+ return catalogId;
+ }
+
+ public String getDbName() {
+ return dbName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ public String getNormalizedLocator() {
+ return normalizedLocator;
+ }
+
+ public String getDisplayIndexName() {
+ return displayIndexName;
+ }
+
+ public final void setDisplayIndexName(String displayIndexName) {
+ LanceIndexNameNormalizer.validateDisplayName(displayIndexName);
+ this.displayIndexName = displayIndexName;
+ }
+
+ public String getNormalizedIndexName() {
+ return normalizedIndexName;
+ }
+
+ public final void setNormalizedIndexName(String normalizedIndexName) {
+ validateNormalizedIndexName(normalizedIndexName);
+ this.normalizedIndexName = normalizedIndexName;
+ }
+
+ private static void validateNormalizedIndexName(String normalizedIndexName) {
+ if (normalizedIndexName == null || normalizedIndexName.isEmpty()) {
+ throw new IllegalArgumentException("normalized index name must not be null or empty");
+ }
+ if (normalizedIndexName.getBytes(StandardCharsets.UTF_8).length
+ > LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES) {
+ throw new IllegalArgumentException(
+ "normalized index name exceeds " + LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + " UTF-8 bytes");
+ }
+ }
+
+ public LanceIndexJobMutationType getMutationType() {
+ return mutationType;
+ }
+
+ public boolean isIfNotExists() {
+ return ifNotExists;
+ }
+
+ public boolean isIfExists() {
+ return ifExists;
+ }
+
+ public String getIndexType() {
+ return indexType;
+ }
+
+ public void setIndexType(String indexType) {
+ this.indexType = checkBytes(indexType, MAX_DURABLE_TEXT_BYTES, "indexType");
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public void setColumnName(String columnName) {
+ this.columnName = checkBytes(columnName, MAX_DURABLE_TEXT_BYTES, "columnName");
+ }
+
+ public String getPropertiesJson() {
+ return propertiesJson;
+ }
+
+ public final void setPropertiesJson(String propertiesJson) {
+ this.propertiesJson = checkBytes(propertiesJson, MAX_PROPERTIES_JSON_BYTES, "propertiesJson");
+ }
+
+ public long getAdmittedDatasetVersion() {
+ return admittedDatasetVersion;
+ }
+
+ public LanceIndexSchemaContract getSchemaContract() {
+ return schemaContract;
+ }
+
+ public LanceIndexJobMutationState getMutationState() {
+ return mutationState;
+ }
+
+ public void setMutationState(LanceIndexJobMutationState mutationState) {
+ this.mutationState = Objects.requireNonNull(mutationState, "mutationState");
+ }
+
+ public LanceIndexJobRefreshState getRefreshState() {
+ return refreshState;
+ }
+
+ public void setRefreshState(LanceIndexJobRefreshState refreshState) {
+ this.refreshState = Objects.requireNonNull(refreshState, "refreshState");
+ }
+
+ public LanceIndexJobResult getResult() {
+ return result;
+ }
+
+ public void setResult(LanceIndexJobResult result) {
+ this.result = result;
+ }
+
+ public Long getBackendId() {
+ return backendId;
+ }
+
+ public void setBackendId(Long backendId) {
+ this.backendId = backendId;
+ }
+
+ public Long getBeProcessEpoch() {
+ return beProcessEpoch;
+ }
+
+ public void setBeProcessEpoch(Long beProcessEpoch) {
+ this.beProcessEpoch = beProcessEpoch;
+ }
+
+ public Long getDispatchRevision() {
+ return dispatchRevision;
+ }
+
+ public void setDispatchRevision(Long dispatchRevision) {
+ this.dispatchRevision = dispatchRevision;
+ }
+
+ public String getInvocationId() {
+ return invocationId;
+ }
+
+ public void setInvocationId(String invocationId) {
+ this.invocationId = checkBytes(invocationId, MAX_INVOCATION_ID_BYTES, "invocationId");
+ }
+
+ public Long getDeadlineMs() {
+ return deadlineMs;
+ }
+
+ public void setDeadlineMs(Long deadlineMs) {
+ this.deadlineMs = deadlineMs;
+ }
+
+ public boolean isPossibleLiveOwned() {
+ return possibleLiveOwned;
+ }
+
+ public void setPossibleLiveOwned(boolean possibleLiveOwned) {
+ this.possibleLiveOwned = possibleLiveOwned;
+ }
+
+ public LanceIndexTerminationProof getTerminationProof() {
+ return terminationProof;
+ }
+
+ public void setTerminationProof(LanceIndexTerminationProof terminationProof) {
+ this.terminationProof = Objects.requireNonNull(terminationProof, "terminationProof");
+ }
+
+ public boolean isForceReleased() {
+ return forceReleased;
+ }
+
+ public void setForceReleased(boolean forceReleased) {
+ this.forceReleased = forceReleased;
+ }
+
+ public String getForceActor() {
+ return forceActor;
+ }
+
+ public void setForceActor(String forceActor) {
+ this.forceActor = checkBytes(forceActor, MAX_DURABLE_TEXT_BYTES, "forceActor");
+ }
+
+ public Long getForceTimeMs() {
+ return forceTimeMs;
+ }
+
+ public void setForceTimeMs(Long forceTimeMs) {
+ this.forceTimeMs = forceTimeMs;
+ }
+
+ public String getForceNote() {
+ return forceNote;
+ }
+
+ public void setForceNote(String forceNote) {
+ this.forceNote = checkBytes(forceNote, MAX_FORCE_TEXT_BYTES, "forceNote");
+ }
+
+ public String getForceWarning() {
+ return forceWarning;
+ }
+
+ public void setForceWarning(String forceWarning) {
+ this.forceWarning = checkBytes(forceWarning, MAX_FORCE_TEXT_BYTES, "forceWarning");
+ }
+
+ private static String checkBytes(String value, int maxBytes, String fieldName) {
+ if (value != null && value.getBytes(StandardCharsets.UTF_8).length > maxBytes) {
+ throw new IllegalArgumentException(fieldName + " exceeds " + maxBytes + " UTF-8 bytes");
+ }
+ return value;
+ }
+
+ private static String checkRequiredBytes(String value, int maxBytes, String fieldName) {
+ if (value == null || value.isEmpty()) {
+ throw new IllegalArgumentException(fieldName + " must not be null or empty");
+ }
+ return checkBytes(value, maxBytes, fieldName);
+ }
+
+ // ------------------------------------------------------------------
+ // Serialization (Gson stream style, see DropIndexPolicyLog)
+ // ------------------------------------------------------------------
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Text.writeString(out, GsonUtils.GSON.toJson(this));
+ }
+
+ public static LanceIndexJob read(DataInput in) throws IOException {
+ return GsonUtils.GSON.fromJson(Text.readString(in), LanceIndexJob.class);
+ }
+
+ /**
+ * Deliberately omits the locator: job lookups must not disclose target
+ * details to callers without privilege, and log lines reuse this form.
+ */
+ @Override
+ public String toString() {
+ return "LanceIndexJob{jobId=" + jobId + ", revision=" + revision + ", catalogId=" + catalogId
+ + ", db=" + dbName + ", table=" + tableName + ", index=" + displayIndexName
+ + ", mutationType=" + mutationType + ", mutationState=" + mutationState
+ + ", refreshState=" + refreshState + ", possibleLiveSlot=" + holdsPossibleLiveSlot()
+ + ", forceReleased=" + forceReleased + '}';
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java
new file mode 100644
index 00000000000000..94bfcd320538d3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobCompletionReason.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+/**
+ * Why a job reached its terminal state. NOOP is not a state: a typed
+ * post-dispatch DROP-not-found for DROP IF EXISTS is NOT_COMMITTED with
+ * completion reason IF_CONDITION_NOOP.
+ */
+public enum LanceIndexJobCompletionReason {
+ NONE,
+ IF_CONDITION_NOOP
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
new file mode 100644
index 00000000000000..8ece27bb62f338
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobManager.java
@@ -0,0 +1,694 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonPostProcessable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Objects;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Master-owned manager of the durable Lance index job records, the same-name
+ * fences, and the three-level unresolved-job quotas. It deliberately reuses
+ * neither the generic JobManager scheduling framework nor the internal
+ * IndexChangeJob machinery: the external one-shot CAS, no-redispatch rule,
+ * same-name fence, and possible-live ownership required here are not provided
+ * by either.
+ *
+ *
Every FE keeps the same in-memory image of {@link #jobs}: the master
+ * writes each durable transition to the edit log and then applies the same
+ * record locally; followers apply it from replay. All transitions share one
+ * write-path shape:
+ *
+ * writeLock -> validate (state, revision CAS, callback identity)
+ * -> writeEditLog(updated copy) // fails only by System.exit
+ * -> applyToMemory(updated copy) // verbatim swap + fence/quota accounting
+ *
+ * A published {@link LanceIndexJob} is never mutated; a transition stages a
+ * copy, logs it, and swaps it in. {@link #replayUpsertJob(LanceIndexJob)} is a
+ * verbatim replace with a monotonic-revision guard and performs no state
+ * transformation at all: a follower tailing a live master must keep a fresh
+ * RUNNING record RUNNING. The only place a durable RUNNING without a complete
+ * matching terminal result becomes UNKNOWN is {@link #onTransferToMaster()},
+ * the master-election sweep that runs after metadata replay and before any
+ * dispatcher could start.
+ *
+ * Fence and quota live and die together ({@link LanceIndexJob#isUnresolved()}):
+ * released when a known terminal job's refresh is NOT_REQUIRED or DONE, or by
+ * a durable FORCE_RELEASE; an unresolved UNKNOWN holds both across failover,
+ * timeout, termination proof, and metadata observations.
+ *
+ *
The class starts no threads, so no checkpoint-thread guard is needed in
+ * the constructor; the derived fence index and quota counters are rebuilt in
+ * {@link #gsonPostProcess()} after image load.
+ */
+public class LanceIndexJobManager implements Writable, GsonPostProcessable {
+ private static final Logger LOG = LogManager.getLogger(LanceIndexJobManager.class);
+
+ @SerializedName(value = "jobs")
+ private ConcurrentMap jobs = Maps.newConcurrentMap();
+
+ /**
+ * Derived: fence key -> all job ids holding that fence. Legal admission creates
+ * exactly one owner; retaining every owner for a corrupt collision keeps the
+ * fence fail-closed when one of those jobs later settles. Identity-less corrupt
+ * records are kept out of the books. Rebuilt after replay/image load.
+ */
+ private final Map> fenceIndex = Maps.newHashMap();
+
+ /** Derived: three-level unresolved counters, rebuilt from the unresolved jobs. */
+ private final LanceIndexJobQuota quota = new LanceIndexJobQuota();
+
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
+
+ public LanceIndexJobManager() {
+ }
+
+ private void readLock() {
+ lock.readLock().lock();
+ }
+
+ private void readUnlock() {
+ lock.readLock().unlock();
+ }
+
+ private void writeLock() {
+ lock.writeLock().lock();
+ }
+
+ private void writeUnlock() {
+ lock.writeLock().unlock();
+ }
+
+ /**
+ * Edit-log seam: the only place this manager writes the journal. Tests
+ * subclass and override to capture or swallow the record.
+ */
+ protected void writeEditLog(LanceIndexJob job) {
+ Env.getCurrentEnv().getEditLog().logLanceIndexJob(job);
+ }
+
+ // ------------------------------------------------------------------
+ // Master-only write paths
+ // ------------------------------------------------------------------
+
+ /**
+ * Admit a new job: same-name fence CAS and unresolved-quota admission
+ * happen inside the write lock, before anything is logged. A rejection
+ * leaves no job, no fence, no quota charge, and no edit-log record.
+ *
+ * @throws DdlException on a fence conflict, a duplicate job id, or quota overload
+ */
+ public void createJob(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit)
+ throws DdlException {
+ Objects.requireNonNull(job, "job");
+ writeLock();
+ try {
+ try {
+ job.validateForAdmission();
+ } catch (IllegalArgumentException e) {
+ throw new DdlException("invalid lance index job: " + e.getMessage(), e);
+ }
+ if (tableLimit <= 0 || catalogLimit <= 0 || globalLimit <= 0) {
+ throw new DdlException("lance index job quota limits must all be positive");
+ }
+ if (jobs.containsKey(job.getJobId())) {
+ throw new DdlException("lance index job id already exists: " + job.getJobId());
+ }
+ NavigableSet fencingJobIds = fenceIndex.get(job.fenceKey());
+ if (fencingJobIds != null && !fencingJobIds.isEmpty()) {
+ // Never disclose the locator in the rejection (the caller may lack target privilege).
+ throw new DdlException("lance index '" + job.getDisplayIndexName()
+ + "' is fenced by unresolved job " + fencingJobIds.first()
+ + "; resolve that job (FORCE_RELEASE) before reusing the name");
+ }
+ // Pure admission check; the charge itself happens in applyToMemory together with the fence,
+ // after the record is durable. An edit-log write failure exits the process, so no rollback exists.
+ if (!quota.hasCapacity(job, tableLimit, catalogLimit, globalLimit)) {
+ throw new DdlException("unresolved lance index job quota exceeded for index '"
+ + job.getDisplayIndexName() + "'; resolve or finish existing jobs first");
+ }
+ long now = System.currentTimeMillis();
+ // Stage a private copy and reset every lifecycle field: the caller's object is
+ // never published, and admission always starts from the same PENDING record no
+ // matter what the caller left in the lifecycle fields.
+ LanceIndexJob admitted = new LanceIndexJob(job);
+ admitted.setMutationState(LanceIndexJobMutationState.PENDING);
+ admitted.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
+ admitted.setRevision(0);
+ admitted.setCreateTimeMs(now);
+ admitted.setUpdateTimeMs(now);
+ admitted.setResult(null);
+ admitted.setBackendId(null);
+ admitted.setBeProcessEpoch(null);
+ admitted.setInvocationId(null);
+ admitted.setDispatchRevision(null);
+ admitted.setDeadlineMs(null);
+ admitted.setPossibleLiveOwned(false);
+ admitted.setTerminationProof(LanceIndexTerminationProof.NONE);
+ admitted.setForceReleased(false);
+ admitted.setForceActor(null);
+ admitted.setForceTimeMs(null);
+ admitted.setForceNote(null);
+ admitted.setForceWarning(null);
+ writeEditLog(admitted);
+ applyToMemory(admitted);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * PENDING -> RUNNING, the durable dispatch boundary. Compare-and-set on
+ * (jobId, revision): only a PENDING job at the expected revision may be
+ * dispatched, which is what makes redispatch after replay impossible.
+ * Records the dispatch identity (backend, BE process epoch, immutable
+ * invocation id, deadline) and takes the possible-live slot.
+ *
+ * @return false (with a warning) on any mismatch; the caller must not send
+ */
+ public boolean markRunning(long jobId, long expectedRevision, long backendId, long beProcessEpoch,
+ String invocationId, long deadlineMs) {
+ if (StringUtils.isBlank(invocationId)) {
+ // A null/blank invocation identity would silently match a null field under
+ // Objects.equals in completeWithResult and defeat the stale-callback guard.
+ LOG.warn("reject markRunning for lance index job {}: invocation id is null or blank", jobId);
+ return false;
+ }
+ writeLock();
+ try {
+ LanceIndexJob current = jobs.get(jobId);
+ if (current == null || current.getRevision() != expectedRevision
+ || current.getMutationState() != LanceIndexJobMutationState.PENDING) {
+ LOG.warn("reject markRunning for lance index job {}: expected revision {}, current {}",
+ jobId, expectedRevision, current);
+ return false;
+ }
+ try {
+ current.validateForAdmission();
+ } catch (IllegalArgumentException e) {
+ LOG.warn("reject markRunning for invalid lance index job {}: {}", jobId, e.getMessage());
+ return false;
+ }
+ if (!hasFenceIdentity(current)) {
+ LOG.warn("reject markRunning for lance index job {} without a valid fence identity", jobId);
+ return false;
+ }
+ LanceIndexJob updated = new LanceIndexJob(current);
+ updated.setMutationState(LanceIndexJobMutationState.RUNNING);
+ updated.setBackendId(backendId);
+ updated.setBeProcessEpoch(beProcessEpoch);
+ updated.setInvocationId(invocationId);
+ updated.setDeadlineMs(deadlineMs);
+ updated.setPossibleLiveOwned(true);
+ long dispatchRevision = current.getRevision() + 1;
+ updated.setRevision(dispatchRevision);
+ updated.setDispatchRevision(dispatchRevision);
+ updated.setUpdateTimeMs(System.currentTimeMillis());
+ writeEditLog(updated);
+ applyToMemory(updated);
+ return true;
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * RUNNING -> terminal, from a worker/supervisor result. A callback must
+ * match the durable dispatch identity exactly (immutable dispatch revision, invocation
+ * id, and BE process epoch); a stale callback only logs a warning and
+ * changes nothing. The typed result is classified into (mutation state,
+ * refresh obligation, completion reason); message text is never inspected.
+ * A known terminal job whose refresh is NOT_REQUIRED releases fence and
+ * quota immediately; an UNKNOWN keeps both until a durable FORCE_RELEASE.
+ *
+ * This is also the channel for marking a job UNKNOWN on an ambiguous
+ * result (result code NO_TRUSTED_RESULT), including the master-transfer
+ * sweep; no separate markUnknown API exists.
+ *
+ * @return false (with a warning) when the callback is stale or the job is not RUNNING
+ */
+ public boolean completeWithResult(long jobId, long expectedDispatchRevision, String invocationId,
+ Long beProcessEpoch,
+ LanceIndexJobResult result) {
+ Objects.requireNonNull(result, "result");
+ writeLock();
+ try {
+ LanceIndexJob current = jobs.get(jobId);
+ if (current == null || dispatchRevisionOf(current) != expectedDispatchRevision) {
+ LOG.warn("reject stale lance index job callback for job {}: expected dispatch revision {}, current {}",
+ jobId, expectedDispatchRevision, current);
+ return false;
+ }
+ if (current.getMutationState() != LanceIndexJobMutationState.RUNNING) {
+ LOG.warn("reject lance index job callback for job {} in state {}: only RUNNING accepts a result",
+ jobId, current.getMutationState());
+ return false;
+ }
+ if (!Objects.equals(current.getInvocationId(), invocationId)
+ || !Objects.equals(current.getBeProcessEpoch(), beProcessEpoch)) {
+ LOG.warn("reject stale lance index job callback for job {}: invocation/epoch mismatch, current {}",
+ jobId, current);
+ return false;
+ }
+ LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify(
+ current.getMutationType(), result.getResultCode(), current.isIfExists(),
+ result.isExternalMetadataAdvanced());
+ LanceIndexJob updated = new LanceIndexJob(current);
+ if (updated.getDispatchRevision() == null) {
+ // Backfill old RUNNING records before the global revision advances so
+ // an independent termination proof can still identify this dispatch.
+ updated.setDispatchRevision(expectedDispatchRevision);
+ }
+ updated.setMutationState(classification.getMutationState());
+ updated.setRefreshState(classification.getRefreshState());
+ updated.setResult(new LanceIndexJobResult(result.getResultCode(), classification.getCompletionReason(),
+ result.getSanitizedMessage(), result.isExternalMetadataAdvanced()));
+ updated.setRevision(current.getRevision() + 1);
+ updated.setUpdateTimeMs(System.currentTimeMillis());
+ writeEditLog(updated);
+ applyToMemory(updated);
+ return true;
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * Refresh REQUIRED -> RUNNING, or FAILED -> RUNNING for a retry
+ * through the idempotent external-table refresh path.
+ */
+ public boolean markRefreshRunning(long jobId, long expectedRevision) {
+ return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.RUNNING);
+ }
+
+ /**
+ * Refresh RUNNING -> DONE. On a known terminal job this releases the
+ * same-name fence and the unresolved quota.
+ */
+ public boolean markRefreshDone(long jobId, long expectedRevision) {
+ return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.DONE);
+ }
+
+ /**
+ * Refresh RUNNING -> FAILED. The job keeps fence and quota; a later
+ * {@link #markRefreshRunning} retries through the idempotent path.
+ */
+ public boolean markRefreshFailed(long jobId, long expectedRevision) {
+ return transitionRefresh(jobId, expectedRevision, LanceIndexJobRefreshState.FAILED);
+ }
+
+ private boolean transitionRefresh(long jobId, long expectedRevision, LanceIndexJobRefreshState target) {
+ writeLock();
+ try {
+ LanceIndexJob current = jobs.get(jobId);
+ if (current == null || current.getRevision() != expectedRevision) {
+ LOG.warn("reject refresh transition to {} for lance index job {}: expected revision {}, current {}",
+ target, jobId, expectedRevision, current);
+ return false;
+ }
+ if (current.getMutationState() == null || !current.getMutationState().isTerminal()) {
+ LOG.warn("reject refresh transition to {} for non-terminal lance index job {} in mutation state {}",
+ target, jobId, current.getMutationState());
+ return false;
+ }
+ LanceIndexJobRefreshState from = current.getRefreshState();
+ boolean legal = (target == LanceIndexJobRefreshState.RUNNING
+ && (from == LanceIndexJobRefreshState.REQUIRED || from == LanceIndexJobRefreshState.FAILED))
+ || (target == LanceIndexJobRefreshState.DONE && from == LanceIndexJobRefreshState.RUNNING)
+ || (target == LanceIndexJobRefreshState.FAILED && from == LanceIndexJobRefreshState.RUNNING);
+ if (!legal) {
+ LOG.warn("reject illegal refresh transition {} -> {} for lance index job {}",
+ from, target, jobId);
+ return false;
+ }
+ LanceIndexJob updated = new LanceIndexJob(current);
+ updated.setRefreshState(target);
+ updated.setRevision(current.getRevision() + 1);
+ updated.setUpdateTimeMs(System.currentTimeMillis());
+ writeEditLog(updated);
+ applyToMemory(updated);
+ return true;
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * Record a matching termination proof for a job that still owns a
+ * possible-live slot. Backend, BE process epoch, invocation id, and immutable
+ * dispatch revision must all match. This releases only the slot: it never
+ * changes the mutation state and never releases the fence or quota.
+ */
+ public boolean recordTerminationProof(long jobId, long expectedDispatchRevision, long backendId,
+ long beProcessEpoch, String invocationId, LanceIndexTerminationProof proof) {
+ Objects.requireNonNull(proof, "proof");
+ writeLock();
+ try {
+ LanceIndexJob current = jobs.get(jobId);
+ if (current == null || dispatchRevisionOf(current) != expectedDispatchRevision) {
+ LOG.warn("reject termination proof for lance index job {}: expected dispatch revision {}, current {}",
+ jobId, expectedDispatchRevision, current);
+ return false;
+ }
+ if (StringUtils.isBlank(invocationId)
+ || !Objects.equals(current.getBackendId(), backendId)
+ || !Objects.equals(current.getBeProcessEpoch(), beProcessEpoch)
+ || !Objects.equals(current.getInvocationId(), invocationId)) {
+ LOG.warn("reject stale termination proof for lance index job {}:"
+ + " dispatch identity mismatch, current {}", jobId, current);
+ return false;
+ }
+ if (proof == LanceIndexTerminationProof.NONE || !current.isPossibleLiveOwned()
+ || current.getTerminationProof() != LanceIndexTerminationProof.NONE) {
+ LOG.warn("reject termination proof {} for lance index job {}: no possible-live slot owned, current {}",
+ proof, jobId, current);
+ return false;
+ }
+ LanceIndexJob updated = new LanceIndexJob(current);
+ if (updated.getDispatchRevision() == null) {
+ updated.setDispatchRevision(expectedDispatchRevision);
+ }
+ updated.setTerminationProof(proof);
+ updated.setPossibleLiveOwned(false);
+ updated.setRevision(current.getRevision() + 1);
+ updated.setUpdateTimeMs(System.currentTimeMillis());
+ writeEditLog(updated);
+ applyToMemory(updated);
+ return true;
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * Master-election sweep, hooked from {@code Env.transferToMaster()} after
+ * metadata replay and before any master-only dispatcher could start. A
+ * durable RUNNING at this point means the terminal result may have been
+ * lost with the old master: the job becomes UNKNOWN through the same
+ * completeWithResult channel (result code NO_TRUSTED_RESULT, fence/quota/
+ * possible-live ownership retained, never redispatched), and an in-flight
+ * refresh is downgraded to REQUIRED so the idempotent refresh retries.
+ * Both transitions are written to the edit log so followers converge.
+ */
+ public void onTransferToMaster() {
+ List snapshot;
+ readLock();
+ try {
+ snapshot = new ArrayList<>(jobs.values());
+ } finally {
+ readUnlock();
+ }
+ for (LanceIndexJob job : snapshot) {
+ if (job.getMutationState() == LanceIndexJobMutationState.RUNNING) {
+ boolean completed = completeWithResult(job.getJobId(), dispatchRevisionOf(job), job.getInvocationId(),
+ job.getBeProcessEpoch(),
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE,
+ "FE master transferred while the job was RUNNING; the result is not trusted", false));
+ if (completed) {
+ LOG.info("lance index job {} transitioned RUNNING -> UNKNOWN on master transfer",
+ job.getJobId());
+ }
+ }
+ if (job.getRefreshState() == LanceIndexJobRefreshState.RUNNING) {
+ downgradeRunningRefresh(job);
+ }
+ }
+ }
+
+ private void downgradeRunningRefresh(LanceIndexJob sweepCandidate) {
+ writeLock();
+ try {
+ LanceIndexJob current = jobs.get(sweepCandidate.getJobId());
+ if (current == null || current.getRefreshState() != LanceIndexJobRefreshState.RUNNING) {
+ return;
+ }
+ LanceIndexJob updated = new LanceIndexJob(current);
+ updated.setRefreshState(LanceIndexJobRefreshState.REQUIRED);
+ updated.setRevision(current.getRevision() + 1);
+ updated.setUpdateTimeMs(System.currentTimeMillis());
+ writeEditLog(updated);
+ applyToMemory(updated);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Replay (all FEs)
+ // ------------------------------------------------------------------
+
+ /**
+ * Apply one durable record from the journal: a verbatim replace of the
+ * in-memory record plus derived-index accounting, with a monotonic
+ * revision guard so a stale record can never roll the state back. No state
+ * transformation happens here on purpose: a follower tailing a live master
+ * must keep a fresh RUNNING record RUNNING. Idempotent and tolerant of
+ * missing fields (Gson defaults); never throws on behalf of record content.
+ */
+ public void replayUpsertJob(LanceIndexJob job) {
+ if (job == null) {
+ LOG.warn("ignore null lance index job record");
+ return;
+ }
+ writeLock();
+ try {
+ LanceIndexJob replayed = new LanceIndexJob(job);
+ LanceIndexJob existing = jobs.get(replayed.getJobId());
+ if (existing != null && replayed.getRevision() < existing.getRevision()) {
+ LOG.warn("ignore stale lance index job record for job {}: replayed revision {} < current {}",
+ replayed.getJobId(), replayed.getRevision(), existing.getRevision());
+ return;
+ }
+ applyToMemory(replayed);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * Swap a staged record into memory and settle fence/quota accounting for
+ * the replaced record. Fence and quota always move together: a record
+ * holds both while {@link LanceIndexJob#isUnresolved()}, provided it
+ * carries fence identity (a corrupt identity-less record stays queryable
+ * but out of the books on both the charge and the release side). A private
+ * copy is always stored so neither a replay input nor an edit-log seam
+ * reference can mutate the published record. Caller holds the write lock.
+ */
+ private void applyToMemory(LanceIndexJob job) {
+ LanceIndexJob stored = new LanceIndexJob(job);
+ LanceIndexJob old = jobs.put(stored.getJobId(), stored);
+ // Release only what the identity guard below booked: an identity-less corrupt
+ // record was stored without fence/quota accounting, so keying on it would throw.
+ if (old != null && old.isUnresolved() && hasFenceIdentity(old)) {
+ removeFenceOwner(old);
+ quota.release(old);
+ }
+ if (stored.isUnresolved()) {
+ if (hasFenceIdentity(stored)) {
+ NavigableSet owners = fenceIndex.computeIfAbsent(stored.fenceKey(), ignored -> new TreeSet<>());
+ if (!owners.isEmpty() && !owners.contains(stored.getJobId())) {
+ LOG.warn("fence key collision between unresolved lance index jobs {} and {};"
+ + " keeping fence owner {}", owners.first(), stored.getJobId(),
+ Math.min(owners.first(), stored.getJobId()));
+ }
+ owners.add(stored.getJobId());
+ quota.charge(stored);
+ } else {
+ // Corrupt record tolerance: keep it queryable but out of the fence/quota books.
+ LOG.warn("lance index job {} lacks fence identity (provider/locator/name);"
+ + " stored without fence/quota accounting", stored.getJobId());
+ }
+ }
+ }
+
+ private void removeFenceOwner(LanceIndexJob job) {
+ LanceIndexFenceKey fenceKey = job.fenceKey();
+ NavigableSet owners = fenceIndex.get(fenceKey);
+ if (owners == null) {
+ return;
+ }
+ owners.remove(job.getJobId());
+ if (owners.isEmpty()) {
+ fenceIndex.remove(fenceKey);
+ }
+ }
+
+ private static long dispatchRevisionOf(LanceIndexJob job) {
+ return job.getDispatchRevision() == null ? job.getRevision() : job.getDispatchRevision();
+ }
+
+ private static boolean hasFenceIdentity(LanceIndexJob job) {
+ return job.getProvider() != null && job.getNormalizedLocator() != null
+ && job.getNormalizedIndexName() != null;
+ }
+
+ // ------------------------------------------------------------------
+ // Queries
+ // ------------------------------------------------------------------
+
+ public LanceIndexJob getJob(long jobId) {
+ readLock();
+ try {
+ LanceIndexJob job = jobs.get(jobId);
+ return job == null ? null : new LanceIndexJob(job);
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * All jobs still holding a fence and unresolved quota: PENDING/RUNNING,
+ * unforced UNKNOWN, and known terminal jobs with unfinished refresh.
+ */
+ public List getUnresolvedJobs() {
+ readLock();
+ try {
+ List result = new ArrayList<>();
+ for (LanceIndexJob job : jobs.values()) {
+ if (job != null && job.isUnresolved() && hasFenceIdentity(job)) {
+ result.add(new LanceIndexJob(job));
+ }
+ }
+ return result;
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * Terminal jobs the refresh driver must pick up: refresh REQUIRED (waiting to
+ * run, including jobs downgraded by the master-transfer sweep) or FAILED
+ * (waiting for a retry). Both resume through the idempotent refresh path via
+ * {@link #markRefreshRunning}; a FAILED job invisible here would hold its
+ * fence forever with no retry channel.
+ */
+ public List getJobsNeedingRefresh() {
+ readLock();
+ try {
+ List result = new ArrayList<>();
+ for (LanceIndexJob job : jobs.values()) {
+ if (job != null && job.getMutationState() != null && job.getMutationState().isTerminal()
+ && hasFenceIdentity(job)
+ && (job.getRefreshState() == LanceIndexJobRefreshState.REQUIRED
+ || job.getRefreshState() == LanceIndexJobRefreshState.FAILED)) {
+ result.add(new LanceIndexJob(job));
+ }
+ }
+ return result;
+ } finally {
+ readUnlock();
+ }
+ }
+
+ public boolean isFenceHeld(LanceIndexFenceKey fenceKey) {
+ readLock();
+ try {
+ return fenceIndex.containsKey(fenceKey);
+ } finally {
+ readUnlock();
+ }
+ }
+
+ @VisibleForTesting
+ LanceIndexJobQuota getQuota() {
+ return quota;
+ }
+
+ @VisibleForTesting
+ public int getJobCount() {
+ readLock();
+ try {
+ return jobs.size();
+ } finally {
+ readUnlock();
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Image serialization (whole-object Gson, IndexPolicyMgr style)
+ // ------------------------------------------------------------------
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ // Serialize against the declared base type: GsonUtils.BLOCK_INACCESSIBLE_JAVA refuses
+ // reflection on non-public runtime classes, so a subclassed manager (the test seam)
+ // would otherwise serialize as "null".
+ Text.writeString(out, GsonUtils.GSON.toJson(this, LanceIndexJobManager.class));
+ }
+
+ public static LanceIndexJobManager read(DataInput in) throws IOException {
+ return GsonUtils.GSON.fromJson(Text.readString(in), LanceIndexJobManager.class);
+ }
+
+ /**
+ * Rebuild the derived fence index and quota counters from the durable
+ * jobs after Gson image load. A fence-key collision between unresolved
+ * jobs (only possible on a corrupt image) retains every owner so the fence
+ * remains held until all colliding jobs settle; conflict reporting still
+ * uses the smaller job id.
+ */
+ @Override
+ public void gsonPostProcess() throws IOException {
+ fenceIndex.clear();
+ List unresolvedJobs = new ArrayList<>();
+ if (jobs == null) {
+ jobs = Maps.newConcurrentMap();
+ }
+ for (LanceIndexJob job : jobs.values()) {
+ if (job == null || !job.isUnresolved()) {
+ continue;
+ }
+ if (!hasFenceIdentity(job)) {
+ LOG.warn("lance index job {} lacks fence identity (provider/locator/name);"
+ + " excluded from fence/quota rebuild", job.getJobId());
+ continue;
+ }
+ unresolvedJobs.add(job);
+ NavigableSet owners = fenceIndex.computeIfAbsent(job.fenceKey(), ignored -> new TreeSet<>());
+ if (!owners.isEmpty() && !owners.contains(job.getJobId())) {
+ LOG.warn("fence key collision between unresolved lance index jobs {} and {}; keeping fence owner {}",
+ owners.first(), job.getJobId(), Math.min(owners.first(), job.getJobId()));
+ }
+ owners.add(job.getJobId());
+ }
+ quota.rebuild(unresolvedJobs);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java
new file mode 100644
index 00000000000000..9add2d7c20fbfa
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationState.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+/**
+ * Compact durable mutation lifecycle:
+ *
+ * PENDING -> RUNNING -> COMMITTED
+ * -> NOT_COMMITTED
+ * -> UNKNOWN
+ *
+ * All three outcomes are terminal. UNKNOWN has no outgoing transition:
+ * metadata never changes it to a known outcome, and FORCE_RELEASE (a later
+ * delivery slice) only releases the fence/quota/slot without rewriting the
+ * outcome.
+ */
+public enum LanceIndexJobMutationState {
+ /** The request and fence are durable; no execute send may have occurred. */
+ PENDING,
+ /** The durable dispatch boundary has been crossed; the one-shot call may execute or may already have executed. */
+ RUNNING,
+ /** A complete identity-matched typed success proves this job committed. */
+ COMMITTED,
+ /** A complete trusted result proves this job did not commit. */
+ NOT_COMMITTED,
+ /** Doris cannot safely prove whether this job committed. */
+ UNKNOWN;
+
+ public boolean isTerminal() {
+ return this == COMMITTED || this == NOT_COMMITTED || this == UNKNOWN;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java
new file mode 100644
index 00000000000000..4d5d3e893ff071
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobMutationType.java
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+/**
+ * The one-shot Lance index mutation a durable job intends to perform.
+ * REPLACE is a full same-name rebuild (CREATE OR REPLACE INDEX).
+ */
+public enum LanceIndexJobMutationType {
+ CREATE,
+ REPLACE,
+ DROP
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java
new file mode 100644
index 00000000000000..5634dc6f9355a9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuota.java
@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Three-level unresolved-job quota counters: per persisted table/locator
+ * identity, per catalog, and globally. An unresolved job is any job that still
+ * holds its same-name fence (PENDING/RUNNING, an unforced UNKNOWN, or a known
+ * terminal job whose required refresh is not DONE); active-plus-UNKNOWN counts
+ * stay bounded at every level.
+ *
+ * This class only counts. Callers hold the manager write lock, so no
+ * internal synchronization exists. Config-gated limits are resolved by the
+ * admission layer and passed in as positive finite values. The counters are rebuilt from the durable jobs after
+ * replay/image load and are never persisted themselves.
+ */
+public class LanceIndexJobQuota {
+ private static final Logger LOG = LogManager.getLogger(LanceIndexJobQuota.class);
+
+ private long globalCount;
+ private final Map catalogCounts = new HashMap<>();
+ private final Map tableCounts = new HashMap<>();
+
+ /**
+ * Check every level whose limit is positive, then increment all three
+ * levels. Returns false (and increments nothing) when any enforced level
+ * is full: "current < limit" must hold at every level.
+ *
+ * The manager admission path uses {@link #hasCapacity} plus
+ * {@link #charge} instead: the check and the charge deliberately straddle
+ * the edit-log write. This self-contained variant is reserved for direct
+ * use by the later admission slice and by tests.
+ */
+ public boolean tryAcquire(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) {
+ if (!hasCapacity(job, tableLimit, catalogLimit, globalLimit)) {
+ return false;
+ }
+ charge(job);
+ return true;
+ }
+
+ /**
+ * Pure check variant of {@link #tryAcquire}: true when incrementing would
+ * not exceed any positive finite limit. A non-positive limit is rejected.
+ */
+ public boolean hasCapacity(LanceIndexJob job, long tableLimit, long catalogLimit, long globalLimit) {
+ Objects.requireNonNull(job, "job");
+ if (tableLimit <= 0 || catalogLimit <= 0 || globalLimit <= 0) {
+ return false;
+ }
+ if (globalCount >= globalLimit) {
+ return false;
+ }
+ if (getCatalogCount(job.getCatalogId()) >= catalogLimit) {
+ return false;
+ }
+ return getTableCount(job.getTableQuotaKey()) < tableLimit;
+ }
+
+ /**
+ * Increment all three levels unconditionally. Used when a durable record
+ * (already admitted, or replayed) is applied to memory.
+ */
+ public void charge(LanceIndexJob job) {
+ globalCount++;
+ catalogCounts.merge(job.getCatalogId(), 1L, Long::sum);
+ tableCounts.merge(job.getTableQuotaKey(), 1L, Long::sum);
+ }
+
+ /**
+ * Decrement all three levels. Underflow is clamped at zero and warned
+ * about: it indicates a bookkeeping bug, not a reason to fail replay.
+ */
+ public void release(LanceIndexJob job) {
+ if (globalCount <= 0) {
+ LOG.warn("lance index job quota global underflow on release of job {}", job.getJobId());
+ } else {
+ globalCount--;
+ }
+ decrement(catalogCounts, job.getCatalogId(), job.getJobId());
+ decrement(tableCounts, job.getTableQuotaKey(), job.getJobId());
+ }
+
+ private static void decrement(Map counts, K key, long jobId) {
+ Long current = counts.get(key);
+ if (current == null || current <= 0) {
+ LOG.warn("lance index job quota underflow on release of job {} at key {}", jobId, key);
+ counts.remove(key);
+ return;
+ }
+ if (current == 1L) {
+ counts.remove(key);
+ } else {
+ counts.put(key, current - 1);
+ }
+ }
+
+ /**
+ * Reset and recount from the unresolved durable jobs (replay / image load).
+ */
+ public void rebuild(Iterable unresolvedJobs) {
+ globalCount = 0;
+ catalogCounts.clear();
+ tableCounts.clear();
+ for (LanceIndexJob job : unresolvedJobs) {
+ charge(job);
+ }
+ }
+
+ public long getGlobalCount() {
+ return globalCount;
+ }
+
+ public long getCatalogCount(long catalogId) {
+ return catalogCounts.getOrDefault(catalogId, 0L);
+ }
+
+ public long getTableCount(TableQuotaKey key) {
+ return tableCounts.getOrDefault(key, 0L);
+ }
+
+ /**
+ * Per persisted table/locator identity: (catalogId, normalizedLocator).
+ */
+ public static final class TableQuotaKey {
+ private final long catalogId;
+ private final String normalizedLocator;
+
+ public TableQuotaKey(long catalogId, String normalizedLocator) {
+ this.catalogId = catalogId;
+ this.normalizedLocator = Objects.requireNonNull(normalizedLocator, "normalizedLocator");
+ }
+
+ public long getCatalogId() {
+ return catalogId;
+ }
+
+ public String getNormalizedLocator() {
+ return normalizedLocator;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TableQuotaKey)) {
+ return false;
+ }
+ TableQuotaKey that = (TableQuotaKey) o;
+ return catalogId == that.catalogId && normalizedLocator.equals(that.normalizedLocator);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(catalogId, normalizedLocator);
+ }
+
+ /**
+ * Deliberately omits the locator, like {@link LanceIndexFenceKey#toString()}.
+ */
+ @Override
+ public String toString() {
+ return "TableQuotaKey{catalogId=" + catalogId + '}';
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java
new file mode 100644
index 00000000000000..f6ece16ac935aa
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobRefreshState.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+/**
+ * Independent metadata-refresh state of a durable Lance index job. Stored and
+ * transitioned independently from the mutation state: refresh success/failure
+ * only sets DONE/FAILED and never changes the mutation outcome.
+ *
+ * NOT_REQUIRED | REQUIRED -> RUNNING -> DONE | FAILED, FAILED -> RUNNING (retry)
+ *
+ * Refresh replays through the existing idempotent external-table refresh path;
+ * a failed refresh may be retried and still holds the same-name fence.
+ */
+public enum LanceIndexJobRefreshState {
+ /** No refresh is owed (e.g. a proven pre-invocation failure with no relevant metadata change). */
+ NOT_REQUIRED,
+ /** A refresh of the authoritative metadata is owed before the fence may be released. */
+ REQUIRED,
+ /** A refresh attempt is in flight. */
+ RUNNING,
+ /** The required refresh finished; the fence may be released. */
+ DONE,
+ /** The refresh attempt failed; retry through the idempotent path is allowed. */
+ FAILED
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java
new file mode 100644
index 00000000000000..49782f275b3c2d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResult.java
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+/**
+ * The immutable typed result of the one-shot invocation, persisted on the job.
+ * Contains only what is needed to classify the invocation: the saved typed
+ * code, the classified completion reason, a bounded sanitized message, and
+ * whether trusted pre-invocation revalidation observed relevant external
+ * metadata advancement. Never carries secrets, raw provider responses, or
+ * unbounded text.
+ */
+public class LanceIndexJobResult {
+ public static final int MAX_MESSAGE_BYTES = 1024;
+
+ @SerializedName(value = "rc")
+ private LanceIndexJobResultCode resultCode = LanceIndexJobResultCode.NO_TRUSTED_RESULT;
+
+ @SerializedName(value = "cr")
+ private LanceIndexJobCompletionReason completionReason = LanceIndexJobCompletionReason.NONE;
+
+ @SerializedName(value = "msg")
+ private String sanitizedMessage;
+
+ @SerializedName(value = "ema")
+ private boolean externalMetadataAdvanced;
+
+ /**
+ * No-arg constructor for Gson replay only; missing fields keep the safe
+ * defaults declared above.
+ */
+ public LanceIndexJobResult() {
+ }
+
+ public LanceIndexJobResult(LanceIndexJobResultCode resultCode, LanceIndexJobCompletionReason completionReason,
+ String sanitizedMessage, boolean externalMetadataAdvanced) {
+ this.resultCode = Objects.requireNonNull(resultCode, "resultCode");
+ this.completionReason = completionReason == null ? LanceIndexJobCompletionReason.NONE : completionReason;
+ this.sanitizedMessage = checkMessageBytes(sanitizedMessage);
+ this.externalMetadataAdvanced = externalMetadataAdvanced;
+ }
+
+ private static String checkMessageBytes(String message) {
+ if (message != null && message.getBytes(StandardCharsets.UTF_8).length > MAX_MESSAGE_BYTES) {
+ throw new IllegalArgumentException(
+ "sanitized message exceeds " + MAX_MESSAGE_BYTES + " UTF-8 bytes");
+ }
+ return message;
+ }
+
+ public LanceIndexJobResultCode getResultCode() {
+ return resultCode;
+ }
+
+ public LanceIndexJobCompletionReason getCompletionReason() {
+ return completionReason;
+ }
+
+ public String getSanitizedMessage() {
+ return sanitizedMessage;
+ }
+
+ public boolean isExternalMetadataAdvanced() {
+ return externalMetadataAdvanced;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof LanceIndexJobResult)) {
+ return false;
+ }
+ LanceIndexJobResult that = (LanceIndexJobResult) o;
+ return externalMetadataAdvanced == that.externalMetadataAdvanced
+ && resultCode == that.resultCode
+ && completionReason == that.completionReason
+ && Objects.equals(sanitizedMessage, that.sanitizedMessage);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(resultCode, completionReason, sanitizedMessage, externalMetadataAdvanced);
+ }
+
+ @Override
+ public String toString() {
+ return "LanceIndexJobResult{resultCode=" + resultCode + ", completionReason=" + completionReason
+ + ", externalMetadataAdvanced=" + externalMetadataAdvanced + '}';
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java
new file mode 100644
index 00000000000000..fad096b5087212
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultCode.java
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+/**
+ * Typed result codes of the one-shot Lance index invocation. Only saved typed
+ * codes are ever classified; message text is never inspected to infer an
+ * outcome. Codes fall into three groups:
+ *
+ *
+ * - PRE_INVOCATION_*: a complete trusted rejection before the native call
+ * (dataset version / schema contract / credential / resource revalidation).
+ * These always prove NOT_COMMITTED.
+ * - NATIVE_*: the saved LanceErrorCode of the single native invocation,
+ * read before the consuming error message.
+ * - NO_TRUSTED_RESULT: any ambiguity after the send may have occurred
+ * (EOF, signal, OOM, panic, timeout, BE loss, malformed/partial protocol,
+ * identity mismatch after acceptance, or a master-transfer sweep that found
+ * a durable RUNNING without a complete matching terminal result).
+ *
+ */
+public enum LanceIndexJobResultCode {
+ /** Pre-invocation: admitted dataset version / schema contract no longer matches. */
+ PRE_INVOCATION_STALE_ADMISSION,
+ /** Pre-invocation: the recomputed contract is not a supported contract v1. */
+ PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT,
+ /** Pre-invocation: credentials are known to be expired. */
+ PRE_INVOCATION_CREDENTIAL_EXPIRED,
+ /** Pre-invocation: trusted busy / pre-FFI resource rejection. */
+ PRE_INVOCATION_RESOURCE_REJECTED,
+ /** LANCE_OK from the one native invocation. */
+ NATIVE_OK,
+ /** LANCE_ERR_COMMIT_CONFLICT: the commit lost a race; the external dataset advanced. */
+ NATIVE_COMMIT_CONFLICT,
+ /** LANCE_ERR_NOT_FOUND after invocation. */
+ NATIVE_NOT_FOUND,
+ /** LANCE_ERR_INVALID_ARGUMENT after invocation. */
+ NATIVE_INVALID_ARGUMENT,
+ /** LANCE_ERR_NOT_SUPPORTED after invocation. */
+ NATIVE_NOT_SUPPORTED,
+ /** LANCE_ERR_INDEX after invocation (also the coarse duplicate-CREATE error). */
+ NATIVE_INDEX,
+ /** LANCE_ERR_IO after invocation. */
+ NATIVE_IO,
+ /** LANCE_ERR_INTERNAL after invocation. */
+ NATIVE_INTERNAL,
+ /** No complete trusted result exists; commitment cannot be proven either way. */
+ NO_TRUSTED_RESULT;
+
+ public boolean isPreInvocation() {
+ return this == PRE_INVOCATION_STALE_ADMISSION
+ || this == PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT
+ || this == PRE_INVOCATION_CREDENTIAL_EXPIRED
+ || this == PRE_INVOCATION_RESOURCE_REJECTED;
+ }
+
+ /**
+ * Classify a complete saved result into the durable (mutationState, refreshState,
+ * completionReason) triple, following the provider-result classification table:
+ *
+ *
+ * - PRE_INVOCATION_* -> NOT_COMMITTED; refresh REQUIRED only when the trusted
+ * pre-invocation revalidation observed relevant external metadata advancement.
+ * - NATIVE_OK -> COMMITTED + REQUIRED.
+ * - NATIVE_COMMIT_CONFLICT -> NOT_COMMITTED + REQUIRED (the external dataset
+ * advanced).
+ * - NATIVE_NOT_FOUND -> CREATE/REPLACE: UNKNOWN + NOT_REQUIRED (no attribution);
+ * DROP: NOT_COMMITTED + REQUIRED, with IF_CONDITION_NOOP for DROP IF EXISTS.
+ * - NATIVE_INVALID_ARGUMENT / NOT_SUPPORTED / INDEX / IO / INTERNAL ->
+ * UNKNOWN + NOT_REQUIRED (no outcome inference).
+ * - NO_TRUSTED_RESULT -> UNKNOWN + NOT_REQUIRED.
+ *
+ */
+ public static Classification classify(LanceIndexJobMutationType mutationType, LanceIndexJobResultCode resultCode,
+ boolean ifExists, boolean externalMetadataAdvanced) {
+ if (resultCode == null) {
+ resultCode = NO_TRUSTED_RESULT;
+ }
+ switch (resultCode) {
+ case PRE_INVOCATION_STALE_ADMISSION:
+ case PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT:
+ case PRE_INVOCATION_CREDENTIAL_EXPIRED:
+ case PRE_INVOCATION_RESOURCE_REJECTED:
+ return new Classification(LanceIndexJobMutationState.NOT_COMMITTED,
+ externalMetadataAdvanced
+ ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED,
+ LanceIndexJobCompletionReason.NONE);
+ case NATIVE_OK:
+ return new Classification(LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.REQUIRED,
+ LanceIndexJobCompletionReason.NONE);
+ case NATIVE_COMMIT_CONFLICT:
+ return new Classification(LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.REQUIRED,
+ LanceIndexJobCompletionReason.NONE);
+ case NATIVE_NOT_FOUND:
+ if (mutationType == LanceIndexJobMutationType.DROP) {
+ return new Classification(LanceIndexJobMutationState.NOT_COMMITTED,
+ LanceIndexJobRefreshState.REQUIRED,
+ ifExists ? LanceIndexJobCompletionReason.IF_CONDITION_NOOP
+ : LanceIndexJobCompletionReason.NONE);
+ }
+ return new Classification(LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED,
+ LanceIndexJobCompletionReason.NONE);
+ case NATIVE_INVALID_ARGUMENT:
+ case NATIVE_NOT_SUPPORTED:
+ case NATIVE_INDEX:
+ case NATIVE_IO:
+ case NATIVE_INTERNAL:
+ case NO_TRUSTED_RESULT:
+ default:
+ return new Classification(LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED,
+ LanceIndexJobCompletionReason.NONE);
+ }
+ }
+
+ /**
+ * The immutable outcome of {@link #classify}: the durable mutation state, the
+ * independent refresh obligation, and the completion reason.
+ */
+ public static final class Classification {
+ private final LanceIndexJobMutationState mutationState;
+ private final LanceIndexJobRefreshState refreshState;
+ private final LanceIndexJobCompletionReason completionReason;
+
+ private Classification(LanceIndexJobMutationState mutationState, LanceIndexJobRefreshState refreshState,
+ LanceIndexJobCompletionReason completionReason) {
+ this.mutationState = mutationState;
+ this.refreshState = refreshState;
+ this.completionReason = completionReason;
+ }
+
+ public LanceIndexJobMutationState getMutationState() {
+ return mutationState;
+ }
+
+ public LanceIndexJobRefreshState getRefreshState() {
+ return refreshState;
+ }
+
+ public LanceIndexJobCompletionReason getCompletionReason() {
+ return completionReason;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java
new file mode 100644
index 00000000000000..b467aa287f7f19
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizer.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+/**
+ * Logical index name normalization v1, the only definition: the UTF-8 result
+ * of Java {@code toLowerCase(Locale.ROOT)}. Both the display name and the
+ * normalized bytes are persisted on the job; Doris preserves display case,
+ * rejects new case-only duplicates, and fails mutation on ambiguous external
+ * case-only collisions. No normalization migration or mixed-version fence
+ * protocol exists.
+ */
+public final class LanceIndexNameNormalizer {
+ /** Bound on the persisted logical index name, aligned with the external string bound. */
+ public static final int MAX_INDEX_NAME_BYTES = 1024;
+
+ private LanceIndexNameNormalizer() {
+ }
+
+ /**
+ * Normalization v1. The result is the identity bytes of the same-name fence key.
+ */
+ public static String normalize(String displayName) {
+ if (displayName == null) {
+ throw new IllegalArgumentException("index name must not be null");
+ }
+ return displayName.toLowerCase(Locale.ROOT);
+ }
+
+ /**
+ * True when two display names differ only by case under normalization v1.
+ */
+ public static boolean isCaseOnlyDuplicate(String displayA, String displayB) {
+ if (displayA == null || displayB == null) {
+ return false;
+ }
+ return !displayA.equals(displayB) && normalize(displayA).equals(normalize(displayB));
+ }
+
+ /**
+ * Validate the persisted display name: non-empty and within the UTF-8 byte bound.
+ */
+ public static void validateDisplayName(String displayName) {
+ if (displayName == null || displayName.isEmpty()) {
+ throw new IllegalArgumentException("index name must not be null or empty");
+ }
+ if (displayName.getBytes(StandardCharsets.UTF_8).length > MAX_INDEX_NAME_BYTES) {
+ throw new IllegalArgumentException("index name exceeds " + MAX_INDEX_NAME_BYTES + " UTF-8 bytes");
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java
new file mode 100644
index 00000000000000..1175de7223da46
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexSchemaContract.java
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Persisted representation of schema contract v1: the ordered list of indexed
+ * fields captured at admission. Before native invocation the worker reopens
+ * the admitted dataset version, independently recomputes contract v1, and
+ * compares the ordered representation; a mismatch or unavailable version is a
+ * complete pre-invocation NOT_COMMITTED result. Equality is order-sensitive.
+ *
+ * Building a contract from an Arrow schema belongs to admission (a later
+ * delivery slice); this class is only the durable, comparable representation.
+ */
+public class LanceIndexSchemaContract {
+ /** The only schema contract version defined. */
+ public static final int SCHEMA_CONTRACT_VERSION_V1 = 1;
+ /** A finite representation bound; 4.2 admission may impose a smaller type-specific arity. */
+ public static final int MAX_INDEXED_FIELDS = 64;
+ /** Bound for every persisted schema-contract string. */
+ public static final int MAX_FIELD_STRING_BYTES = 1024;
+
+ @SerializedName(value = "scv")
+ private int schemaContractVersion = SCHEMA_CONTRACT_VERSION_V1;
+
+ @SerializedName(value = "flds")
+ private List fields = new ArrayList<>();
+
+ /**
+ * No-arg constructor for Gson replay only.
+ */
+ public LanceIndexSchemaContract() {
+ }
+
+ public LanceIndexSchemaContract(List fields) {
+ this.schemaContractVersion = SCHEMA_CONTRACT_VERSION_V1;
+ this.fields = validatedFields(fields);
+ }
+
+ /**
+ * Revalidate a possibly Gson-created contract before it enters a new durable
+ * admission record. Replay remains tolerant, but newly journaled records must
+ * use the supported version and bounded representation.
+ */
+ public void validateForAdmission() {
+ if (schemaContractVersion != SCHEMA_CONTRACT_VERSION_V1) {
+ throw new IllegalArgumentException("unsupported schema contract version: " + schemaContractVersion);
+ }
+ validatedFields(fields);
+ }
+
+ private static List validatedFields(List source) {
+ if (source == null) {
+ throw new IllegalArgumentException("schema contract fields must not be null");
+ }
+ if (source.size() > MAX_INDEXED_FIELDS) {
+ throw new IllegalArgumentException(
+ "schema contract exceeds " + MAX_INDEXED_FIELDS + " indexed fields");
+ }
+ List copy = new ArrayList<>(source.size());
+ for (IndexedField field : source) {
+ if (field == null) {
+ throw new IllegalArgumentException("schema contract indexed field must not be null");
+ }
+ field.validateForAdmission();
+ copy.add(field);
+ }
+ return Collections.unmodifiableList(copy);
+ }
+
+ private static String checkString(String value, boolean nullable, String fieldName) {
+ if (value == null) {
+ if (nullable) {
+ return null;
+ }
+ throw new IllegalArgumentException(fieldName + " must not be null");
+ }
+ if (value.getBytes(StandardCharsets.UTF_8).length > MAX_FIELD_STRING_BYTES) {
+ throw new IllegalArgumentException(
+ fieldName + " exceeds " + MAX_FIELD_STRING_BYTES + " UTF-8 bytes");
+ }
+ return value;
+ }
+
+ public int getSchemaContractVersion() {
+ return schemaContractVersion;
+ }
+
+ public List getFields() {
+ return fields == null ? Collections.emptyList() : Collections.unmodifiableList(fields);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof LanceIndexSchemaContract)) {
+ return false;
+ }
+ LanceIndexSchemaContract that = (LanceIndexSchemaContract) o;
+ return schemaContractVersion == that.schemaContractVersion && getFields().equals(that.getFields());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(schemaContractVersion, getFields());
+ }
+
+ @Override
+ public String toString() {
+ return "LanceIndexSchemaContract{version=" + schemaContractVersion + ", fields=" + getFields() + '}';
+ }
+
+ /**
+ * One indexed field of the ordered contract. {@code normalizedType}
+ * carries the relevant parameters such as decimal precision/scale and
+ * timestamp unit/time-zone semantics; unindexed fields are excluded from
+ * the contract.
+ */
+ public static final class IndexedField {
+ @SerializedName(value = "fid")
+ private long fieldId;
+
+ @SerializedName(value = "nn")
+ private String normalizedName;
+
+ @SerializedName(value = "nt")
+ private String normalizedType;
+
+ @SerializedName(value = "nul")
+ private boolean nullable;
+
+ @SerializedName(value = "fsd")
+ private Integer fixedSizeListDimension;
+
+ @SerializedName(value = "vet")
+ private String vectorElementType;
+
+ @SerializedName(value = "ven")
+ private Boolean vectorElementNullable;
+
+ /**
+ * No-arg constructor for Gson replay only.
+ */
+ public IndexedField() {
+ }
+
+ public IndexedField(long fieldId, String normalizedName, String normalizedType, boolean nullable,
+ Integer fixedSizeListDimension, String vectorElementType, Boolean vectorElementNullable) {
+ this.fieldId = fieldId;
+ this.normalizedName = checkString(normalizedName, false, "normalizedName");
+ this.normalizedType = checkString(normalizedType, false, "normalizedType");
+ this.nullable = nullable;
+ this.fixedSizeListDimension = fixedSizeListDimension;
+ this.vectorElementType = checkString(vectorElementType, true, "vectorElementType");
+ this.vectorElementNullable = vectorElementNullable;
+ }
+
+ private void validateForAdmission() {
+ checkString(normalizedName, false, "normalizedName");
+ checkString(normalizedType, false, "normalizedType");
+ checkString(vectorElementType, true, "vectorElementType");
+ }
+
+ public long getFieldId() {
+ return fieldId;
+ }
+
+ public String getNormalizedName() {
+ return normalizedName;
+ }
+
+ public String getNormalizedType() {
+ return normalizedType;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ public Integer getFixedSizeListDimension() {
+ return fixedSizeListDimension;
+ }
+
+ public String getVectorElementType() {
+ return vectorElementType;
+ }
+
+ public Boolean getVectorElementNullable() {
+ return vectorElementNullable;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof IndexedField)) {
+ return false;
+ }
+ IndexedField that = (IndexedField) o;
+ return fieldId == that.fieldId
+ && nullable == that.nullable
+ && Objects.equals(normalizedName, that.normalizedName)
+ && Objects.equals(normalizedType, that.normalizedType)
+ && Objects.equals(fixedSizeListDimension, that.fixedSizeListDimension)
+ && Objects.equals(vectorElementType, that.vectorElementType)
+ && Objects.equals(vectorElementNullable, that.vectorElementNullable);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fieldId, normalizedName, normalizedType, nullable, fixedSizeListDimension,
+ vectorElementType, vectorElementNullable);
+ }
+
+ @Override
+ public String toString() {
+ return "IndexedField{fieldId=" + fieldId + ", normalizedName=" + normalizedName
+ + ", normalizedType=" + normalizedType + ", nullable=" + nullable
+ + ", fixedSizeListDimension=" + fixedSizeListDimension
+ + ", vectorElementType=" + vectorElementType
+ + ", vectorElementNullable=" + vectorElementNullable + '}';
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java
new file mode 100644
index 00000000000000..d5de9b5cbdb225
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/job/LanceIndexTerminationProof.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+/**
+ * Evidence that releases a possible-live worker slot. Termination proof
+ * releases only that slot: it neither changes an UNKNOWN outcome nor releases
+ * the same-name fence. Deadlines bound wait/runtime but never prove
+ * termination.
+ */
+public enum LanceIndexTerminationProof {
+ /** No proof; the worker may still be running. */
+ NONE,
+ /** The supervisor reaped the exact matching child process. */
+ CHILD_REAPED,
+ /** The recorded BE process epoch no longer exists (the BE process was replaced). */
+ BE_PROCESS_EPOCH_GONE
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index 75cbbe7b487abc..d2c1952211dc61 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -49,6 +49,7 @@
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.InitDatabaseLog;
import org.apache.doris.datasource.MetaIdMappingsLog;
+import org.apache.doris.datasource.lance.job.LanceIndexJob;
import org.apache.doris.ha.MasterInfo;
import org.apache.doris.indexpolicy.DropIndexPolicyLog;
import org.apache.doris.indexpolicy.IndexPolicy;
@@ -1013,6 +1014,11 @@ public void readFields(DataInput in) throws IOException {
isRead = true;
break;
}
+ case OperationType.OP_LANCE_INDEX_JOB_UPSERT: {
+ data = LanceIndexJob.read(in);
+ isRead = true;
+ break;
+ }
case OperationType.OP_BEGIN_SNAPSHOT: {
data = SnapshotState.read(in);
isRead = true;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index bb01746a0ab826..9ccb4c5844528c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -64,6 +64,7 @@
import org.apache.doris.datasource.InitDatabaseLog;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.MetaIdMappingsLog;
+import org.apache.doris.datasource.lance.job.LanceIndexJob;
import org.apache.doris.dictionary.Dictionary;
import org.apache.doris.ha.MasterInfo;
import org.apache.doris.indexpolicy.DropIndexPolicyLog;
@@ -1442,6 +1443,11 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
env.getKeyManager().replayKeyOperation(info);
break;
}
+ case OperationType.OP_LANCE_INDEX_JOB_UPSERT: {
+ LanceIndexJob job = (LanceIndexJob) journal.getData();
+ env.getLanceIndexJobManager().replayUpsertJob(job);
+ break;
+ }
case OperationType.OP_BEGIN_SNAPSHOT: {
// SnapshotState info = (SnapshotState) journal.getData();
// TODO: implement
@@ -2353,6 +2359,10 @@ public void logDropIndexPolicy(DropIndexPolicyLog policy) {
logEdit(OperationType.OP_DROP_INDEX_POLICY, policy);
}
+ public void logLanceIndexJob(LanceIndexJob job) {
+ logEdit(OperationType.OP_LANCE_INDEX_JOB_UPSERT, job);
+ }
+
public void logCatalogLog(short id, CatalogLog log) {
logEdit(id, log);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index ba2580f4408e87..e2fbc2131f7000 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -422,6 +422,9 @@ public class OperationType {
public static final short OP_CREATE_ROLE_MAPPING = 496;
public static final short OP_DROP_ROLE_MAPPING = 497;
+ // lance index job 500 ~ 509
+ public static final short OP_LANCE_INDEX_JOB_UPSERT = 500;
+
// For cloud.
public static final short OP_UPDATE_CLOUD_REPLICA = 1000;
@Deprecated
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
index 8f8425ec4a80cc..6ed7645d4610b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
@@ -286,6 +286,12 @@ public static MetaPersistMethod create(String name) throws NoSuchMethodException
metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveKeyManagerStore",
CountingDataOutputStream.class, long.class);
break;
+ case "lanceIndexJobManager":
+ metaPersistMethod.readMethod = Env.class.getDeclaredMethod("loadLanceIndexJobManager",
+ DataInputStream.class, long.class);
+ metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveLanceIndexJobManager",
+ CountingDataOutputStream.class, long.class);
+ break;
default:
break;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
index 68ee92c8e6d7bd..8f8f00dd64ea2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
@@ -44,7 +44,7 @@ public class PersistMetaModules {
"globalFunction", "workloadGroups",
"binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "workloadSchedPolicy",
"insertOverwrite", "plsql", "dictionaryManager", "indexPolicy", "KeyManagerStore",
- "authenticationIntegrations", "roleMappings"
+ "authenticationIntegrations", "roleMappings", "lanceIndexJobManager"
);
// The modules in `CloudEnv`.
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java
new file mode 100644
index 00000000000000..2e46203b815959
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexDatasetLocatorTest.java
@@ -0,0 +1,183 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Unit coverage for dataset-locator normalization v1, the fence-key identity for the
+ * target dataset: trim, lowercase scheme, userinfo rejection (credential-bearing URLs
+ * are never identity), rejection of locators with neither authority nor path,
+ * trailing-slash stripping with a kept root, and the absolute-path requirement for
+ * scheme-less locators.
+ */
+public class LanceIndexDatasetLocatorTest {
+
+ @Test
+ public void lowercasesSchemeOnly() {
+ Assertions.assertEquals("s3://bucket/path", LanceIndexDatasetLocator.normalize("S3://bucket/path"));
+ Assertions.assertEquals("file:///data/x", LanceIndexDatasetLocator.normalize("FILE:///data/x"));
+ Assertions.assertEquals("hdfs://nn:8020/data", LanceIndexDatasetLocator.normalize("HDFS://nn:8020/data"));
+ }
+
+ @Test
+ public void preservesAuthorityAndPathCase() {
+ // Bucket and path components are case-sensitive on the supported providers.
+ Assertions.assertEquals("s3://MyBucket/MyPath", LanceIndexDatasetLocator.normalize("s3://MyBucket/MyPath"));
+ }
+
+ @Test
+ public void trimsSurroundingWhitespace() {
+ Assertions.assertEquals("/data/x", LanceIndexDatasetLocator.normalize(" /data/x \t\n"));
+ Assertions.assertEquals("s3://bucket/p", LanceIndexDatasetLocator.normalize(" s3://bucket/p "));
+ }
+
+ @Test
+ public void stripsTrailingSlashesFromPath() {
+ Assertions.assertEquals("s3://b/p", LanceIndexDatasetLocator.normalize("s3://b/p///"));
+ Assertions.assertEquals("/data/lance", LanceIndexDatasetLocator.normalize("/data/lance/"));
+ }
+
+ @Test
+ public void keepsSchemeLessRoot() {
+ Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("/"));
+ Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("//"));
+ Assertions.assertEquals("/", LanceIndexDatasetLocator.normalize("///"));
+ }
+
+ @Test
+ public void stripsTrailingSlashBehindAuthority() {
+ Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket/"));
+ Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket//"));
+ Assertions.assertEquals("s3://bucket", LanceIndexDatasetLocator.normalize("s3://bucket"));
+ }
+
+ @Test
+ public void acceptsFileUrlWithEmptyAuthority() {
+ Assertions.assertEquals("file:///data/x", LanceIndexDatasetLocator.normalize("file:///data/x"));
+ }
+
+ @Test
+ public void keepsAuthorityPortAndQueryFreePathVerbatim() {
+ Assertions.assertEquals("hdfs://nn:8020/base/table.lance",
+ LanceIndexDatasetLocator.normalize("hdfs://nn:8020/base/table.lance"));
+ }
+
+ @Test
+ public void allowsAtSignInPathButNotInAuthority() {
+ Assertions.assertEquals("s3://bucket/p@th", LanceIndexDatasetLocator.normalize("s3://bucket/p@th"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("s3://user@bucket/path"));
+ }
+
+ @Test
+ public void rejectsCredentialBearingUserinfo() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("s3://user:secret@bucket/path"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("https://user:secret@example.com/ds"));
+ }
+
+ @Test
+ public void rejectsQueriesAndFragmentsThatCouldCarryCredentials() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize(
+ "https://bucket.example/ds?X-Amz-Credential=AKIA_TEST&X-Amz-Signature=secret"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("https://account.blob.core.windows.net/ds?sig=secret"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("s3://bucket/ds#credential-fragment"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("/data/ds?token=secret"));
+ }
+
+ @Test
+ public void malformedLocatorErrorsDoNotEchoPotentialSecrets() {
+ String malformed = "s 3://user:top-secret@bucket/path";
+ IllegalArgumentException malformedError = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize(malformed));
+ Assertions.assertFalse(malformedError.getMessage().contains(malformed));
+ Assertions.assertFalse(malformedError.getMessage().contains("top-secret"));
+
+ String relative = "access-key:top-secret@bucket/path";
+ IllegalArgumentException relativeError = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize(relative));
+ Assertions.assertFalse(relativeError.getMessage().contains(relative));
+ Assertions.assertFalse(relativeError.getMessage().contains("top-secret"));
+ }
+
+ @Test
+ public void rejectsMalformedOrNonHierarchicalSchemes() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("1s3://bucket/path"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("s3:/bucket/path"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("s3:bucket/path"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("https://[malformed/path"));
+ }
+
+ @Test
+ public void locatorBoundCountsUtf8Bytes() {
+ String prefix = "s3://bucket/";
+ String atLimit = prefix + StringUtils.repeat(
+ "a", LanceIndexDatasetLocator.MAX_LOCATOR_BYTES - prefix.length());
+ Assertions.assertEquals(atLimit, LanceIndexDatasetLocator.normalize(atLimit));
+
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize(atLimit + "a"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize(prefix + StringUtils.repeat("é",
+ (LanceIndexDatasetLocator.MAX_LOCATOR_BYTES - prefix.length()) / 2 + 1)));
+ }
+
+ @Test
+ public void rejectsSchemeLessRelativePath() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("data/x"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("./x"));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexDatasetLocator.normalize("bucket/path"));
+ }
+
+ @Test
+ public void rejectsNullEmptyAndBlank() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(null));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(""));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize(" "));
+ }
+
+ @Test
+ public void rejectsEmptyScheme() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("://path"));
+ }
+
+ @Test
+ public void rejectsLocatorWithNeitherAuthorityNorPath() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("s3://"));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("file://"));
+ // Trailing-slash stripping still leaves nothing behind the scheme.
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexDatasetLocator.normalize("s3:///"));
+ // An empty authority with a real path stays legal.
+ Assertions.assertEquals("file:///x", LanceIndexDatasetLocator.normalize("file:///x"));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java
new file mode 100644
index 00000000000000..c63733355c9cd8
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerPersistTest.java
@@ -0,0 +1,436 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.journal.JournalEntity;
+import org.apache.doris.persist.OperationType;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Persistence coverage for the Lance index job infrastructure: the manager image
+ * write/read round trip (including the fence-index and quota rebuild in
+ * gsonPostProcess), the journal mounting point for op code 500
+ * ({@link OperationType#OP_LANCE_INDEX_JOB_UPSERT}) through
+ * {@link JournalEntity#write}/{@link JournalEntity#readFields}, the single-record Gson
+ * round trip field by field, and the bounded-text rejection at construction time.
+ */
+public class LanceIndexJobManagerPersistTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final String INVOCATION_ID = "invocation-1";
+
+ @Test
+ public void managerImageRoundtripRebuildsDerivedFenceAndQuota() throws Exception {
+ TestManager source = new TestManager();
+ source.createJob(newCreateJob(1L, "IdxPending"), 100, 100, 100);
+ source.createJob(newCreateJob(2L, "IdxRunning"), 100, 100, 100);
+ source.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
+ source.createJob(newCreateJob(3L, "IdxCommitted"), 100, 100, 100);
+ source.markRunning(3L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
+ source.completeWithResult(3L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, "ok", false));
+ source.createJob(newCreateJob(4L, "IdxUnknown"), 100, 100, 100);
+ source.markRunning(4L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
+ source.completeWithResult(4L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE, "lost", false));
+ source.replayUpsertJob(forceReleasedUnknownJob(5L, "idxforced"));
+
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ source.write(new DataOutputStream(byteStream));
+ LanceIndexJobManager loaded =
+ LanceIndexJobManager.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
+
+ Assertions.assertEquals(source.getJobCount(), loaded.getJobCount());
+ for (long jobId = 1L; jobId <= 5L; jobId++) {
+ Assertions.assertEquals(GsonUtils.GSON.toJson(source.getJob(jobId)),
+ GsonUtils.GSON.toJson(loaded.getJob(jobId)), "job " + jobId);
+ }
+
+ // The derived fence index survived: unresolved jobs still fence their names.
+ for (long jobId = 1L; jobId <= 4L; jobId++) {
+ Assertions.assertTrue(loaded.isFenceHeld(loaded.getJob(jobId).fenceKey()), "fence of job " + jobId);
+ }
+ Assertions.assertFalse(loaded.isFenceHeld(loaded.getJob(5L).fenceKey()));
+ Assertions.assertEquals(4L, loaded.getQuota().getGlobalCount());
+ Assertions.assertEquals(4L, loaded.getQuota().getCatalogCount(CATALOG_ID));
+ Assertions.assertEquals(4, loaded.getUnresolvedJobs().size());
+ Assertions.assertEquals(1, loaded.getJobsNeedingRefresh().size());
+
+ // The fence is enforceable after the image load: a conflict is rejected by the fence
+ // CAS, which precedes any journal write.
+ Assertions.assertThrows(DdlException.class,
+ () -> loaded.createJob(newCreateJob(9L, "idxunknown"), 100, 100, 100));
+
+ // And a FORCE-released name is free again once the loaded image is rebuilt through replay.
+ TestManager rebuilt = new TestManager();
+ for (long jobId = 1L; jobId <= 5L; jobId++) {
+ rebuilt.replayUpsertJob(loaded.getJob(jobId));
+ }
+ rebuilt.createJob(newCreateJob(9L, "IdxForced"), 100, 100, 100);
+ Assertions.assertEquals(5L, rebuilt.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void emptyManagerRoundtripStaysEmpty() throws Exception {
+ TestManager source = new TestManager();
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ source.write(new DataOutputStream(byteStream));
+ LanceIndexJobManager loaded =
+ LanceIndexJobManager.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
+
+ Assertions.assertEquals(0, loaded.getJobCount());
+ Assertions.assertEquals(0L, loaded.getQuota().getGlobalCount());
+ Assertions.assertTrue(loaded.getUnresolvedJobs().isEmpty());
+ }
+
+ @Test
+ public void imageLoadKeepsTheSmallerJobIdOnAFenceCollision() {
+ // Only a corrupt image can hold two unresolved jobs on the same fence key;
+ // gsonPostProcess keeps the smaller job id (Gson auto-runs it after fromJson).
+ String jobJson = "\"jid\":%d,\"rev\":0,\"cid\":" + CATALOG_ID
+ + ",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY + "\",\"loc\":\"" + LOCATOR
+ + "\",\"din\":\"IdxA\",\"nin\":\"idxa\",\"ms\":\"PENDING\",\"rs\":\"NOT_REQUIRED\"";
+ String json = "{\"jobs\":{\"2\":{" + String.format(jobJson, 2L) + "},"
+ + "\"1\":{" + String.format(jobJson, 1L) + "}}}";
+ LanceIndexJobManager loaded = GsonUtils.GSON.fromJson(json, LanceIndexJobManager.class);
+
+ Assertions.assertNotNull(loaded.getJob(1L));
+ Assertions.assertTrue(loaded.isFenceHeld(loaded.getJob(1L).fenceKey()));
+ // Both unresolved jobs still charge the quota.
+ Assertions.assertEquals(2L, loaded.getQuota().getGlobalCount());
+ // The fence conflict message names the surviving smaller job id.
+ DdlException exception = Assertions.assertThrows(DdlException.class,
+ () -> loaded.createJob(newCreateJob(9L, "idxa"), 100, 100, 100));
+ Assertions.assertTrue(exception.getMessage().contains("unresolved job 1"));
+ }
+
+ @Test
+ public void schemaContractEqualityIsOrderSensitive() {
+ LanceIndexSchemaContract ordered = contract();
+ LanceIndexSchemaContract sameOrder = contract();
+ List reversed = new ArrayList<>();
+ reversed.add(ordered.getFields().get(1));
+ reversed.add(ordered.getFields().get(0));
+ LanceIndexSchemaContract reordered = new LanceIndexSchemaContract(reversed);
+
+ Assertions.assertEquals(ordered, sameOrder);
+ Assertions.assertEquals(ordered.hashCode(), sameOrder.hashCode());
+ // The same fields in a different order are a different contract.
+ Assertions.assertNotEquals(ordered, reordered);
+ }
+
+ @Test
+ public void schemaContractIsBoundedNullFreeAndImmutable() {
+ LanceIndexSchemaContract.IndexedField field = new LanceIndexSchemaContract.IndexedField(
+ 1L, "v", "fixed_size_list[float;192]", false, 192, "float", false);
+ List atLimit = new ArrayList<>();
+ for (int index = 0; index < LanceIndexSchemaContract.MAX_INDEXED_FIELDS; index++) {
+ atLimit.add(field);
+ }
+ LanceIndexSchemaContract contractAtLimit = new LanceIndexSchemaContract(atLimit);
+ Assertions.assertEquals(LanceIndexSchemaContract.MAX_INDEXED_FIELDS, contractAtLimit.getFields().size());
+
+ atLimit.add(field);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexSchemaContract(atLimit));
+ List withNull = new ArrayList<>();
+ withNull.add(null);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexSchemaContract(withNull));
+
+ List source = new ArrayList<>();
+ source.add(field);
+ LanceIndexSchemaContract copied = new LanceIndexSchemaContract(source);
+ source.clear();
+ Assertions.assertEquals(1, copied.getFields().size());
+ Assertions.assertThrows(UnsupportedOperationException.class,
+ () -> copied.getFields().add(field));
+ }
+
+ @Test
+ public void schemaContractStringBoundsCountUtf8Bytes() {
+ String atLimit = StringUtils.repeat("é", LanceIndexSchemaContract.MAX_FIELD_STRING_BYTES / 2);
+ new LanceIndexSchemaContract.IndexedField(1L, atLimit, atLimit, false, null, atLimit, null);
+
+ String overLimit = atLimit + "é";
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexSchemaContract.IndexedField(
+ 1L, overLimit, "type", false, null, null, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexSchemaContract.IndexedField(
+ 1L, "name", overLimit, false, null, null, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexSchemaContract.IndexedField(
+ 1L, "name", "type", false, null, overLimit, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexSchemaContract.IndexedField(
+ 1L, null, "type", false, null, null, null));
+ }
+
+ @Test
+ public void journalEntityRoundtripUsesOpCode500() throws Exception {
+ LanceIndexJob job = fullyPopulatedJob();
+
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream output = new DataOutputStream(byteStream);
+ JournalEntity journalEntity = new JournalEntity();
+ journalEntity.setData(job);
+ journalEntity.setOpCode(OperationType.OP_LANCE_INDEX_JOB_UPSERT);
+ journalEntity.write(output);
+ output.flush();
+
+ JournalEntity replayed = new JournalEntity();
+ replayed.readFields(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
+
+ Assertions.assertEquals(OperationType.OP_LANCE_INDEX_JOB_UPSERT, replayed.getOpCode());
+ Assertions.assertEquals(500, replayed.getOpCode());
+ Assertions.assertTrue(replayed.getData() instanceof LanceIndexJob);
+ Assertions.assertEquals(GsonUtils.GSON.toJson(job), GsonUtils.GSON.toJson(replayed.getData()));
+ }
+
+ @Test
+ public void journalEntityStreamPreservesRecordOrder() throws Exception {
+ List jobs = new ArrayList<>();
+ jobs.add(fullyPopulatedJob());
+ LanceIndexJob second = newCreateJob(77L, "IdxSecond");
+ second.setMutationState(LanceIndexJobMutationState.PENDING);
+ jobs.add(second);
+
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream output = new DataOutputStream(byteStream);
+ for (LanceIndexJob job : jobs) {
+ JournalEntity journalEntity = new JournalEntity();
+ journalEntity.setData(job);
+ journalEntity.setOpCode(OperationType.OP_LANCE_INDEX_JOB_UPSERT);
+ journalEntity.write(output);
+ }
+ output.flush();
+
+ DataInputStream input = new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()));
+ for (LanceIndexJob expected : jobs) {
+ JournalEntity replayed = new JournalEntity();
+ replayed.readFields(input);
+ Assertions.assertEquals(OperationType.OP_LANCE_INDEX_JOB_UPSERT, replayed.getOpCode());
+ Assertions.assertEquals(GsonUtils.GSON.toJson(expected), GsonUtils.GSON.toJson(replayed.getData()));
+ }
+ Assertions.assertEquals(0, input.available());
+ }
+
+ @Test
+ public void jobStreamRoundtripPreservesAllFields() throws Exception {
+ LanceIndexJob job = fullyPopulatedJob();
+
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ job.write(new DataOutputStream(byteStream));
+ LanceIndexJob restored =
+ LanceIndexJob.read(new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray())));
+
+ assertSameJobFields(job, restored);
+ }
+
+ @Test
+ public void jobGsonRoundtripPreservesAllFields() {
+ LanceIndexJob job = fullyPopulatedJob();
+ LanceIndexJob restored = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(job), LanceIndexJob.class);
+ assertSameJobFields(job, restored);
+ }
+
+ @Test
+ public void boundedTextFieldsRejectOverflow() {
+ String overMessage = StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES + 1);
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, overMessage, false));
+ // Multibyte characters count as UTF-8 bytes, not chars.
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, StringUtils.repeat("é", 513), false));
+ new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK, LanceIndexJobCompletionReason.NONE,
+ StringUtils.repeat("m", LanceIndexJobResult.MAX_MESSAGE_BYTES), false);
+
+ LanceIndexJob job = newCreateJob(1L, "IdxA");
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setCreator(StringUtils.repeat("c", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setIndexType(StringUtils.repeat("t", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setColumnName(StringUtils.repeat("c", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setInvocationId(StringUtils.repeat("i", LanceIndexJob.MAX_INVOCATION_ID_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setForceActor(StringUtils.repeat("a", LanceIndexJob.MAX_DURABLE_TEXT_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES + 1)));
+ job.setPropertiesJson(StringUtils.repeat("p", LanceIndexJob.MAX_PROPERTIES_JSON_BYTES));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setForceNote(StringUtils.repeat("n", LanceIndexJob.MAX_FORCE_TEXT_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setForceWarning(StringUtils.repeat("w", LanceIndexJob.MAX_FORCE_TEXT_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setNormalizedIndexName(
+ StringUtils.repeat("n", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> job.setDisplayIndexName(
+ StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> newCreateJob(2L, StringUtils.repeat("d", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
+
+ // Optional durable text remains nullable; required creator identity does not.
+ job.setCreator(null);
+ Assertions.assertThrows(IllegalArgumentException.class, job::validateForAdmission);
+ job.setCreator("tester");
+ job.setIndexType(null);
+ job.setColumnName(null);
+ job.setInvocationId(null);
+ job.setForceActor(null);
+ job.setForceNote(null);
+ job.setForceWarning(null);
+ job.setPropertiesJson(null);
+ job.validateForAdmission();
+ }
+
+ @Test
+ public void resultRejectsNullResultCode() {
+ Assertions.assertThrows(NullPointerException.class,
+ () -> new LanceIndexJobResult(null, LanceIndexJobCompletionReason.NONE, "msg", false));
+ }
+
+ private static LanceIndexJob newCreateJob(long jobId, String displayName) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
+ null, 7L, null);
+ }
+
+ private static LanceIndexSchemaContract contract() {
+ List fields = new ArrayList<>();
+ fields.add(new LanceIndexSchemaContract.IndexedField(1L, "v", "fixed_size_list[float;192]",
+ false, 192, "float", false));
+ fields.add(new LanceIndexSchemaContract.IndexedField(2L, "s", "decimal(10,2)", true, null, null, null));
+ return new LanceIndexSchemaContract(fields);
+ }
+
+ /**
+ * A job with every durable field populated, including result, schema contract,
+ * dispatch identity, and the FORCE audit fields.
+ */
+ private static LanceIndexJob fullyPopulatedJob() {
+ LanceIndexJob job = new LanceIndexJob(42L, "creator", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxΩ", "idxω",
+ LanceIndexJobMutationType.REPLACE, true, false, "IVF_PQ", "v",
+ "{\"num_partitions\":\"256\"}", 99L, contract());
+ job.setRevision(7L);
+ job.setCreateTimeMs(111L);
+ job.setUpdateTimeMs(222L);
+ job.setMutationState(LanceIndexJobMutationState.RUNNING);
+ job.setRefreshState(LanceIndexJobRefreshState.RUNNING);
+ job.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_IO,
+ LanceIndexJobCompletionReason.NONE, "io error", true));
+ job.setBackendId(BACKEND_ID);
+ job.setBeProcessEpoch(BE_EPOCH);
+ job.setDispatchRevision(7L);
+ job.setInvocationId(INVOCATION_ID);
+ job.setDeadlineMs(123456L);
+ job.setPossibleLiveOwned(true);
+ job.setTerminationProof(LanceIndexTerminationProof.NONE);
+ job.setForceActor("admin");
+ job.setForceTimeMs(777L);
+ job.setForceNote("note");
+ job.setForceWarning("warning");
+ return job;
+ }
+
+ private static LanceIndexJob forceReleasedUnknownJob(long jobId, String normalizedName) {
+ String json = "{\"jid\":" + jobId + ",\"cr\":\"tester\",\"rev\":2,\"cid\":" + CATALOG_ID
+ + ",\"dbn\":\"db1\",\"tbn\":\"tbl1\",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY
+ + "\",\"loc\":\"" + LOCATOR + "\",\"din\":\"" + normalizedName + "\",\"nin\":\"" + normalizedName
+ + "\",\"mt\":\"CREATE\",\"ms\":\"UNKNOWN\",\"rs\":\"NOT_REQUIRED\",\"fr\":true}";
+ return GsonUtils.GSON.fromJson(json, LanceIndexJob.class);
+ }
+
+ private static void assertSameJobFields(LanceIndexJob expected, LanceIndexJob actual) {
+ Assertions.assertEquals(expected.getJobId(), actual.getJobId());
+ Assertions.assertEquals(expected.getCreator(), actual.getCreator());
+ Assertions.assertEquals(expected.getRevision(), actual.getRevision());
+ Assertions.assertEquals(expected.getCreateTimeMs(), actual.getCreateTimeMs());
+ Assertions.assertEquals(expected.getUpdateTimeMs(), actual.getUpdateTimeMs());
+ Assertions.assertEquals(expected.getCatalogId(), actual.getCatalogId());
+ Assertions.assertEquals(expected.getDbName(), actual.getDbName());
+ Assertions.assertEquals(expected.getTableName(), actual.getTableName());
+ Assertions.assertEquals(expected.getProvider(), actual.getProvider());
+ Assertions.assertEquals(expected.getNormalizedLocator(), actual.getNormalizedLocator());
+ Assertions.assertEquals(expected.getDisplayIndexName(), actual.getDisplayIndexName());
+ Assertions.assertEquals(expected.getNormalizedIndexName(), actual.getNormalizedIndexName());
+ Assertions.assertEquals(expected.getMutationType(), actual.getMutationType());
+ Assertions.assertEquals(expected.isIfNotExists(), actual.isIfNotExists());
+ Assertions.assertEquals(expected.isIfExists(), actual.isIfExists());
+ Assertions.assertEquals(expected.getIndexType(), actual.getIndexType());
+ Assertions.assertEquals(expected.getColumnName(), actual.getColumnName());
+ Assertions.assertEquals(expected.getPropertiesJson(), actual.getPropertiesJson());
+ Assertions.assertEquals(expected.getAdmittedDatasetVersion(), actual.getAdmittedDatasetVersion());
+ Assertions.assertEquals(expected.getSchemaContract(), actual.getSchemaContract());
+ Assertions.assertEquals(expected.getMutationState(), actual.getMutationState());
+ Assertions.assertEquals(expected.getRefreshState(), actual.getRefreshState());
+ Assertions.assertEquals(expected.getResult().getResultCode(), actual.getResult().getResultCode());
+ Assertions.assertEquals(expected.getResult().getCompletionReason(), actual.getResult().getCompletionReason());
+ Assertions.assertEquals(expected.getResult().getSanitizedMessage(), actual.getResult().getSanitizedMessage());
+ Assertions.assertEquals(expected.getResult().isExternalMetadataAdvanced(),
+ actual.getResult().isExternalMetadataAdvanced());
+ Assertions.assertEquals(expected.getBackendId(), actual.getBackendId());
+ Assertions.assertEquals(expected.getBeProcessEpoch(), actual.getBeProcessEpoch());
+ Assertions.assertEquals(expected.getDispatchRevision(), actual.getDispatchRevision());
+ Assertions.assertEquals(expected.getInvocationId(), actual.getInvocationId());
+ Assertions.assertEquals(expected.getDeadlineMs(), actual.getDeadlineMs());
+ Assertions.assertEquals(expected.isPossibleLiveOwned(), actual.isPossibleLiveOwned());
+ Assertions.assertEquals(expected.getTerminationProof(), actual.getTerminationProof());
+ Assertions.assertEquals(expected.isForceReleased(), actual.isForceReleased());
+ Assertions.assertEquals(expected.getForceActor(), actual.getForceActor());
+ Assertions.assertEquals(expected.getForceTimeMs(), actual.getForceTimeMs());
+ Assertions.assertEquals(expected.getForceNote(), actual.getForceNote());
+ Assertions.assertEquals(expected.getForceWarning(), actual.getForceWarning());
+ Assertions.assertEquals(expected.fenceKey(), actual.fenceKey());
+ Assertions.assertEquals(expected.getTableQuotaKey(), actual.getTableQuotaKey());
+ }
+
+ /**
+ * Edit-log seam: captures every durable record instead of writing the journal.
+ */
+ private static class TestManager extends LanceIndexJobManager {
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ // No journal in a pure persistence unit test.
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java
new file mode 100644
index 00000000000000..68d51a134e9057
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobManagerReplayTest.java
@@ -0,0 +1,498 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Replay and master-transfer coverage for {@link LanceIndexJobManager}. The journal
+ * records of a scenario are captured from a source manager through the edit-log seam
+ * and replayed verbatim into a fresh target manager, exactly like a follower tailing
+ * the edit log. The pinned invariants: replay is a verbatim replace with a monotonic
+ * revision guard (no state transformation on followers); a replayed PENDING permits
+ * exactly one dispatch; only the master-election sweep turns a durable RUNNING into
+ * UNKNOWN, after which redispatch is permanently refused; an unresolved UNKNOWN keeps
+ * its fence and quota across replay while a FORCE-released UNKNOWN frees both; stale
+ * callbacks (revision / invocation id / BE epoch) never change state; and a rejected
+ * admission leaves no job, fence, quota charge, or journal record behind.
+ */
+public class LanceIndexJobManagerReplayTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final String INVOCATION_ID = "invocation-1";
+ private static final long DEADLINE_MS = 9999L;
+
+ @Test
+ public void replayedPendingPermitsExactlyOneDispatch() throws DdlException {
+ List records = runningRecords(1L, "IdxA");
+ TestManager target = new TestManager();
+ target.replayUpsertJob(records.get(0));
+
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(1L).getMutationState());
+ Assertions.assertTrue(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(target.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, target.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void replayedRunningStaysRunningWithoutSweep() throws DdlException {
+ List records = runningRecords(1L, "IdxA");
+ TestManager target = new TestManager();
+ target.replayUpsertJob(records.get(0));
+ target.replayUpsertJob(records.get(1));
+
+ LanceIndexJob stored = target.getJob(1L);
+ // A follower tailing a live master must not transform a fresh RUNNING record.
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertEquals(1L, stored.getRevision());
+ Assertions.assertTrue(stored.holdsPossibleLiveSlot());
+ Assertions.assertTrue(target.isFenceHeld(stored.fenceKey()));
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ // Replay itself never writes the journal.
+ Assertions.assertTrue(target.editLog.isEmpty());
+ }
+
+ @Test
+ public void transferToMasterConvertsRunningToUnknownAndNeverRedispatches() throws DdlException {
+ List records = runningRecords(1L, "IdxA");
+ TestManager target = new TestManager();
+ target.replayUpsertJob(records.get(0));
+ target.replayUpsertJob(records.get(1));
+ LanceIndexFenceKey fenceKey = target.getJob(1L).fenceKey();
+
+ target.onTransferToMaster();
+
+ LanceIndexJob swept = target.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState());
+ Assertions.assertEquals(2L, swept.getRevision());
+ Assertions.assertEquals(LanceIndexJobResultCode.NO_TRUSTED_RESULT, swept.getResult().getResultCode());
+ // Fence, quota, and the possible-live slot survive the sweep: only FORCE releases them.
+ Assertions.assertTrue(target.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ Assertions.assertTrue(swept.holdsPossibleLiveSlot());
+ Assertions.assertTrue(containsJob(target.getUnresolvedJobs(), swept.getJobId()));
+ Assertions.assertEquals(1, target.editLog.size());
+
+ Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(target.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void transferToMasterDowngradesRunningRefreshToRequired() throws DdlException {
+ TestManager source = new TestManager();
+ source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
+ source.markRefreshRunning(1L, 2L);
+
+ TestManager target = new TestManager();
+ for (LanceIndexJob record : source.editLog) {
+ target.replayUpsertJob(record);
+ }
+ Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, target.getJob(1L).getRefreshState());
+
+ target.onTransferToMaster();
+
+ LanceIndexJob swept = target.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, swept.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, swept.getRefreshState());
+ Assertions.assertEquals(4L, swept.getRevision());
+ Assertions.assertTrue(containsJob(target.getJobsNeedingRefresh(), swept.getJobId()));
+ Assertions.assertTrue(target.isFenceHeld(swept.fenceKey()));
+ }
+
+ @Test
+ public void replayedTerminalWithRefreshRequiredOnlyAllowsRefreshPath() throws DdlException {
+ TestManager source = new TestManager();
+ source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
+
+ TestManager target = new TestManager();
+ for (LanceIndexJob record : source.editLog) {
+ target.replayUpsertJob(record);
+ }
+ LanceIndexJob stored = target.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
+ LanceIndexFenceKey fenceKey = stored.fenceKey();
+
+ // The mutation lifecycle is closed; only the refresh transitions remain.
+ Assertions.assertFalse(target.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(target.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH, okResult()));
+ Assertions.assertTrue(containsJob(target.getJobsNeedingRefresh(), stored.getJobId()));
+ Assertions.assertTrue(target.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+
+ Assertions.assertTrue(target.markRefreshRunning(1L, 2L));
+ Assertions.assertTrue(target.markRefreshDone(1L, 3L));
+ Assertions.assertFalse(target.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
+ Assertions.assertTrue(target.getUnresolvedJobs().isEmpty());
+ }
+
+ @Test
+ public void replayedUnresolvedUnknownFencesTheSameName() throws DdlException {
+ TestManager source = new TestManager();
+ source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE, "ambiguous", false));
+
+ TestManager target = new TestManager();
+ for (LanceIndexJob record : source.editLog) {
+ target.replayUpsertJob(record);
+ }
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, target.getJob(1L).getMutationState());
+
+ // The unforced UNKNOWN still holds the same-name fence and the quota.
+ Assertions.assertThrows(DdlException.class,
+ () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100));
+ Assertions.assertThrows(DdlException.class,
+ () -> target.createJob(newCreateJob(9L, "idxa"), 100, 100, 100));
+ target.createJob(newCreateJob(9L, "IdxB"), 100, 100, 100);
+ Assertions.assertEquals(2L, target.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void replayedForceReleasedUnknownFreesNameAndQuota() {
+ TestManager target = new TestManager();
+ target.replayUpsertJob(forceReleasedUnknownJob(7L, "idxforce"));
+
+ Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
+ Assertions.assertFalse(target.isFenceHeld(target.getJob(7L).fenceKey()));
+ Assertions.assertTrue(target.getUnresolvedJobs().isEmpty());
+
+ Assertions.assertDoesNotThrow(() -> target.createJob(newCreateJob(8L, "IdxForce"), 100, 100, 100));
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ Assertions.assertEquals(1, target.editLog.size());
+ }
+
+ @Test
+ public void staleCallbacksAreRejectedWithoutStateChange() throws DdlException {
+ List records = runningRecords(1L, "IdxA");
+ TestManager target = new TestManager();
+ target.replayUpsertJob(records.get(0));
+ target.replayUpsertJob(records.get(1));
+
+ Assertions.assertFalse(target.completeWithResult(1L, 0L, INVOCATION_ID, BE_EPOCH, okResult()));
+ Assertions.assertFalse(target.completeWithResult(1L, 1L, "invocation-x", BE_EPOCH, okResult()));
+ Assertions.assertFalse(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH + 1, okResult()));
+ Assertions.assertFalse(target.completeWithResult(1L, 1L, INVOCATION_ID, null, okResult()));
+ Assertions.assertFalse(target.completeWithResult(404L, 1L, INVOCATION_ID, BE_EPOCH, okResult()));
+
+ LanceIndexJob stored = target.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertEquals(1L, stored.getRevision());
+ Assertions.assertNull(stored.getResult());
+ Assertions.assertTrue(target.editLog.isEmpty());
+
+ Assertions.assertTrue(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult()));
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, target.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void replayIsIdempotentForTheSameRecord() throws DdlException {
+ List records = runningRecords(1L, "IdxA");
+ TestManager target = new TestManager();
+ target.replayUpsertJob(records.get(0));
+ target.replayUpsertJob(records.get(0));
+ target.replayUpsertJob(records.get(1));
+ target.replayUpsertJob(records.get(1));
+
+ Assertions.assertEquals(1, target.getJobCount());
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, target.getJob(1L).getMutationState());
+ Assertions.assertEquals(1L, target.getJob(1L).getRevision());
+ }
+
+ @Test
+ public void lowerRevisionRecordNeverOverwritesHigherRevision() throws DdlException {
+ TestManager source = new TestManager();
+ source.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ source.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ source.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
+ // Records: PENDING rev0, RUNNING rev1, COMMITTED+REQUIRED rev2.
+
+ TestManager target = new TestManager();
+ target.replayUpsertJob(source.editLog.get(2));
+ target.replayUpsertJob(source.editLog.get(0));
+ target.replayUpsertJob(source.editLog.get(1));
+
+ LanceIndexJob stored = target.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(2L, stored.getRevision());
+ Assertions.assertTrue(target.isFenceHeld(stored.fenceKey()));
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ Assertions.assertTrue(target.editLog.isEmpty());
+
+ // An equal revision replaces verbatim (idempotent re-delivery).
+ target.replayUpsertJob(source.editLog.get(2));
+ Assertions.assertEquals(2L, target.getJob(1L).getRevision());
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void replayToleratesNullAndIdentityLessRecords() {
+ TestManager target = new TestManager();
+ target.replayUpsertJob(null);
+ Assertions.assertEquals(0, target.getJobCount());
+
+ // A corrupt record without fence identity stays queryable but out of the books.
+ LanceIndexJob sparse = GsonUtils.GSON.fromJson(
+ "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class);
+ target.replayUpsertJob(sparse);
+ Assertions.assertEquals(1, target.getJobCount());
+ Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(5L).getMutationState());
+ Assertions.assertTrue(target.getUnresolvedJobs().isEmpty());
+ Assertions.assertFalse(target.markRunning(5L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ }
+
+ @Test
+ public void resolvingTheSmallestCollidingJobLeavesTheOtherFenceOwner() throws DdlException {
+ TestManager target = new TestManager();
+ target.replayUpsertJob(pendingRecord(1L, "IdxA"));
+ target.replayUpsertJob(pendingRecord(2L, "IdxA"));
+ LanceIndexFenceKey fenceKey = target.getJob(1L).fenceKey();
+
+ Assertions.assertEquals(2L, target.getQuota().getGlobalCount());
+ DdlException initialConflict = Assertions.assertThrows(DdlException.class,
+ () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100));
+ Assertions.assertTrue(initialConflict.getMessage().contains("unresolved job 1"));
+
+ Assertions.assertTrue(target.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertTrue(target.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED,
+ LanceIndexJobCompletionReason.NONE, "rejected before invocation", false)));
+
+ Assertions.assertTrue(target.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, target.getQuota().getGlobalCount());
+ DdlException remainingConflict = Assertions.assertThrows(DdlException.class,
+ () -> target.createJob(newCreateJob(9L, "IdxA"), 100, 100, 100));
+ Assertions.assertTrue(remainingConflict.getMessage().contains("unresolved job 2"));
+ }
+
+ @Test
+ public void replayAndQueryBoundariesReturnDefensiveCopies() throws DdlException {
+ TestManager target = new TestManager();
+ LanceIndexJob replayed = pendingRecord(1L, "IdxA");
+ target.replayUpsertJob(replayed);
+
+ replayed.setMutationState(LanceIndexJobMutationState.COMMITTED);
+ replayed.setRefreshState(LanceIndexJobRefreshState.DONE);
+ replayed.setRevision(99L);
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, target.getJob(1L).getMutationState());
+ Assertions.assertEquals(0L, target.getJob(1L).getRevision());
+
+ LanceIndexJob queried = target.getJob(1L);
+ queried.setForceReleased(true);
+ queried.setRevision(88L);
+ Assertions.assertFalse(target.getJob(1L).isForceReleased());
+ Assertions.assertEquals(0L, target.getJob(1L).getRevision());
+
+ LanceIndexJob listed = target.getUnresolvedJobs().get(0);
+ listed.setForceReleased(true);
+ Assertions.assertFalse(target.getJob(1L).isForceReleased());
+
+ target.createJob(newCreateJob(2L, "IdxB"), 100, 100, 100);
+ Assertions.assertTrue(target.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, "invocation-2", DEADLINE_MS));
+ Assertions.assertTrue(target.completeWithResult(2L, 1L, "invocation-2", BE_EPOCH, okResult()));
+ LanceIndexJob refreshCandidate = target.getJobsNeedingRefresh().get(0);
+ refreshCandidate.setRefreshState(LanceIndexJobRefreshState.DONE);
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, target.getJob(2L).getRefreshState());
+ }
+
+ @Test
+ public void replayingOverAnIdentityLessRecordNeverThrows() {
+ TestManager target = new TestManager();
+ LanceIndexJob sparse = GsonUtils.GSON.fromJson(
+ "{\"jid\":5,\"rev\":0,\"ms\":\"PENDING\"}", LanceIndexJob.class);
+ target.replayUpsertJob(sparse);
+ Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
+
+ // Re-delivering the same identity-less record must not key the release side on it.
+ Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(sparse));
+ Assertions.assertEquals(1, target.getJobCount());
+ Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
+
+ // Neither must a higher-revision identity-less upsert of the same job id.
+ LanceIndexJob unknown = GsonUtils.GSON.fromJson(
+ "{\"jid\":5,\"rev\":1,\"ms\":\"UNKNOWN\"}", LanceIndexJob.class);
+ Assertions.assertDoesNotThrow(() -> target.replayUpsertJob(unknown));
+ LanceIndexJob stored = target.getJob(5L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(1L, stored.getRevision());
+ Assertions.assertEquals(1, target.getJobCount());
+ Assertions.assertEquals(0L, target.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void transferToMasterSweepsOnlyTheRunningJobInAMixedPopulation() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxPending"), 100, 100, 100);
+ manager.createJob(newCreateJob(2L, "IdxCommitted"), 100, 100, 100);
+ manager.markRunning(2L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH, okResult());
+ manager.markRefreshRunning(2L, 2L);
+ manager.markRefreshDone(2L, 3L);
+ manager.createJob(newCreateJob(3L, "IdxRunning"), 100, 100, 100);
+ manager.markRunning(3L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ // Journal so far: create(1), create+run+complete+refreshRun+refreshDone(2), create+run(3).
+ Assertions.assertEquals(8, manager.editLog.size());
+
+ manager.onTransferToMaster();
+
+ // PENDING and the settled terminal job are replay-faithful: untouched.
+ LanceIndexJob pending = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, pending.getMutationState());
+ Assertions.assertEquals(0L, pending.getRevision());
+ LanceIndexJob committed = manager.getJob(2L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, committed.getRefreshState());
+ Assertions.assertEquals(4L, committed.getRevision());
+
+ // Only the durable RUNNING becomes UNKNOWN, through exactly one journal record.
+ LanceIndexJob swept = manager.getJob(3L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, swept.getMutationState());
+ Assertions.assertEquals(2L, swept.getRevision());
+ Assertions.assertEquals(9, manager.editLog.size());
+ }
+
+ @Test
+ public void fenceRejectionLeavesNoJobNoQuotaAndNoJournalRecord() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ DdlException exception = Assertions.assertThrows(DdlException.class,
+ () -> manager.createJob(newCreateJob(2L, "idxa"), 100, 100, 100));
+ // The rejection must not disclose the dataset locator to an unauthorized caller.
+ Assertions.assertFalse(exception.getMessage().contains("bucket"));
+ Assertions.assertFalse(exception.getMessage().contains(LOCATOR));
+
+ Assertions.assertEquals(1, manager.getJobCount());
+ Assertions.assertEquals(1, manager.editLog.size());
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ // Only the original job's fence exists; nothing new was registered for the rejection.
+ Assertions.assertFalse(manager.isFenceHeld(newCreateJob(3L, "IdxB").fenceKey()));
+ }
+
+ @Test
+ public void quotaRejectionLeavesNoJobNoFenceAndNoJournalRecord() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 1, 1, 1);
+
+ Assertions.assertThrows(DdlException.class,
+ () -> manager.createJob(newCreateJob(2L, "IdxB"), 1, 1, 1));
+
+ Assertions.assertEquals(1, manager.getJobCount());
+ Assertions.assertEquals(1, manager.editLog.size());
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertFalse(manager.isFenceHeld(newCreateJob(2L, "IdxB").fenceKey()));
+ }
+
+ @Test
+ public void duplicateJobIdIsRejectedBeforeAnythingElse() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ Assertions.assertThrows(DdlException.class,
+ () -> manager.createJob(newCreateJob(1L, "IdxB"), 100, 100, 100));
+ Assertions.assertEquals(1, manager.getJobCount());
+ Assertions.assertEquals(1, manager.editLog.size());
+ Assertions.assertFalse(manager.isFenceHeld(newCreateJob(1L, "IdxB").fenceKey()));
+ }
+
+ private static LanceIndexJob newCreateJob(long jobId, String displayName) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
+ null, 7L, null);
+ }
+
+ private static LanceIndexJob pendingRecord(long jobId, String displayName) {
+ LanceIndexJob job = newCreateJob(jobId, displayName);
+ job.setMutationState(LanceIndexJobMutationState.PENDING);
+ job.setRefreshState(LanceIndexJobRefreshState.NOT_REQUIRED);
+ return job;
+ }
+
+ private static boolean containsJob(List jobs, long jobId) {
+ return jobs.stream().anyMatch(job -> job.getJobId() == jobId);
+ }
+
+ private static LanceIndexJobResult okResult() {
+ return new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, "ok", false);
+ }
+
+ /**
+ * Runs create + dispatch on a throwaway source manager and returns its journal
+ * records: [PENDING rev0, RUNNING rev1].
+ */
+ private static List runningRecords(long jobId, String displayName) throws DdlException {
+ TestManager source = new TestManager();
+ source.createJob(newCreateJob(jobId, displayName), 100, 100, 100);
+ source.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ return source.editLog;
+ }
+
+ /**
+ * Builds the durable form of a FORCE-released UNKNOWN job. The FORCE slice owns the
+ * transition itself; here the record only exists as replay input, so it is built
+ * from its JSON journal form.
+ */
+ private static LanceIndexJob forceReleasedUnknownJob(long jobId, String normalizedName) {
+ String json = "{\"jid\":" + jobId + ",\"cr\":\"tester\",\"rev\":2,\"cid\":" + CATALOG_ID
+ + ",\"dbn\":\"db1\",\"tbn\":\"tbl1\",\"prv\":\"" + LanceIndexFenceKey.PROVIDER_DIRECTORY
+ + "\",\"loc\":\"" + LOCATOR + "\",\"din\":\"" + normalizedName + "\",\"nin\":\"" + normalizedName
+ + "\",\"mt\":\"CREATE\",\"ms\":\"UNKNOWN\",\"rs\":\"NOT_REQUIRED\",\"fr\":true,"
+ + "\"fa\":\"admin\",\"ftm\":12345,\"fn\":\"operator note\"}";
+ LanceIndexJob job = GsonUtils.GSON.fromJson(json, LanceIndexJob.class);
+ Assertions.assertTrue(job.isForceReleased());
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, job.getMutationState());
+ return job;
+ }
+
+ /**
+ * Edit-log seam: captures every durable record instead of writing the journal.
+ */
+ private static class TestManager extends LanceIndexJobManager {
+ private final List editLog = new ArrayList<>();
+
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ editLog.add(job);
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java
new file mode 100644
index 00000000000000..9511f7f1bc06a2
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobQuotaTest.java
@@ -0,0 +1,208 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Unit coverage for the three-level unresolved-job quota counters (table/locator,
+ * catalog, global): the "current < limit" boundary at each level, rejection of
+ * non-positive limits, release recovery with underflow clamping, and rebuild
+ * equivalence with live counting. The "which jobs count" semantics are
+ * {@link LanceIndexJob#isUnresolved()}; the rebuild-side composition is pinned here too.
+ */
+public class LanceIndexJobQuotaTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR_A = "s3://bucket/table-a";
+ private static final String LOCATOR_B = "s3://bucket/table-b";
+
+ @Test
+ public void tryAcquireChargesAllThreeLevels() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ LanceIndexJob job = newJob(1L, CATALOG_ID, LOCATOR_A);
+
+ Assertions.assertTrue(quota.tryAcquire(job, 5, 5, 5));
+ Assertions.assertEquals(1L, quota.getGlobalCount());
+ Assertions.assertEquals(1L, quota.getCatalogCount(CATALOG_ID));
+ Assertions.assertEquals(1L, quota.getTableCount(job.getTableQuotaKey()));
+ }
+
+ @Test
+ public void tableLimitRejectsTheNextJobExactlyAtLimit() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 2, 100, 100));
+ Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 2, 100, 100));
+
+ LanceIndexJob third = newJob(3L, CATALOG_ID, LOCATOR_A);
+ Assertions.assertFalse(quota.tryAcquire(third, 2, 100, 100));
+ // A rejected acquire charges nothing at any level.
+ Assertions.assertEquals(2L, quota.getGlobalCount());
+ Assertions.assertEquals(2L, quota.getTableCount(third.getTableQuotaKey()));
+
+ // The limit is per table/locator identity: another table still has room.
+ Assertions.assertTrue(quota.tryAcquire(newJob(4L, CATALOG_ID, LOCATOR_B), 2, 100, 100));
+ }
+
+ @Test
+ public void catalogLimitRejectsAcrossTables() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 100, 2, 100));
+ Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_B), 100, 2, 100));
+ Assertions.assertFalse(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 100, 2, 100));
+
+ // Another catalog is a separate level.
+ Assertions.assertTrue(quota.tryAcquire(newJob(4L, 20L, LOCATOR_A), 100, 2, 100));
+ }
+
+ @Test
+ public void globalLimitRejectsAcrossCatalogs() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ Assertions.assertTrue(quota.tryAcquire(newJob(1L, CATALOG_ID, LOCATOR_A), 100, 100, 2));
+ Assertions.assertTrue(quota.tryAcquire(newJob(2L, 20L, LOCATOR_A), 100, 100, 2));
+ Assertions.assertFalse(quota.tryAcquire(newJob(3L, 30L, LOCATOR_B), 100, 100, 2));
+ Assertions.assertEquals(2L, quota.getGlobalCount());
+ }
+
+ @Test
+ public void nonPositiveLimitIsRejectedWithoutACharge() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ LanceIndexJob job = newJob(1L, CATALOG_ID, LOCATOR_A);
+ Assertions.assertFalse(quota.tryAcquire(job, 0, 1, 1));
+ Assertions.assertFalse(quota.tryAcquire(job, 1, 0, 1));
+ Assertions.assertFalse(quota.tryAcquire(job, 1, 1, 0));
+ Assertions.assertFalse(quota.tryAcquire(job, -1, 1, 1));
+ Assertions.assertEquals(0L, quota.getGlobalCount());
+ }
+
+ @Test
+ public void releaseRecoversCapacityAtAllLevels() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ LanceIndexJob first = newJob(1L, CATALOG_ID, LOCATOR_A);
+ Assertions.assertTrue(quota.tryAcquire(first, 1, 1, 1));
+ Assertions.assertFalse(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 1, 1, 1));
+
+ quota.release(first);
+ Assertions.assertEquals(0L, quota.getGlobalCount());
+ Assertions.assertEquals(0L, quota.getCatalogCount(CATALOG_ID));
+ Assertions.assertEquals(0L, quota.getTableCount(first.getTableQuotaKey()));
+
+ Assertions.assertTrue(quota.tryAcquire(newJob(3L, CATALOG_ID, LOCATOR_A), 1, 1, 1));
+ }
+
+ @Test
+ public void releaseUnderflowClampsAtZero() {
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ // Releasing a job that was never charged must not fail and must not go negative.
+ quota.release(newJob(1L, CATALOG_ID, LOCATOR_A));
+ Assertions.assertEquals(0L, quota.getGlobalCount());
+ Assertions.assertEquals(0L, quota.getCatalogCount(CATALOG_ID));
+ Assertions.assertTrue(quota.tryAcquire(newJob(2L, CATALOG_ID, LOCATOR_A), 1, 1, 1));
+ }
+
+ @Test
+ public void rebuildMatchesIncrementalCounting() {
+ List unresolved = new ArrayList<>();
+ unresolved.add(newJob(1L, CATALOG_ID, LOCATOR_A));
+ unresolved.add(newJob(2L, CATALOG_ID, LOCATOR_A));
+ unresolved.add(newJob(3L, CATALOG_ID, LOCATOR_B));
+ unresolved.add(newJob(4L, 20L, LOCATOR_A));
+
+ LanceIndexJobQuota incremental = new LanceIndexJobQuota();
+ for (LanceIndexJob job : unresolved) {
+ Assertions.assertTrue(incremental.tryAcquire(job, 100, 100, 100));
+ }
+ LanceIndexJobQuota rebuilt = new LanceIndexJobQuota();
+ rebuilt.rebuild(unresolved);
+
+ Assertions.assertEquals(incremental.getGlobalCount(), rebuilt.getGlobalCount());
+ Assertions.assertEquals(incremental.getCatalogCount(CATALOG_ID), rebuilt.getCatalogCount(CATALOG_ID));
+ Assertions.assertEquals(incremental.getCatalogCount(20L), rebuilt.getCatalogCount(20L));
+ Assertions.assertEquals(incremental.getTableCount(unresolved.get(0).getTableQuotaKey()),
+ rebuilt.getTableCount(unresolved.get(0).getTableQuotaKey()));
+ Assertions.assertEquals(incremental.getTableCount(unresolved.get(2).getTableQuotaKey()),
+ rebuilt.getTableCount(unresolved.get(2).getTableQuotaKey()));
+
+ rebuilt.rebuild(Collections.emptyList());
+ Assertions.assertEquals(0L, rebuilt.getGlobalCount());
+ Assertions.assertEquals(0L, rebuilt.getCatalogCount(CATALOG_ID));
+ }
+
+ @Test
+ public void rebuildCountsUnresolvedJobsOnly() {
+ List jobs = new ArrayList<>();
+ // Active and unforced-unknown jobs count.
+ jobs.add(jobInState(1L, LanceIndexJobMutationState.PENDING, LanceIndexJobRefreshState.NOT_REQUIRED, false));
+ jobs.add(jobInState(2L, LanceIndexJobMutationState.RUNNING, LanceIndexJobRefreshState.NOT_REQUIRED, false));
+ jobs.add(jobInState(3L, LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, false));
+ jobs.add(jobInState(4L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.REQUIRED, false));
+ jobs.add(jobInState(5L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.RUNNING, false));
+ jobs.add(jobInState(6L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.FAILED, false));
+ jobs.add(jobInState(7L, LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.FAILED, false));
+ // Resolved jobs do not count: forced UNKNOWN and terminal jobs with a settled refresh.
+ jobs.add(jobInState(8L, LanceIndexJobMutationState.UNKNOWN, LanceIndexJobRefreshState.NOT_REQUIRED, true));
+ jobs.add(jobInState(9L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.DONE, false));
+ jobs.add(jobInState(10L, LanceIndexJobMutationState.COMMITTED, LanceIndexJobRefreshState.NOT_REQUIRED, false));
+ jobs.add(jobInState(11L, LanceIndexJobMutationState.NOT_COMMITTED, LanceIndexJobRefreshState.NOT_REQUIRED,
+ false));
+
+ List unresolved = new ArrayList<>();
+ for (LanceIndexJob job : jobs) {
+ if (job.isUnresolved()) {
+ unresolved.add(job);
+ }
+ }
+ Assertions.assertEquals(7, unresolved.size());
+
+ LanceIndexJobQuota quota = new LanceIndexJobQuota();
+ quota.rebuild(unresolved);
+ Assertions.assertEquals(7L, quota.getGlobalCount());
+ Assertions.assertEquals(7L, quota.getCatalogCount(CATALOG_ID));
+ Assertions.assertEquals(7L, quota.getTableCount(unresolved.get(0).getTableQuotaKey()));
+ }
+
+ @Test
+ public void tableQuotaKeyIdentityAndLocatorHiding() {
+ LanceIndexJobQuota.TableQuotaKey key = new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A);
+ Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A), key);
+ Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_A).hashCode(), key.hashCode());
+ Assertions.assertNotEquals(new LanceIndexJobQuota.TableQuotaKey(20L, LOCATOR_A), key);
+ Assertions.assertNotEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR_B), key);
+ Assertions.assertFalse(key.toString().contains(LOCATOR_A));
+ }
+
+ private static LanceIndexJob newJob(long jobId, long catalogId, String locator) {
+ return new LanceIndexJob(jobId, "tester", catalogId, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, locator,
+ "idx" + jobId, "idx" + jobId,
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 1L, null);
+ }
+
+ private static LanceIndexJob jobInState(long jobId, LanceIndexJobMutationState mutationState,
+ LanceIndexJobRefreshState refreshState, boolean forceReleased) {
+ LanceIndexJob job = newJob(jobId, CATALOG_ID, LOCATOR_A);
+ job.setMutationState(mutationState);
+ job.setRefreshState(refreshState);
+ job.setForceReleased(forceReleased);
+ return job;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java
new file mode 100644
index 00000000000000..cbceb49caf1f05
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobResultClassifyTest.java
@@ -0,0 +1,239 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Exhaustive coverage of the provider-result classification table: every saved typed
+ * result code is classified into the durable (mutationState, refreshState,
+ * completionReason) triple for the full cross product of 13 codes x 3 mutation types
+ * x ifExists x externalMetadataAdvanced (156 combinations). The expected values are an
+ * independent restatement of the design table, so a row flipped on either side fails.
+ *
+ * Key invariants pinned here: PRE_INVOCATION_* always prove NOT_COMMITTED and owe a
+ * refresh only when trusted revalidation observed external metadata advancement;
+ * NATIVE_NOT_FOUND has no attribution for CREATE/REPLACE (UNKNOWN) but is a clean
+ * NOT_COMMITTED for DROP; IF_CONDITION_NOOP exists only for DROP IF EXISTS + NOT_FOUND;
+ * every other ambiguous post-invocation outcome is UNKNOWN + NOT_REQUIRED.
+ */
+public class LanceIndexJobResultClassifyTest {
+
+ @Test
+ public void fullClassificationProductMatchesDesignTable() {
+ int combos = 0;
+ for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ for (boolean ifExists : new boolean[]{false, true}) {
+ for (boolean advanced : new boolean[]{false, true}) {
+ LanceIndexJobResultCode.Classification classification =
+ LanceIndexJobResultCode.classify(type, code, ifExists, advanced);
+ String context = "code=" + code + ", type=" + type
+ + ", ifExists=" + ifExists + ", advanced=" + advanced;
+ Assertions.assertEquals(expectedMutationState(type, code),
+ classification.getMutationState(), context);
+ Assertions.assertEquals(expectedRefreshState(type, code, advanced),
+ classification.getRefreshState(), context);
+ Assertions.assertEquals(expectedCompletionReason(type, code, ifExists),
+ classification.getCompletionReason(), context);
+ combos++;
+ }
+ }
+ }
+ }
+ Assertions.assertEquals(13 * 3 * 2 * 2, combos);
+ }
+
+ @Test
+ public void ifConditionNoopExistsOnlyForDropIfExistsNotFound() {
+ int noopCombos = 0;
+ for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ for (boolean ifExists : new boolean[]{false, true}) {
+ for (boolean advanced : new boolean[]{false, true}) {
+ LanceIndexJobResultCode.Classification classification =
+ LanceIndexJobResultCode.classify(type, code, ifExists, advanced);
+ if (classification.getCompletionReason() == LanceIndexJobCompletionReason.IF_CONDITION_NOOP) {
+ noopCombos++;
+ Assertions.assertEquals(LanceIndexJobMutationType.DROP, type);
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_NOT_FOUND, code);
+ Assertions.assertTrue(ifExists);
+ }
+ }
+ }
+ }
+ }
+ // Exactly the two advanced-flag variants of (DROP, NATIVE_NOT_FOUND, ifExists).
+ Assertions.assertEquals(2, noopCombos);
+ }
+
+ @Test
+ public void preInvocationRefreshOwesOnlyWhenExternalMetadataAdvanced() {
+ for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
+ if (!code.isPreInvocation()) {
+ continue;
+ }
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED,
+ LanceIndexJobResultCode.classify(type, code, false, false).getRefreshState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED,
+ LanceIndexJobResultCode.classify(type, code, false, true).getRefreshState());
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED,
+ LanceIndexJobResultCode.classify(type, code, true, true).getMutationState());
+ }
+ }
+ }
+
+ @Test
+ public void nativeOkCommitsAndOwesRefreshForEveryMutationType() {
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ LanceIndexJobResultCode.Classification classification =
+ LanceIndexJobResultCode.classify(type, LanceIndexJobResultCode.NATIVE_OK, false, false);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, classification.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, classification.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
+ }
+ }
+
+ @Test
+ public void commitConflictIsNotCommittedWithRefreshOwed() {
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify(
+ type, LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT, false, false);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, classification.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, classification.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
+ }
+ }
+
+ @Test
+ public void nativeNotFoundAttributionDependsOnMutationType() {
+ for (LanceIndexJobMutationType type : new LanceIndexJobMutationType[]{
+ LanceIndexJobMutationType.CREATE, LanceIndexJobMutationType.REPLACE}) {
+ LanceIndexJobResultCode.Classification classification = LanceIndexJobResultCode.classify(
+ type, LanceIndexJobResultCode.NATIVE_NOT_FOUND, false, false);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
+ }
+ LanceIndexJobResultCode.Classification dropClassification = LanceIndexJobResultCode.classify(
+ LanceIndexJobMutationType.DROP, LanceIndexJobResultCode.NATIVE_NOT_FOUND, false, false);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, dropClassification.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, dropClassification.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, dropClassification.getCompletionReason());
+ }
+
+ @Test
+ public void ambiguousPostInvocationOutcomesAreUnknownWithoutRefresh() {
+ for (LanceIndexJobResultCode code : new LanceIndexJobResultCode[]{
+ LanceIndexJobResultCode.NATIVE_INVALID_ARGUMENT, LanceIndexJobResultCode.NATIVE_NOT_SUPPORTED,
+ LanceIndexJobResultCode.NATIVE_INDEX, LanceIndexJobResultCode.NATIVE_IO,
+ LanceIndexJobResultCode.NATIVE_INTERNAL, LanceIndexJobResultCode.NO_TRUSTED_RESULT}) {
+ for (LanceIndexJobMutationType type : LanceIndexJobMutationType.values()) {
+ LanceIndexJobResultCode.Classification classification =
+ LanceIndexJobResultCode.classify(type, code, true, true);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState(),
+ "code=" + code + ", type=" + type);
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState(),
+ "code=" + code + ", type=" + type);
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason(),
+ "code=" + code + ", type=" + type);
+ }
+ }
+ }
+
+ @Test
+ public void nullResultCodeFallsBackToNoTrustedResult() {
+ LanceIndexJobResultCode.Classification classification =
+ LanceIndexJobResultCode.classify(LanceIndexJobMutationType.CREATE, null, false, false);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, classification.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, classification.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, classification.getCompletionReason());
+ }
+
+ @Test
+ public void isPreInvocationCoversExactlyTheFourTrustedRejections() {
+ int preInvocationCount = 0;
+ for (LanceIndexJobResultCode code : LanceIndexJobResultCode.values()) {
+ if (code.isPreInvocation()) {
+ preInvocationCount++;
+ }
+ }
+ Assertions.assertEquals(4, preInvocationCount);
+ Assertions.assertEquals(13, LanceIndexJobResultCode.values().length);
+ }
+
+ /**
+ * Independent restatement of the design table, mutation-state column.
+ */
+ private static LanceIndexJobMutationState expectedMutationState(
+ LanceIndexJobMutationType type, LanceIndexJobResultCode code) {
+ if (isPreInvocationSpec(code)) {
+ return LanceIndexJobMutationState.NOT_COMMITTED;
+ }
+ switch (code) {
+ case NATIVE_OK:
+ return LanceIndexJobMutationState.COMMITTED;
+ case NATIVE_COMMIT_CONFLICT:
+ return LanceIndexJobMutationState.NOT_COMMITTED;
+ case NATIVE_NOT_FOUND:
+ return type == LanceIndexJobMutationType.DROP
+ ? LanceIndexJobMutationState.NOT_COMMITTED : LanceIndexJobMutationState.UNKNOWN;
+ default:
+ return LanceIndexJobMutationState.UNKNOWN;
+ }
+ }
+
+ /**
+ * Independent restatement of the design table, refresh-obligation column.
+ */
+ private static LanceIndexJobRefreshState expectedRefreshState(
+ LanceIndexJobMutationType type, LanceIndexJobResultCode code, boolean externalMetadataAdvanced) {
+ if (isPreInvocationSpec(code)) {
+ return externalMetadataAdvanced
+ ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED;
+ }
+ switch (code) {
+ case NATIVE_OK:
+ case NATIVE_COMMIT_CONFLICT:
+ return LanceIndexJobRefreshState.REQUIRED;
+ case NATIVE_NOT_FOUND:
+ return type == LanceIndexJobMutationType.DROP
+ ? LanceIndexJobRefreshState.REQUIRED : LanceIndexJobRefreshState.NOT_REQUIRED;
+ default:
+ return LanceIndexJobRefreshState.NOT_REQUIRED;
+ }
+ }
+
+ /**
+ * Independent restatement of the design table, completion-reason column.
+ */
+ private static LanceIndexJobCompletionReason expectedCompletionReason(
+ LanceIndexJobMutationType type, LanceIndexJobResultCode code, boolean ifExists) {
+ return type == LanceIndexJobMutationType.DROP && code == LanceIndexJobResultCode.NATIVE_NOT_FOUND && ifExists
+ ? LanceIndexJobCompletionReason.IF_CONDITION_NOOP : LanceIndexJobCompletionReason.NONE;
+ }
+
+ private static boolean isPreInvocationSpec(LanceIndexJobResultCode code) {
+ return code == LanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION
+ || code == LanceIndexJobResultCode.PRE_INVOCATION_UNSUPPORTED_SCHEMA_CONTRACT
+ || code == LanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED
+ || code == LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java
new file mode 100644
index 00000000000000..54a5945eb79b16
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobStateMachineTest.java
@@ -0,0 +1,640 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.common.DdlException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Lifecycle state-machine coverage driven through the master write paths of
+ * {@link LanceIndexJobManager} with the edit-log seam captured in memory. Covers the
+ * legal mutation chain PENDING -> RUNNING -> (COMMITTED | NOT_COMMITTED |
+ * UNKNOWN), the impossibility of leaving UNKNOWN, the independent refresh chain
+ * REQUIRED -> RUNNING -> DONE / FAILED with FAILED -> RUNNING retry, the
+ * revision compare-and-set on every transition, and the fence/quota release timing
+ * (immediately on a terminal outcome with refresh NOT_REQUIRED, only at refresh DONE
+ * otherwise, never for UNKNOWN without FORCE).
+ */
+public class LanceIndexJobStateMachineTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final String INVOCATION_ID = "invocation-1";
+ private static final long DEADLINE_MS = 9999L;
+
+ @Test
+ public void createInitializesPendingRecord() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(0L, stored.getRevision());
+ Assertions.assertFalse(stored.isPossibleLiveOwned());
+ Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof());
+ Assertions.assertTrue(stored.getCreateTimeMs() > 0L);
+ Assertions.assertTrue(manager.isFenceHeld(stored.fenceKey()));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertEquals(1L, manager.getQuota().getCatalogCount(CATALOG_ID));
+ Assertions.assertEquals(1L, manager.getQuota().getTableCount(stored.getTableQuotaKey()));
+ Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
+ Assertions.assertEquals(1, manager.editLog.size());
+ }
+
+ @Test
+ public void pendingToRunningToCommitted() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+
+ LanceIndexJob running = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, running.getMutationState());
+ Assertions.assertEquals(1L, running.getRevision());
+ Assertions.assertEquals(BACKEND_ID, running.getBackendId().longValue());
+ Assertions.assertEquals(BE_EPOCH, running.getBeProcessEpoch().longValue());
+ Assertions.assertEquals(INVOCATION_ID, running.getInvocationId());
+ Assertions.assertEquals(DEADLINE_MS, running.getDeadlineMs().longValue());
+ Assertions.assertTrue(running.holdsPossibleLiveSlot());
+
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ LanceIndexJob committed = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, committed.getRefreshState());
+ Assertions.assertEquals(2L, committed.getRevision());
+ Assertions.assertEquals(LanceIndexJobResultCode.NATIVE_OK, committed.getResult().getResultCode());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.NONE, committed.getResult().getCompletionReason());
+ Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), committed.getJobId()));
+ }
+
+ @Test
+ public void pendingToRunningToNotCommitted() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.PRE_INVOCATION_CREDENTIAL_EXPIRED)));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(2L, stored.getRevision());
+ }
+
+ @Test
+ public void pendingToRunningToUnknown() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NO_TRUSTED_RESULT)));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(2L, stored.getRevision());
+ }
+
+ @Test
+ public void dropIfExistsNotFoundCompletesWithIfConditionNoop() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newDropJob(1L, "IdxA", true), 100, 100, 100);
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_NOT_FOUND)));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(LanceIndexJobCompletionReason.IF_CONDITION_NOOP,
+ stored.getResult().getCompletionReason());
+ }
+
+ @Test
+ public void markRunningRejectsSecondDispatch() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+
+ Assertions.assertFalse(manager.markRunning(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(1L, manager.getJob(1L).getRevision());
+ }
+
+ @Test
+ public void markRunningRejectsWrongRevisionAndUnknownJob() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ Assertions.assertFalse(manager.markRunning(1L, 5L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(manager.markRunning(404L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ Assertions.assertEquals(1, manager.editLog.size());
+ }
+
+ @Test
+ public void completeRejectsNonRunningJob() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ Assertions.assertFalse(manager.completeWithResult(1L, 0L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+
+ createAndRun(manager, 2L, "IdxB");
+ Assertions.assertTrue(manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertFalse(manager.completeWithResult(2L, 2L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, manager.getJob(2L).getMutationState());
+ }
+
+ @Test
+ public void unknownHasNoOutgoingTransitions() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NO_TRUSTED_RESULT)));
+ int loggedRecords = manager.editLog.size();
+
+ Assertions.assertFalse(manager.markRunning(1L, 2L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertFalse(manager.completeWithResult(1L, 2L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertFalse(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertFalse(manager.markRefreshDone(1L, 2L));
+ Assertions.assertFalse(manager.markRefreshFailed(1L, 2L));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, stored.getMutationState());
+ Assertions.assertEquals(2L, stored.getRevision());
+ Assertions.assertEquals(loggedRecords, manager.editLog.size());
+ }
+
+ @Test
+ public void refreshLifecycleReleasesFenceAndQuotaAtDone() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, manager.getJob(1L).getRefreshState());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+
+ Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, stored.getRefreshState());
+ Assertions.assertEquals(4L, stored.getRevision());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
+ Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
+ }
+
+ @Test
+ public void refreshFailureKeepsFenceAndRetriesThroughRunning() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertTrue(manager.markRefreshFailed(1L, 3L));
+ LanceIndexJob failed = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobRefreshState.FAILED, failed.getRefreshState());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(containsJob(manager.getUnresolvedJobs(), failed.getJobId()));
+
+ // FAILED -> RUNNING is the retry entry through the idempotent refresh path.
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 4L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.RUNNING, manager.getJob(1L).getRefreshState());
+ Assertions.assertTrue(manager.markRefreshDone(1L, 5L));
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void refreshTransitionsRejectedFromNotRequired() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED)));
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(1L).getRefreshState());
+
+ Assertions.assertFalse(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertFalse(manager.markRefreshDone(1L, 2L));
+ Assertions.assertFalse(manager.markRefreshFailed(1L, 2L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(1L).getRefreshState());
+ Assertions.assertEquals(2L, manager.getJob(1L).getRevision());
+ }
+
+ @Test
+ public void refreshTransitionsRejectedFromDone() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
+
+ Assertions.assertFalse(manager.markRefreshRunning(1L, 4L));
+ Assertions.assertFalse(manager.markRefreshDone(1L, 4L));
+ Assertions.assertFalse(manager.markRefreshFailed(1L, 4L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.DONE, manager.getJob(1L).getRefreshState());
+ }
+
+ @Test
+ public void refreshTransitionIsRevisionGuarded() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+
+ Assertions.assertFalse(manager.markRefreshRunning(1L, 99L));
+ Assertions.assertFalse(manager.markRefreshRunning(404L, 2L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
+ Assertions.assertEquals(2L, manager.getJob(1L).getRevision());
+ }
+
+ @Test
+ public void fenceAndQuotaReleasedImmediatelyWhenRefreshNotRequired() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.PRE_INVOCATION_STALE_ADMISSION)));
+
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, manager.getJob(1L).getMutationState());
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertEquals(0L, manager.getQuota().getCatalogCount(CATALOG_ID));
+ Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
+ }
+
+ @Test
+ public void fenceAndQuotaSurviveUnknown() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_IO)));
+
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, manager.getJob(1L).getMutationState());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertEquals(1, manager.getUnresolvedJobs().size());
+ Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
+ }
+
+ @Test
+ public void terminationProofReleasesSlotOnly() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+ Assertions.assertTrue(manager.getJob(1L).holdsPossibleLiveSlot());
+
+ Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.NONE));
+ Assertions.assertFalse(manager.recordTerminationProof(1L, 99L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexTerminationProof.CHILD_REAPED, stored.getTerminationProof());
+ Assertions.assertFalse(stored.holdsPossibleLiveSlot());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+
+ // A slot may be proven exactly once.
+ Assertions.assertFalse(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE));
+ }
+
+ @Test
+ public void everyAcceptedTransitionWritesExactlyOneEditLogRecord() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS);
+ manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH, result(LanceIndexJobResultCode.NATIVE_OK));
+ manager.markRefreshRunning(1L, 2L);
+ manager.markRefreshDone(1L, 3L);
+ Assertions.assertEquals(5, manager.editLog.size());
+
+ // Rejected transitions never reach the journal.
+ Assertions.assertFalse(manager.markRefreshDone(1L, 4L));
+ Assertions.assertFalse(manager.markRunning(1L, 4L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertEquals(5, manager.editLog.size());
+ }
+
+ @Test
+ public void markRunningRejectsBlankInvocationId() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ // A null/blank invocation identity would match a null field under Objects.equals
+ // in completeWithResult and silently defeat the stale-callback guard.
+ Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, null, DEADLINE_MS));
+ Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, "", DEADLINE_MS));
+ Assertions.assertFalse(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, " \t\n", DEADLINE_MS));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
+ Assertions.assertEquals(0L, stored.getRevision());
+ Assertions.assertNull(stored.getInvocationId());
+ Assertions.assertEquals(1, manager.editLog.size());
+
+ // A well-formed dispatch is still accepted afterwards.
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, manager.getJob(1L).getMutationState());
+ }
+
+ @Test
+ public void failedRefreshJobStaysVisibleToTheRefreshDriver() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertTrue(manager.markRefreshFailed(1L, 3L));
+
+ // FAILED still owes the idempotent retry: the driver must see the job.
+ Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), 1L));
+
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 4L));
+ Assertions.assertTrue(manager.markRefreshDone(1L, 5L));
+ Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
+
+ // Terminal jobs with refresh DONE or NOT_REQUIRED never show up.
+ createAndRun(manager, 2L, "IdxB");
+ Assertions.assertTrue(manager.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.PRE_INVOCATION_RESOURCE_REJECTED)));
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, manager.getJob(2L).getRefreshState());
+ Assertions.assertTrue(manager.getJobsNeedingRefresh().isEmpty());
+ }
+
+ @Test
+ public void createJobResetsLifecycleFieldsAndPublishesAPrivateCopy() throws DdlException {
+ LanceIndexJob dirty = newCreateJob(1L, "IdxA");
+ dirty.setRevision(9L);
+ dirty.setMutationState(LanceIndexJobMutationState.RUNNING);
+ dirty.setRefreshState(LanceIndexJobRefreshState.RUNNING);
+ dirty.setResult(result(LanceIndexJobResultCode.NATIVE_OK));
+ dirty.setBackendId(BACKEND_ID);
+ dirty.setBeProcessEpoch(BE_EPOCH);
+ dirty.setInvocationId(INVOCATION_ID);
+ dirty.setDeadlineMs(DEADLINE_MS);
+ dirty.setPossibleLiveOwned(true);
+ dirty.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED);
+ dirty.setForceReleased(true);
+ dirty.setForceActor("admin");
+ dirty.setForceTimeMs(7L);
+ dirty.setForceNote("note");
+ dirty.setForceWarning("warning");
+
+ TestManager manager = new TestManager();
+ manager.createJob(dirty, 100, 100, 100);
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertNotSame(dirty, stored);
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.NOT_REQUIRED, stored.getRefreshState());
+ Assertions.assertEquals(0L, stored.getRevision());
+ Assertions.assertNull(stored.getResult());
+ Assertions.assertNull(stored.getBackendId());
+ Assertions.assertNull(stored.getBeProcessEpoch());
+ Assertions.assertNull(stored.getInvocationId());
+ Assertions.assertNull(stored.getDeadlineMs());
+ Assertions.assertFalse(stored.isPossibleLiveOwned());
+ Assertions.assertEquals(LanceIndexTerminationProof.NONE, stored.getTerminationProof());
+ Assertions.assertFalse(stored.isForceReleased());
+ Assertions.assertNull(stored.getForceActor());
+ Assertions.assertNull(stored.getForceTimeMs());
+ Assertions.assertNull(stored.getForceNote());
+ Assertions.assertNull(stored.getForceWarning());
+
+ // Mutating the caller's object after admission touches nothing inside the manager.
+ dirty.setMutationState(LanceIndexJobMutationState.UNKNOWN);
+ dirty.setNormalizedIndexName("idxb");
+ Assertions.assertEquals(LanceIndexJobMutationState.PENDING, manager.getJob(1L).getMutationState());
+ Assertions.assertTrue(manager.isFenceHeld(newCreateJob(2L, "idxa").fenceKey()));
+ Assertions.assertFalse(manager.isFenceHeld(newCreateJob(3L, "IdxB").fenceKey()));
+ }
+
+ @Test
+ public void refreshDoneOrFailedRequiresAPrecedingRefreshRunning() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
+
+ // No direct REQUIRED -> DONE / FAILED: a refresh must actually run first.
+ Assertions.assertFalse(manager.markRefreshDone(1L, 2L));
+ Assertions.assertFalse(manager.markRefreshFailed(1L, 2L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
+ Assertions.assertEquals(2L, manager.getJob(1L).getRevision());
+ Assertions.assertEquals(3, manager.editLog.size());
+ }
+
+ @Test
+ public void terminationProofNeedsASlotAndStillLandsAfterTheTerminalResult() throws DdlException {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+
+ // PENDING owns no possible-live slot: a proof has nothing to release.
+ Assertions.assertFalse(manager.recordTerminationProof(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertEquals(0L, manager.getJob(1L).getRevision());
+ Assertions.assertEquals(1, manager.editLog.size());
+
+ Assertions.assertTrue(manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ // The terminal outcome does not release the slot; the proof still lands afterwards.
+ LanceIndexJob committed = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED, committed.getMutationState());
+ Assertions.assertTrue(committed.holdsPossibleLiveSlot());
+ LanceIndexFenceKey fenceKey = committed.fenceKey();
+
+ Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+ LanceIndexJob proven = manager.getJob(1L);
+ Assertions.assertFalse(proven.holdsPossibleLiveSlot());
+ // Fence and quota still follow the refresh rule, not the proof.
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 3L));
+ Assertions.assertTrue(manager.markRefreshDone(1L, 4L));
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ }
+
+ @Test
+ public void terminationProofAndResultUseTheImmutableDispatchRevisionInEitherOrder() throws DdlException {
+ TestManager proofFirst = new TestManager();
+ createAndRun(proofFirst, 1L, "IdxA");
+ Assertions.assertEquals(1L, proofFirst.getJob(1L).getDispatchRevision());
+
+ Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID + 1, BE_EPOCH,
+ INVOCATION_ID, LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH + 1,
+ INVOCATION_ID, LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertFalse(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH,
+ "wrong-invocation", LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertEquals(1L, proofFirst.getJob(1L).getRevision());
+
+ Assertions.assertTrue(proofFirst.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertEquals(2L, proofFirst.getJob(1L).getRevision());
+ Assertions.assertEquals(1L, proofFirst.getJob(1L).getDispatchRevision());
+ Assertions.assertTrue(proofFirst.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertEquals(LanceIndexJobMutationState.COMMITTED,
+ proofFirst.getJob(1L).getMutationState());
+
+ TestManager resultFirst = new TestManager();
+ createAndRun(resultFirst, 2L, "IdxB");
+ Assertions.assertTrue(resultFirst.completeWithResult(2L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertTrue(resultFirst.recordTerminationProof(2L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertFalse(resultFirst.getJob(2L).holdsPossibleLiveSlot());
+ Assertions.assertEquals(1L, resultFirst.getJob(2L).getDispatchRevision());
+
+ TestManager legacyResultFirst = new TestManager();
+ createAndRun(legacyResultFirst, 3L, "IdxC");
+ LanceIndexJob legacyRunning = legacyResultFirst.getJob(3L);
+ legacyRunning.setDispatchRevision(null);
+ legacyResultFirst.replayUpsertJob(legacyRunning);
+ Assertions.assertTrue(legacyResultFirst.completeWithResult(3L, 1L, INVOCATION_ID, BE_EPOCH,
+ result(LanceIndexJobResultCode.NATIVE_OK)));
+ Assertions.assertEquals(1L, legacyResultFirst.getJob(3L).getDispatchRevision());
+ Assertions.assertTrue(legacyResultFirst.recordTerminationProof(3L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.CHILD_REAPED));
+ Assertions.assertFalse(legacyResultFirst.getJob(3L).holdsPossibleLiveSlot());
+ }
+
+ @Test
+ public void refreshTransitionsRejectNonTerminalMutationState() {
+ TestManager manager = new TestManager();
+ LanceIndexJob corrupt = newCreateJob(1L, "IdxA");
+ corrupt.setMutationState(LanceIndexJobMutationState.PENDING);
+ corrupt.setRefreshState(LanceIndexJobRefreshState.REQUIRED);
+ manager.replayUpsertJob(corrupt);
+
+ Assertions.assertFalse(manager.markRefreshRunning(1L, 0L));
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, manager.getJob(1L).getRefreshState());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ }
+
+ @Test
+ public void admissionRejectsNonPositiveQuotaLimitsBeforeDurableWrite() {
+ TestManager manager = new TestManager();
+ Assertions.assertThrows(DdlException.class,
+ () -> manager.createJob(newCreateJob(1L, "IdxA"), 0, 1, 1));
+ Assertions.assertThrows(DdlException.class,
+ () -> manager.createJob(newCreateJob(1L, "IdxA"), 1, 0, 1));
+ Assertions.assertThrows(DdlException.class,
+ () -> manager.createJob(newCreateJob(1L, "IdxA"), 1, 1, -1));
+ Assertions.assertEquals(0, manager.getJobCount());
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(manager.editLog.isEmpty());
+ }
+
+ @Test
+ public void notCommittedWithRefreshRequiredReleasesFenceAtRefreshDone() throws DdlException {
+ TestManager manager = new TestManager();
+ createAndRun(manager, 1L, "IdxA");
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_COMMIT_CONFLICT,
+ LanceIndexJobCompletionReason.NONE, "commit conflict", false)));
+
+ LanceIndexJob stored = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.NOT_COMMITTED, stored.getMutationState());
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, stored.getRefreshState());
+ LanceIndexFenceKey fenceKey = stored.fenceKey();
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(containsJob(manager.getJobsNeedingRefresh(), stored.getJobId()));
+
+ Assertions.assertTrue(manager.markRefreshRunning(1L, 2L));
+ Assertions.assertTrue(manager.markRefreshDone(1L, 3L));
+ Assertions.assertFalse(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(0L, manager.getQuota().getGlobalCount());
+ Assertions.assertTrue(manager.getUnresolvedJobs().isEmpty());
+ }
+
+ private static LanceIndexJob newCreateJob(long jobId, String displayName) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
+ null, 7L, null);
+ }
+
+ private static boolean containsJob(List jobs, long jobId) {
+ return jobs.stream().anyMatch(job -> job.getJobId() == jobId);
+ }
+
+ private static LanceIndexJob newDropJob(long jobId, String displayName, boolean ifExists) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.DROP, false, ifExists, null, "v",
+ null, 7L, null);
+ }
+
+ private static LanceIndexJobResult result(LanceIndexJobResultCode code) {
+ return new LanceIndexJobResult(code, LanceIndexJobCompletionReason.NONE, "sanitized message", false);
+ }
+
+ private static LanceIndexJob createAndRun(TestManager manager, long jobId, String displayName) throws DdlException {
+ manager.createJob(newCreateJob(jobId, displayName), 100, 100, 100);
+ Assertions.assertTrue(manager.markRunning(jobId, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, DEADLINE_MS));
+ return manager.getJob(jobId);
+ }
+
+ /**
+ * Edit-log seam: captures every durable record instead of writing the journal.
+ */
+ private static class TestManager extends LanceIndexJobManager {
+ private final List editLog = new ArrayList<>();
+
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ editLog.add(job);
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java
new file mode 100644
index 00000000000000..1f61b184d196af
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobTest.java
@@ -0,0 +1,303 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.persist.gson.GsonUtils;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Record-level coverage for {@link LanceIndexJob}: the derived fence and quota keys,
+ * the full {@link LanceIndexJob#isUnresolved()} matrix that keeps fence and quota
+ * alive together, the possible-live slot semantics (released only by a matching
+ * termination proof or a durable FORCE_RELEASE, never by a deadline), corrupt-record
+ * fallbacks toward the safe direction, and the deep-copy constructor.
+ */
+public class LanceIndexJobTest {
+ private static final long CATALOG_ID = 10L;
+ private static final String LOCATOR = "s3://bucket/dataset";
+ private static final long BACKEND_ID = 1001L;
+ private static final long BE_EPOCH = 55L;
+ private static final String INVOCATION_ID = "invocation-1";
+
+ @Test
+ public void fenceKeyCarriesIdentityAndHidesLocator() {
+ LanceIndexJob job = newCreateJob(1L, "IdxA");
+ LanceIndexFenceKey key = job.fenceKey();
+
+ Assertions.assertEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
+ LOCATOR, "idxa"), key);
+ Assertions.assertEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
+ LOCATOR, "idxa").hashCode(), key.hashCode());
+ Assertions.assertNotEquals(new LanceIndexFenceKey(20L, LanceIndexFenceKey.PROVIDER_DIRECTORY,
+ LOCATOR, "idxa"), key);
+ Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, "REST", LOCATOR, "idxa"), key);
+ Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
+ "s3://bucket/other", "idxa"), key);
+ Assertions.assertNotEquals(new LanceIndexFenceKey(CATALOG_ID, LanceIndexFenceKey.PROVIDER_DIRECTORY,
+ LOCATOR, "idxb"), key);
+
+ // Fence-conflict messages may surface to users without target privileges.
+ Assertions.assertFalse(key.toString().contains(LOCATOR));
+ Assertions.assertTrue(key.toString().contains("idxa"));
+ }
+
+ @Test
+ public void fenceKeyIsStableAcrossJobsWithTheSameIdentity() {
+ LanceIndexJob first = newCreateJob(1L, "IdxA");
+ LanceIndexJob second = newCreateJob(2L, "idxa");
+ // Display case differs, normalized identity does not.
+ Assertions.assertEquals(first.fenceKey(), second.fenceKey());
+ Assertions.assertEquals(first.fenceKey().hashCode(), second.fenceKey().hashCode());
+ }
+
+ @Test
+ public void tableQuotaKeyDerivesFromCatalogAndLocator() {
+ LanceIndexJob job = newCreateJob(1L, "IdxA");
+ Assertions.assertEquals(new LanceIndexJobQuota.TableQuotaKey(CATALOG_ID, LOCATOR), job.getTableQuotaKey());
+ }
+
+ @Test
+ public void activeStatesAreAlwaysUnresolved() {
+ for (LanceIndexJobRefreshState refresh : LanceIndexJobRefreshState.values()) {
+ Assertions.assertTrue(jobInState(LanceIndexJobMutationState.PENDING, refresh, false).isUnresolved());
+ Assertions.assertTrue(jobInState(LanceIndexJobMutationState.RUNNING, refresh, false).isUnresolved());
+ }
+ }
+
+ @Test
+ public void knownTerminalStatesFollowTheRefreshState() {
+ for (LanceIndexJobMutationState terminal : new LanceIndexJobMutationState[]{
+ LanceIndexJobMutationState.COMMITTED, LanceIndexJobMutationState.NOT_COMMITTED}) {
+ Assertions.assertFalse(jobInState(terminal, LanceIndexJobRefreshState.NOT_REQUIRED, false).isUnresolved());
+ Assertions.assertFalse(jobInState(terminal, LanceIndexJobRefreshState.DONE, false).isUnresolved());
+ Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.REQUIRED, false).isUnresolved());
+ Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.RUNNING, false).isUnresolved());
+ // A failed refresh still holds the fence: the retry goes through the idempotent path.
+ Assertions.assertTrue(jobInState(terminal, LanceIndexJobRefreshState.FAILED, false).isUnresolved());
+ }
+ }
+
+ @Test
+ public void unknownIsUnresolvedUntilForceReleased() {
+ for (LanceIndexJobRefreshState refresh : LanceIndexJobRefreshState.values()) {
+ Assertions.assertTrue(jobInState(LanceIndexJobMutationState.UNKNOWN, refresh, false).isUnresolved());
+ Assertions.assertFalse(jobInState(LanceIndexJobMutationState.UNKNOWN, refresh, true).isUnresolved());
+ }
+ }
+
+ @Test
+ public void nullStatesFallBackToTheSafeDirection() {
+ // A corrupt record missing its states must keep the fence (treated as UNKNOWN)
+ // and must never become redispatchable PENDING.
+ LanceIndexJob job = GsonUtils.GSON.fromJson(
+ "{\"jid\":1,\"ms\":null,\"rs\":null}", LanceIndexJob.class);
+ Assertions.assertNull(job.getMutationState());
+ Assertions.assertNull(job.getRefreshState());
+ Assertions.assertTrue(job.isUnresolved());
+
+ LanceIndexJob forced = GsonUtils.GSON.fromJson(
+ "{\"jid\":1,\"ms\":null,\"rs\":null,\"fr\":true}", LanceIndexJob.class);
+ Assertions.assertFalse(forced.isUnresolved());
+
+ // A terminal record missing its refresh state owes one (treated as REQUIRED).
+ LanceIndexJob committedNoRefresh = GsonUtils.GSON.fromJson(
+ "{\"jid\":1,\"ms\":\"COMMITTED\",\"rs\":null}", LanceIndexJob.class);
+ Assertions.assertTrue(committedNoRefresh.isUnresolved());
+ }
+
+ @Test
+ public void missingRefreshStateKeyFallsBackToRequired() {
+ // A corrupt terminal record without the "rs" key at all keeps the fence: the
+ // field initial value is REQUIRED, the same safe direction as the null fallback.
+ LanceIndexJob missingKey = GsonUtils.GSON.fromJson(
+ "{\"jid\":1,\"ms\":\"COMMITTED\"}", LanceIndexJob.class);
+ Assertions.assertEquals(LanceIndexJobRefreshState.REQUIRED, missingKey.getRefreshState());
+ Assertions.assertTrue(missingKey.isUnresolved());
+
+ // A legal terminal record always carries the key explicitly.
+ LanceIndexJob legal = GsonUtils.GSON.fromJson(
+ "{\"jid\":1,\"ms\":\"COMMITTED\",\"rs\":\"NOT_REQUIRED\"}", LanceIndexJob.class);
+ Assertions.assertFalse(legal.isUnresolved());
+ }
+
+ @Test
+ public void possibleLiveSlotMatrix() {
+ LanceIndexJob job = new LanceIndexJob();
+ Assertions.assertFalse(job.holdsPossibleLiveSlot());
+
+ job.setPossibleLiveOwned(true);
+ Assertions.assertTrue(job.holdsPossibleLiveSlot());
+
+ job.setTerminationProof(LanceIndexTerminationProof.CHILD_REAPED);
+ Assertions.assertFalse(job.holdsPossibleLiveSlot());
+
+ job.setTerminationProof(LanceIndexTerminationProof.NONE);
+ job.setForceReleased(true);
+ Assertions.assertFalse(job.holdsPossibleLiveSlot());
+
+ // A corrupt record missing the proof is treated as NONE (slot still owned).
+ LanceIndexJob nullProof = GsonUtils.GSON.fromJson(
+ "{\"jid\":1,\"plo\":true,\"tp\":null}", LanceIndexJob.class);
+ Assertions.assertNull(nullProof.getTerminationProof());
+ Assertions.assertTrue(nullProof.holdsPossibleLiveSlot());
+ }
+
+ @Test
+ public void terminationProofClearsSlotButKeepsFenceAndOutcome() throws Exception {
+ TestManager manager = new TestManager();
+ manager.createJob(newCreateJob(1L, "IdxA"), 100, 100, 100);
+ manager.markRunning(1L, 0L, BACKEND_ID, BE_EPOCH, INVOCATION_ID, 9999L);
+ LanceIndexFenceKey fenceKey = manager.getJob(1L).fenceKey();
+
+ Assertions.assertTrue(manager.recordTerminationProof(1L, 1L, BACKEND_ID, BE_EPOCH, INVOCATION_ID,
+ LanceIndexTerminationProof.BE_PROCESS_EPOCH_GONE));
+ LanceIndexJob proven = manager.getJob(1L);
+ Assertions.assertFalse(proven.holdsPossibleLiveSlot());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, proven.getMutationState());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ Assertions.assertEquals(1L, manager.getQuota().getGlobalCount());
+
+ // The ambiguous result still lands afterwards: UNKNOWN keeps the fence, and the
+ // already-recorded proof keeps the slot released.
+ Assertions.assertTrue(manager.completeWithResult(1L, 1L, INVOCATION_ID, BE_EPOCH,
+ new LanceIndexJobResult(LanceIndexJobResultCode.NO_TRUSTED_RESULT,
+ LanceIndexJobCompletionReason.NONE, "ambiguous", false)));
+ LanceIndexJob unknown = manager.getJob(1L);
+ Assertions.assertEquals(LanceIndexJobMutationState.UNKNOWN, unknown.getMutationState());
+ Assertions.assertFalse(unknown.holdsPossibleLiveSlot());
+ Assertions.assertTrue(unknown.isUnresolved());
+ Assertions.assertTrue(manager.isFenceHeld(fenceKey));
+ }
+
+ @Test
+ public void copyConstructorDuplicatesEveryFieldIndependently() {
+ LanceIndexJob original = newCreateJob(1L, "IdxA");
+ original.setRevision(3L);
+ original.setMutationState(LanceIndexJobMutationState.RUNNING);
+ original.setRefreshState(LanceIndexJobRefreshState.RUNNING);
+ original.setResult(new LanceIndexJobResult(LanceIndexJobResultCode.NATIVE_OK,
+ LanceIndexJobCompletionReason.NONE, "ok", false));
+ original.setBackendId(BACKEND_ID);
+ original.setBeProcessEpoch(BE_EPOCH);
+ original.setDispatchRevision(1L);
+ original.setInvocationId(INVOCATION_ID);
+ original.setDeadlineMs(123L);
+ original.setPossibleLiveOwned(true);
+ original.setForceActor("admin");
+ original.setForceTimeMs(9L);
+ original.setForceNote("note");
+ original.setForceWarning("warning");
+
+ LanceIndexJob copy = new LanceIndexJob(original);
+ Assertions.assertEquals(GsonUtils.GSON.toJson(original), GsonUtils.GSON.toJson(copy));
+
+ copy.setRevision(99L);
+ copy.setMutationState(LanceIndexJobMutationState.UNKNOWN);
+ copy.setPossibleLiveOwned(false);
+ Assertions.assertEquals(3L, original.getRevision());
+ Assertions.assertEquals(LanceIndexJobMutationState.RUNNING, original.getMutationState());
+ Assertions.assertTrue(original.isPossibleLiveOwned());
+ }
+
+ @Test
+ public void admissionConstructorRejectsNullIdentity() {
+ Assertions.assertThrows(NullPointerException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ null, LOCATOR, "IdxA", "idxa",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(NullPointerException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, null, "IdxA", "idxa",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(NullPointerException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", "idxa",
+ null, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, null, "idxa",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", null,
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ }
+
+ @Test
+ public void admissionRejectsPseudoCanonicalFenceIdentity() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ "directory", LOCATOR, "IdxA", "idxa",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, "S3://bucket/dataset/", "IdxA", "idxa",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR, "IdxA", "IdxA",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> new LanceIndexJob(1L, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY,
+ "https://bucket.example/ds?X-Amz-Signature=secret", "IdxA", "idxa",
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v", null, 7L, null));
+ }
+
+ @Test
+ public void toStringNamesTheIndexButHidesTheLocator() {
+ LanceIndexJob job = newCreateJob(1L, "IdxA");
+ Assertions.assertTrue(job.toString().contains("IdxA"));
+ Assertions.assertFalse(job.toString().contains("bucket"));
+ Assertions.assertFalse(job.toString().contains(LOCATOR));
+ }
+
+ private static LanceIndexJob newCreateJob(long jobId, String displayName) {
+ return new LanceIndexJob(jobId, "tester", CATALOG_ID, "db1", "tbl1",
+ LanceIndexFenceKey.PROVIDER_DIRECTORY, LOCATOR,
+ displayName, LanceIndexNameNormalizer.normalize(displayName),
+ LanceIndexJobMutationType.CREATE, false, false, "IVF_PQ", "v",
+ null, 7L, null);
+ }
+
+ private static LanceIndexJob jobInState(LanceIndexJobMutationState mutationState,
+ LanceIndexJobRefreshState refreshState, boolean forceReleased) {
+ LanceIndexJob job = new LanceIndexJob();
+ job.setMutationState(mutationState);
+ job.setRefreshState(refreshState);
+ job.setForceReleased(forceReleased);
+ return job;
+ }
+
+ /**
+ * Edit-log seam: captures every durable record instead of writing the journal.
+ */
+ private static class TestManager extends LanceIndexJobManager {
+ private final List editLog = new ArrayList<>();
+
+ @Override
+ protected void writeEditLog(LanceIndexJob job) {
+ editLog.add(job);
+ }
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
new file mode 100644
index 00000000000000..ecbd2b42799bb1
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexJobWiringTest.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.io.CountingDataOutputStream;
+import org.apache.doris.persist.OperationType;
+import org.apache.doris.persist.meta.MetaPersistMethod;
+import org.apache.doris.persist.meta.PersistMetaModules;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.DataInputStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class LanceIndexJobWiringTest {
+ private static final short LANCE_INDEX_JOB_OPCODE = 500;
+ private static final String LANCE_INDEX_JOB_MODULE = "lanceIndexJobManager";
+
+ @Test
+ public void lanceIndexJobOpcodeIsUniquelyAssigned() throws IllegalAccessException {
+ List fieldsUsingOpcode = new ArrayList<>();
+ for (Field field : OperationType.class.getDeclaredFields()) {
+ if (Modifier.isStatic(field.getModifiers())
+ && field.getType() == short.class
+ && field.getShort(null) == LANCE_INDEX_JOB_OPCODE) {
+ fieldsUsingOpcode.add(field.getName());
+ }
+ }
+
+ Assertions.assertEquals(
+ Collections.singletonList("OP_LANCE_INDEX_JOB_UPSERT"),
+ fieldsUsingOpcode,
+ "operation code 500 must remain uniquely assigned to Lance index job upserts");
+ Assertions.assertEquals(LANCE_INDEX_JOB_OPCODE, OperationType.OP_LANCE_INDEX_JOB_UPSERT);
+ }
+
+ @Test
+ public void lanceIndexJobManagerIsTheLastBaseImageModuleWithEnvBindings() throws Exception {
+ Assertions.assertEquals(
+ LANCE_INDEX_JOB_MODULE,
+ PersistMetaModules.MODULE_NAMES.get(PersistMetaModules.MODULE_NAMES.size() - 1),
+ "new image modules must be appended without reordering existing base modules");
+
+ MetaPersistMethod persistMethod = PersistMetaModules.MODULES_MAP.get(LANCE_INDEX_JOB_MODULE);
+ Assertions.assertNotNull(persistMethod);
+
+ Method expectedReadMethod = Env.class.getDeclaredMethod(
+ "loadLanceIndexJobManager", DataInputStream.class, long.class);
+ Method expectedWriteMethod = Env.class.getDeclaredMethod(
+ "saveLanceIndexJobManager", CountingDataOutputStream.class, long.class);
+ Assertions.assertEquals(expectedReadMethod, persistMethod.readMethod);
+ Assertions.assertEquals(expectedWriteMethod, persistMethod.writeMethod);
+ Assertions.assertEquals(long.class, persistMethod.readMethod.getReturnType());
+ Assertions.assertEquals(long.class, persistMethod.writeMethod.getReturnType());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java
new file mode 100644
index 00000000000000..d2ad765697cf52
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/job/LanceIndexNameNormalizerTest.java
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lance.job;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+/**
+ * Unit coverage for index-name normalization v1: the fence-key identity is exactly
+ * {@code toLowerCase(Locale.ROOT)}, so the tests pin the locale-sensitive corners
+ * (the Turkish dotted-I family) in addition to plain ASCII case folding and the
+ * persisted-name byte bound.
+ */
+public class LanceIndexNameNormalizerTest {
+
+ @Test
+ public void normalizeLowercasesAscii() {
+ Assertions.assertEquals("my_index", LanceIndexNameNormalizer.normalize("My_Index"));
+ Assertions.assertEquals("idx", LanceIndexNameNormalizer.normalize("IDX"));
+ Assertions.assertEquals("idx", LanceIndexNameNormalizer.normalize("idx"));
+ Assertions.assertEquals("", LanceIndexNameNormalizer.normalize(""));
+ }
+
+ @Test
+ public void normalizeKeepsDigitsAndUnderscores() {
+ Assertions.assertEquals("idx_2024_v2", LanceIndexNameNormalizer.normalize("Idx_2024_V2"));
+ }
+
+ @Test
+ public void normalizeUsesRootLocaleForConditionalMappings() {
+ // İ (capital I with dot above) folds to i + combining dot above under the ROOT/default
+ // mapping, never to the Turkish locale's dotless ı. This pins normalization v1 as
+ // environment-independent.
+ String dottedCapitalI = "İ";
+ String normalized = LanceIndexNameNormalizer.normalize(dottedCapitalI);
+ Assertions.assertEquals("i̇", normalized);
+ Assertions.assertEquals(dottedCapitalI.toLowerCase(Locale.ROOT), normalized);
+ Assertions.assertNotEquals(dottedCapitalI.toLowerCase(new Locale("tr")), normalized);
+ }
+
+ @Test
+ public void normalizeHandlesUnicodeLettersAndLeavesCjkUntouched() {
+ Assertions.assertEquals("äöü", LanceIndexNameNormalizer.normalize("ÄÖÜ"));
+ Assertions.assertEquals("索引", LanceIndexNameNormalizer.normalize("索引"));
+ Assertions.assertEquals("ß", LanceIndexNameNormalizer.normalize("ß"));
+ }
+
+ @Test
+ public void normalizeRejectsNull() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> LanceIndexNameNormalizer.normalize(null));
+ }
+
+ @Test
+ public void isCaseOnlyDuplicateTrueOnlyForPureCaseDifference() {
+ Assertions.assertTrue(LanceIndexNameNormalizer.isCaseOnlyDuplicate("MyIdx", "myidx"));
+ Assertions.assertTrue(LanceIndexNameNormalizer.isCaseOnlyDuplicate("MYIDX", "myidx"));
+ Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate("myidx", "myidx"));
+ Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate("idxA", "idxB"));
+ Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate(null, "idx"));
+ Assertions.assertFalse(LanceIndexNameNormalizer.isCaseOnlyDuplicate("idx", null));
+ }
+
+ @Test
+ public void validateDisplayNameAcceptsBoundarySizes() {
+ LanceIndexNameNormalizer.validateDisplayName("i");
+ LanceIndexNameNormalizer.validateDisplayName(
+ StringUtils.repeat("a", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES));
+ // 512 two-byte characters are exactly at the byte bound.
+ String exactMultibyte = StringUtils.repeat("é", 512);
+ Assertions.assertEquals(1024, exactMultibyte.getBytes(StandardCharsets.UTF_8).length);
+ LanceIndexNameNormalizer.validateDisplayName(exactMultibyte);
+ }
+
+ @Test
+ public void validateDisplayNameRejectsNullEmptyAndOversize() {
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexNameNormalizer.validateDisplayName(null));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexNameNormalizer.validateDisplayName(""));
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexNameNormalizer.validateDisplayName(
+ StringUtils.repeat("a", LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES + 1)));
+ // 513 two-byte characters exceed the byte bound even though the char count is small.
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> LanceIndexNameNormalizer.validateDisplayName(StringUtils.repeat("é", 513)));
+ }
+
+ @Test
+ public void boundIsPinnedAt1024Utf8Bytes() {
+ Assertions.assertEquals(1024, LanceIndexNameNormalizer.MAX_INDEX_NAME_BYTES);
+ }
+}