diff --git a/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml b/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml new file mode 100644 index 000000000000..bf8bad38c896 --- /dev/null +++ b/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml @@ -0,0 +1,7 @@ +title: Update org.apache.tika:tika-core to v4 +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#4853 + url: https://github.com/apache/solr/pull/4853 diff --git a/changelog/unreleased/SolrCellTikaChunks.yml b/changelog/unreleased/SolrCellTikaChunks.yml new file mode 100644 index 000000000000..a4d430184b4f --- /dev/null +++ b/changelog/unreleased/SolrCellTikaChunks.yml @@ -0,0 +1,4 @@ +title: Solr Cell can index Tika 4.x chunk embeddings into dense_vector fields via new tikaserver.chunks parameter +type: added +authors: + - name: Eric Pugh diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 607ed54ecfe1..abdb17a97a5d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -42,7 +42,7 @@ apache-log4j = "2.26.1" apache-lucene = "10.4.0" apache-opennlp = "2.5.10" apache-rat = "0.18" -apache-tika = "3.3.1" +apache-tika = "4.0.0" apache-tomcat = "6.0.53" apache-zookeeper = "3.9.5" # @keep for version alignment diff --git a/solr/licenses/commonmark-0.30.0.jar.sha1 b/solr/licenses/commonmark-0.30.0.jar.sha1 new file mode 100644 index 000000000000..ef17e4e5f72b --- /dev/null +++ b/solr/licenses/commonmark-0.30.0.jar.sha1 @@ -0,0 +1 @@ +fc611136b3191a08d2b84c5d5a26116e3531afee diff --git a/solr/licenses/commonmark-LICENSE-BSD.txt b/solr/licenses/commonmark-LICENSE-BSD.txt new file mode 100644 index 000000000000..604b777d32c7 --- /dev/null +++ b/solr/licenses/commonmark-LICENSE-BSD.txt @@ -0,0 +1,23 @@ +Copyright (c) 2015, Robin Stocker +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/solr/licenses/commonmark-NOTICE.txt b/solr/licenses/commonmark-NOTICE.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 b/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 new file mode 100644 index 000000000000..79f99ad6fa50 --- /dev/null +++ b/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 @@ -0,0 +1 @@ +e9addacba704ffce26d6b8bcd1c50e235ec2c991 diff --git a/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 b/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 new file mode 100644 index 000000000000..8f580d0ef23c --- /dev/null +++ b/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 @@ -0,0 +1 @@ +84560a4af5afe4808543bd9c5587a660ce148b14 diff --git a/solr/licenses/tika-core-3.3.1.jar.sha1 b/solr/licenses/tika-core-3.3.1.jar.sha1 deleted file mode 100644 index 04665fed0998..000000000000 --- a/solr/licenses/tika-core-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -075c89f5a662324e7186da5e315ea9966acb7e8a diff --git a/solr/licenses/tika-core-4.0.0.jar.sha1 b/solr/licenses/tika-core-4.0.0.jar.sha1 new file mode 100644 index 000000000000..bab6900e1458 --- /dev/null +++ b/solr/licenses/tika-core-4.0.0.jar.sha1 @@ -0,0 +1 @@ +f370d8619de9e19329b2cc117943c83468c2ff9f diff --git a/solr/modules/extraction/build.gradle b/solr/modules/extraction/build.gradle index 2d8dc4d2d1fe..0fa9ea5138a4 100644 --- a/solr/modules/extraction/build.gradle +++ b/solr/modules/extraction/build.gradle @@ -46,4 +46,6 @@ dependencies { testImplementation libs.junit.junit testImplementation libs.testcontainers testImplementation libs.carrotsearch.randomizedtesting.runner + // Embedded mock HTTP server for TikaServerExtractionBackendTest's chunk-embeddings tests + testImplementation libs.eclipse.jetty.server } diff --git a/solr/modules/extraction/gradle.lockfile b/solr/modules/extraction/gradle.lockfile index 461a77d6a907..5be430e73763 100644 --- a/solr/modules/extraction/gradle.lockfile +++ b/solr/modules/extraction/gradle.lockfile @@ -118,11 +118,14 @@ org.apache.lucene:lucene-spatial-extras:10.4.0=jarValidation,runtimeClasspath,ru org.apache.lucene:lucene-spatial3d:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.lucene:lucene-suggest:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.lucene:lucene-test-framework:10.4.0=jarValidation,testCompileClasspath,testRuntimeClasspath -org.apache.tika:tika-core:3.3.1=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.apache.tika:tika-core:4.0.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.zookeeper:zookeeper-jute:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.zookeeper:zookeeper:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspath org.codehaus.woodstox:stax2-api:4.3.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.commonmark:commonmark-ext-gfm-tables:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.commonmark:commonmark:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.compression:jetty-compression-common:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.compression:jetty-compression-gzip:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.ee10:jetty-ee10-servlet:12.1.10=jarValidation,testRuntimeClasspath @@ -140,7 +143,7 @@ org.eclipse.jetty:jetty-http:12.1.10=compileClasspath,jarValidation,runtimeClass org.eclipse.jetty:jetty-io:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty:jetty-rewrite:12.1.10=jarValidation,testRuntimeClasspath org.eclipse.jetty:jetty-security:12.1.10=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath -org.eclipse.jetty:jetty-server:12.1.10=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.eclipse.jetty:jetty-server:12.1.10=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty:jetty-session:12.1.10=jarValidation,testRuntimeClasspath org.eclipse.jetty:jetty-util:12.1.10=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.glassfish.hk2.external:aopalliance-repackaged:4.0.1=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java index d4fd1c4cbd1b..33f9d07bb567 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java @@ -19,10 +19,14 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.regex.Pattern; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.ContentStream; @@ -143,8 +147,24 @@ public void load( .tikaServerRecursive(tikaserverRecursive) .tikaServerTimeoutSeconds(tikaTimeoutSecs) .tikaServerRequestHeaders(Map.of()) + .tikaServerConfigJson(params.get(ExtractingParams.TIKASERVER_CONFIG_JSON)) + .tikaServerChunks(params.getBool(ExtractingParams.TIKASERVER_CHUNKS, false)) .build(); + if (extractionRequest.tikaServerChunks) { + try { + loadChunks(req, params, stream, inputStream, extractionRequest); + } catch (Exception e) { + if (ignoreTikaException) { + if (log.isWarnEnabled()) + log.warn("skip extracting chunks due to {}.", e.getLocalizedMessage(), e); + return; + } + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + return; + } + boolean captureAttr = params.getBool(ExtractingParams.CAPTURE_ATTRIBUTES, false); String[] captureElems = params.getParams(ExtractingParams.CAPTURE_ELEMENTS); boolean needsSaxParsing = @@ -231,6 +251,69 @@ public void load( } } + /** + * Indexes one Solr document per Tika 4.x {@code tk:chunks} entry instead of one document for the + * whole source file. See {@link ExtractingParams#TIKASERVER_CHUNKS}. + */ + private void loadChunks( + SolrQueryRequest req, + SolrParams params, + ContentStream stream, + InputStream inputStream, + ExtractionRequest extractionRequest) + throws Exception { + if (!(backend instanceof TikaServerExtractionBackend tikaServerBackend)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + ExtractingParams.TIKASERVER_CHUNKS + + "=true requires the " + + TikaServerExtractionBackend.NAME + + " extraction backend"); + } + List chunks = + tikaServerBackend.extractChunks(inputStream, extractionRequest); + + String contentField = params.get(ExtractingParams.TIKASERVER_CHUNKS_CONTENT_FIELD, "content"); + String vectorField = params.get(ExtractingParams.TIKASERVER_CHUNKS_VECTOR_FIELD, "vector"); + String parentField = + params.get(ExtractingParams.TIKASERVER_CHUNKS_PARENT_FIELD, "chunk_parent_id"); + + String uniqueKeyField = + req.getSchema().getUniqueKeyField() != null + ? req.getSchema().getUniqueKeyField().getName() + : "id"; + + String parentId = params.get(ExtractingParams.LITERALS_PREFIX + uniqueKeyField); + if (parentId == null) parentId = params.get(ExtractingParams.RESOURCE_NAME); + if (parentId == null) parentId = stream.getName(); + if (parentId == null) parentId = UUID.randomUUID().toString(); + + int n = 0; + for (TikaServerExtractionBackend.Chunk chunk : chunks) { + SolrInputDocument doc = new SolrInputDocument(); + doc.setField(uniqueKeyField, parentId + "-chunk-" + n); + doc.setField(contentField, chunk.text); + doc.setField(vectorField, chunk.vector); + doc.setField(parentField, parentId); + + Iterator paramNames = params.getParameterNamesIterator(); + while (paramNames.hasNext()) { + String pname = paramNames.next(); + if (!pname.startsWith(ExtractingParams.LITERALS_PREFIX)) continue; + String name = pname.substring(ExtractingParams.LITERALS_PREFIX.length()); + if (name.equals(uniqueKeyField)) continue; + for (String v : params.getParams(pname)) { + doc.addField(name, v); + } + } + + templateAdd.clear(); + templateAdd.solrDoc = doc; + processor.processAdd(templateAdd); + n++; + } + } + /* * Extracts content from the given input stream using an optional XPath expression * and a SAX content handler. The extraction process may filter content based on diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index a99025127b19..7ef92d1bc914 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -157,4 +157,39 @@ public interface ExtractingParams { /** Default or per-request timeout in seconds for TikaServer HTTP calls. */ String TIKASERVER_TIMEOUT_SECS = "tikaserver.timeoutSeconds"; + + /** + * Optional raw JSON object sent as the "config" part of a per-request TikaServer configuration + * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). Tika Server 4.x removed its + * X-Tika-* configuration headers in favor of this JSON mechanism; the server must additionally + * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Ignored for + * recursive (tikaserver.recursive) requests, since TikaServer has no XML-output variant of + * /rmeta/config. + */ + String TIKASERVER_CONFIG_JSON = "tikaserver.config"; + + /** + * If {@code true}, splits the extracted document into Tika 4.x's {@code tk:chunks} (produced by a + * server-side embedding metadata filter, e.g. {@code openai-embedding-filter}) and indexes one + * Solr document per chunk, each carrying its own embedding vector, instead of one document for + * the whole source file. Requires the Tika Server to be configured with an embedding filter; + * throws if the response carries no {@code tk:chunks}. See {@link + * #TIKASERVER_CHUNKS_CONTENT_FIELD}, {@link #TIKASERVER_CHUNKS_VECTOR_FIELD}, and {@link + * #TIKASERVER_CHUNKS_PARENT_FIELD} for the Solr field names used; all three must already exist in + * the schema. + */ + String TIKASERVER_CHUNKS = "tikaserver.chunks"; + + /** Solr field to hold a chunk's text when {@link #TIKASERVER_CHUNKS} is enabled. */ + String TIKASERVER_CHUNKS_CONTENT_FIELD = "tikaserver.chunks.contentField"; + + /** Solr field to hold a chunk's embedding vector when {@link #TIKASERVER_CHUNKS} is enabled. */ + String TIKASERVER_CHUNKS_VECTOR_FIELD = "tikaserver.chunks.vectorField"; + + /** + * Solr field to hold the source document's id on every chunk document when {@link + * #TIKASERVER_CHUNKS} is enabled. Chunks are indexed as plain sibling documents referencing their + * parent by this field, not as Solr nested/block-join child documents. + */ + String TIKASERVER_CHUNKS_PARENT_FIELD = "tikaserver.chunks.parentField"; } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java index 79068816271a..2f19cf23fffd 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java @@ -38,6 +38,8 @@ public class ExtractionRequest { public final boolean tikaServerRecursive; public final Integer tikaServerTimeoutSeconds; // optional per-request override public final Map tikaServerRequestHeaders = new HashMap<>(); + public final String tikaServerConfigJson; // optional raw JSON "config" part; TikaServer 4.x only + public final boolean tikaServerChunks; // split into tk:chunks instead of one whole document /** * Constructs an ExtractionRequest object containing metadata and configurations for extraction @@ -59,6 +61,10 @@ public class ExtractionRequest { * only). If null or ≤ 0, the default timeout will be used * @param tikaServerRequestHeaders optional headers to be included in requests to the extraction * service. TikaServer only + * @param tikaServerConfigJson optional raw JSON object sent as the per-request "config" part. + * TikaServer 4.x only; requires allowPerRequestConfig=true on the server + * @param tikaServerChunks if true, extract Tika 4.x's tk:chunks (one per embedding-filter chunk) + * instead of a single whole-document result. TikaServer 4.x only */ private ExtractionRequest( String streamType, @@ -73,7 +79,9 @@ private ExtractionRequest( String extractFormat, boolean tikaServerRecursive, Integer tikaServerTimeoutSeconds, - Map tikaServerRequestHeaders) { + Map tikaServerRequestHeaders, + String tikaServerConfigJson, + boolean tikaServerChunks) { this.streamType = streamType; this.resourceName = resourceName; this.contentType = contentType; @@ -89,6 +97,8 @@ private ExtractionRequest( if (tikaServerRequestHeaders != null) { this.tikaServerRequestHeaders.putAll(tikaServerRequestHeaders); } + this.tikaServerConfigJson = tikaServerConfigJson; + this.tikaServerChunks = tikaServerChunks; } /** Creates a new Builder for constructing ExtractionRequest instances. */ @@ -111,6 +121,8 @@ public static class Builder { private boolean tikaServerRecursive = false; private Integer tikaServerTimeoutSeconds; private Map tikaServerRequestHeaders; + private String tikaServerConfigJson; + private boolean tikaServerChunks = false; private Builder() {} @@ -179,6 +191,16 @@ public Builder tikaServerRequestHeaders(Map tikaServerRequestHea return this; } + public Builder tikaServerConfigJson(String tikaServerConfigJson) { + this.tikaServerConfigJson = tikaServerConfigJson; + return this; + } + + public Builder tikaServerChunks(boolean tikaServerChunks) { + this.tikaServerChunks = tikaServerChunks; + return this; + } + public ExtractionRequest build() { return new ExtractionRequest( streamType, @@ -193,7 +215,9 @@ public ExtractionRequest build() { extractFormat, tikaServerRecursive, tikaServerTimeoutSeconds, - tikaServerRequestHeaders); + tikaServerRequestHeaders, + tikaServerConfigJson, + tikaServerChunks); } } } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index a6d08bb11f77..1e37280f7741 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -18,15 +18,21 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PushbackInputStream; +import java.lang.invoke.MethodHandles; import java.net.ConnectException; import java.net.MalformedURLException; import java.net.SocketTimeoutException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -37,15 +43,23 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.Utils; import org.apache.solr.util.RefCounted; import org.apache.tika.sax.BodyContentHandler; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.InputStreamRequestContent; import org.eclipse.jetty.client.InputStreamResponseListener; +import org.eclipse.jetty.client.MultiPartRequestContent; import org.eclipse.jetty.client.Request; import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.client.StringRequestContent; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.MultiPart; import org.eclipse.jetty.io.EofException; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.helpers.DefaultHandler; /** @@ -53,6 +67,8 @@ * import of org.apache.tika.sax.BodyContentHandler; */ public class TikaServerExtractionBackend implements ExtractionBackend { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** * Default maximum response size (100MB) to prevent excessive memory usage from large documents */ @@ -173,26 +189,99 @@ public void extractWithSaxHandler( * request.tikaserverRecursive */ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) throws Exception { - String url = baseUrl + (request.tikaServerRecursive ? "/rmeta" : "/tika"); + ExtractionMetadata md = buildMetadataFromRequest(request); + String pwd = resolvePassword(request, md); + String configJson = resolveConfigJson(request, pwd); HttpClient client = acquiredResourcesRef.get().client; - - Request req = client.newRequest(url).method("PUT"); Duration effectiveTimeout = (request.tikaServerTimeoutSeconds != null && request.tikaServerTimeoutSeconds > 0) ? Duration.ofSeconds(request.tikaServerTimeoutSeconds) : defaultTimeout; - req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); - // Also set idle timeout in case of heavy server side work like OCR - req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); - - // Headers - String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); - req.headers(h -> h.add("Accept", accept)); String contentType = (request.streamType != null) ? request.streamType : request.contentType; - if (contentType != null) { - req.headers(h -> h.add("Content-Type", contentType)); + + String url; + Request req; + if (configJson != null && request.tikaServerRecursive) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Per-request TikaServer config (password or " + + ExtractingParams.TIKASERVER_CONFIG_JSON + + ") is not supported together with " + + ExtractingParams.TIKASERVER_RECURSIVE + + "=true: TikaServer 4.x has no XML-output variant of /rmeta/config."); + } + if (configJson != null) { + // Tika 4.x dropped its X-Tika-* configuration headers (including Password) in favor of a + // per-request JSON "config" part on a multipart request; the server must additionally opt + // in with allowPerRequestConfig=true. There is no XML content-handler variant of + // /rmeta/config in Tika 4.x, so this path only covers non-recursive extraction (checked + // above). + url = baseUrl + "/tika/config/xml"; + req = client.newRequest(url).method("POST"); + req.headers(h -> h.add("Accept", "text/xml")); + + HttpFields.Mutable fileFields = HttpFields.build(); + if (contentType != null) { + fileFields.add(HttpHeader.CONTENT_TYPE, contentType); + } + try (MultiPartRequestContent multiPart = new MultiPartRequestContent()) { + multiPart.addPart( + new MultiPart.ContentSourcePart( + "file", + request.resourceName, + fileFields, + new InputStreamRequestContent(inputStream))); + multiPart.addPart( + new MultiPart.ContentSourcePart( + "config", + null, + HttpFields.build().add(HttpHeader.CONTENT_TYPE, "application/json"), + new StringRequestContent(configJson))); + req.body(multiPart); + } + } else { + // Tika 4.x's default content handler is Markdown, not XHTML/XML (TIKA-4663); Solr's SAX-based + // content handling needs the previous XHTML/XML output, requested via the /xml path variants. + url = baseUrl + (request.tikaServerRecursive ? "/rmeta/xml" : "/tika/xml"); + req = client.newRequest(url).method("PUT"); + String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); + req.headers(h -> h.add("Accept", accept)); + if (contentType != null) { + req.headers(h -> h.add("Content-Type", contentType)); + } + if (request.resourceName != null) { + req.headers( + h -> + h.add( + "Content-Disposition", + "attachment; filename=\"" + request.resourceName + "\"")); + } + if (contentType != null) { + req.body(new InputStreamRequestContent(contentType, inputStream)); + } else { + req.body(new InputStreamRequestContent(inputStream)); + } } + + return sendAndReadResponse(req, url, effectiveTimeout, request, !request.tikaServerRecursive); + } + + /** + * Sends an already-built request and validates/returns its response body as a bounded {@link + * InputStream}, shared by {@link #callTikaServer} and {@link #callTikaServerForChunks}. + * + * @param allowPartial422 if true, a 422 (Unprocessable Entity) response with a non-empty body is + * treated as a partial success rather than a hard failure (see {@link #callTikaServer} for + * why this only applies to the non-recursive raw endpoints) + */ + private InputStream sendAndReadResponse( + Request req, + String url, + Duration effectiveTimeout, + ExtractionRequest request, + boolean allowPartial422) + throws Exception { if (!request.tikaServerRequestHeaders.isEmpty()) { req.headers( h -> @@ -202,32 +291,9 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t })); } - ExtractionMetadata md = buildMetadataFromRequest(request); - if (request.resourcePassword != null || request.passwordsMap != null) { - RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider(); - if (request.resourcePassword != null) { - passwordProvider.setExplicitPassword(request.resourcePassword); - } - if (request.passwordsMap != null) { - passwordProvider.setPasswordMap(request.passwordsMap); - } - String pwd = passwordProvider.getPassword(md); - if (pwd != null) { - req.headers(h -> h.add("Password", pwd)); // Tika Server expects this header if provided - } - } - if (request.resourceName != null) { - req.headers( - h -> - h.add( - "Content-Disposition", "attachment; filename=\"" + request.resourceName + "\"")); - } - - if (contentType != null) { - req.body(new InputStreamRequestContent(contentType, inputStream)); - } else { - req.body(new InputStreamRequestContent(inputStream)); - } + req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); + // Also set idle timeout in case of heavy server side work like OCR + req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); InputStreamResponseListener listener = new InputStreamResponseListener(); req.send(listener); @@ -273,7 +339,33 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t } int code = response.getStatus(); - if (code < 200 || code >= 300) { + InputStream responseStream = listener.getInputStream(); + // Tika 4.x's raw /tika* endpoints (non-recursive) return 422 whenever a container-level + // exception occurred during parsing -- including a non-aborting one like a writeLimit + // truncation -- but the body still carries whatever content was successfully extracted + // (there's no envelope to carry the exception itself on these endpoints; use /rmeta for + // that). A request that extracted nothing at all (e.g. a wrong password) also gets 422, but + // with an empty body -- treat that case as the failure it is instead of a silent empty + // "success". Peek the first byte to tell the two apart. + if (code == 422 && allowPartial422) { + PushbackInputStream peekable = new PushbackInputStream(responseStream, 1); + int firstByte = peekable.read(); + if (firstByte == -1) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "TikaServer " + + url + + " returned status 422 (Unprocessable Entity) with no content -- the document" + + " could not be parsed at all (check the password, if one was required)."); + } + peekable.unread(firstByte); + log.warn( + "TikaServer {} returned 422 (a container-level exception occurred during parsing); " + + "using the partial content it still returned. Use tikaserver.recursive=true " + + "against /rmeta for the exception detail.", + url); + responseStream = peekable; + } else if (code < 200 || code >= 300) { SolrException.ErrorCode errorCode = SolrException.ErrorCode.getErrorCode(code); String reason = response.getReason(); String msg = @@ -285,11 +377,206 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t throw new SolrException(errorCode, msg); } - InputStream responseStream = listener.getInputStream(); // Bound the amount of data we read from Tika Server to avoid excessive memory/CPU usage return new LimitingInputStream(responseStream, maxCharsLimit); } + /** + * A single Tika 4.x {@code tk:chunks} entry carrying chunk text and its embedding vector. Image + * chunks (text-less, produced by an image embedding filter) are not surfaced here; v1 only + * indexes text chunks. + */ + static final class Chunk { + final String text; + final float[] vector; + + Chunk(String text, float[] vector) { + this.text = text; + this.vector = vector; + } + } + + /** + * Calls TikaServer's {@code /rmeta} endpoint (always the JSON/Markdown-handler variant, since + * chunking requires Markdown content -- see {@link ExtractingParams#TIKASERVER_CHUNKS}) and + * flattens every {@code tk:chunks} entry found across the response array (one array entry per + * embedded file) into a single list. + * + * @throws SolrException if the response carries no {@code tk:chunks} anywhere, which almost + * always means the TikaServer instance has no embedding metadata filter configured + */ + List extractChunks(InputStream inputStream, ExtractionRequest request) throws Exception { + try (InputStream tikaResponse = callTikaServerForChunks(inputStream, request)) { + Object parsed = Utils.fromJSON(tikaResponse); + if (!(parsed instanceof List list)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Unexpected /rmeta response, expected JSON array"); + } + List chunks = new ArrayList<>(); + for (Object o : list) { + if (!(o instanceof Map map)) continue; + String chunksJson = firstString(map.get("tk:chunks")); + if (chunksJson == null || chunksJson.isBlank()) continue; + Object parsedChunks = Utils.fromJSONString(chunksJson); + if (!(parsedChunks instanceof List chunkList)) continue; + for (Object co : chunkList) { + if (!(co instanceof Map chunkMap)) continue; + Object textObj = chunkMap.get("text"); + Object vectorObj = chunkMap.get("vector"); + // v1 only indexes text chunks; image chunks (no "text") are skipped. + if (textObj == null || vectorObj == null) continue; + chunks.add(new Chunk(String.valueOf(textObj), decodeVector(String.valueOf(vectorObj)))); + } + } + if (chunks.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "tikaserver.chunks=true but the TikaServer response carried no tk:chunks. Configure " + + "an embedding metadata filter (e.g. openai-embedding-filter) on the TikaServer " + + "instance -- see Tika's tika-inference module documentation."); + } + return chunks; + } + } + + /** Returns the first (and typically only) value of a Tika metadata JSON value as a String. */ + private static String firstString(Object val) { + if (val instanceof List l) { + return l.isEmpty() ? null : String.valueOf(l.get(0)); + } + return val == null ? null : String.valueOf(val); + } + + /** + * Decodes a chunk's {@code vector} field: a base64-encoded, big-endian (Java default {@link + * ByteBuffer} order) float32 array, per Tika's {@code ChunkSerializer} -- confirmed by + * round-tripping known embedding values against a live TikaServer 4.0.0 instance. + */ + private static float[] decodeVector(String base64) { + byte[] bytes = Base64.getDecoder().decode(base64); + java.nio.FloatBuffer floatBuffer = ByteBuffer.wrap(bytes).asFloatBuffer(); + float[] vector = new float[floatBuffer.remaining()]; + floatBuffer.get(vector); + return vector; + } + + /** + * Builds and sends the request for {@link #extractChunks}: always hits {@code /rmeta} (or {@code + * /rmeta/config} when a per-request config, e.g. a resolved password, applies), requesting JSON + * so the default Markdown content handler -- required for chunking -- is used. Independent of + * {@code request.tikaServerRecursive}: {@code /rmeta} already returns one array entry per + * embedded file regardless. + */ + private InputStream callTikaServerForChunks(InputStream inputStream, ExtractionRequest request) + throws Exception { + ExtractionMetadata md = buildMetadataFromRequest(request); + String pwd = resolvePassword(request, md); + String configJson = resolveConfigJson(request, pwd); + + HttpClient client = acquiredResourcesRef.get().client; + Duration effectiveTimeout = + (request.tikaServerTimeoutSeconds != null && request.tikaServerTimeoutSeconds > 0) + ? Duration.ofSeconds(request.tikaServerTimeoutSeconds) + : defaultTimeout; + String contentType = (request.streamType != null) ? request.streamType : request.contentType; + + String url; + Request req; + if (configJson != null) { + url = baseUrl + "/rmeta/config"; + req = client.newRequest(url).method("POST"); + req.headers(h -> h.add("Accept", "application/json")); + + HttpFields.Mutable fileFields = HttpFields.build(); + if (contentType != null) { + fileFields.add(HttpHeader.CONTENT_TYPE, contentType); + } + try (MultiPartRequestContent multiPart = new MultiPartRequestContent()) { + multiPart.addPart( + new MultiPart.ContentSourcePart( + "file", + request.resourceName, + fileFields, + new InputStreamRequestContent(inputStream))); + multiPart.addPart( + new MultiPart.ContentSourcePart( + "config", + null, + HttpFields.build().add(HttpHeader.CONTENT_TYPE, "application/json"), + new StringRequestContent(configJson))); + req.body(multiPart); + } + } else { + url = baseUrl + "/rmeta"; + req = client.newRequest(url).method("PUT"); + req.headers(h -> h.add("Accept", "application/json")); + if (contentType != null) { + req.headers(h -> h.add("Content-Type", contentType)); + } + if (request.resourceName != null) { + req.headers( + h -> + h.add( + "Content-Disposition", + "attachment; filename=\"" + request.resourceName + "\"")); + } + if (contentType != null) { + req.body(new InputStreamRequestContent(contentType, inputStream)); + } else { + req.body(new InputStreamRequestContent(inputStream)); + } + } + + return sendAndReadResponse(req, url, effectiveTimeout, request, false); + } + + /** Resolves the password to use for an encrypted document, or null if none applies. */ + private String resolvePassword(ExtractionRequest request, ExtractionMetadata md) { + if (request.resourcePassword == null && request.passwordsMap == null) { + return null; + } + RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider(); + if (request.resourcePassword != null) { + passwordProvider.setExplicitPassword(request.resourcePassword); + } + if (request.passwordsMap != null) { + passwordProvider.setPasswordMap(request.passwordsMap); + } + return passwordProvider.getPassword(md); + } + + /** + * Builds the per-request TikaServer JSON "config" payload, merging any resolved password with any + * caller-supplied {@link ExtractingParams#TIKASERVER_CONFIG_JSON}. Returns null if neither + * applies, meaning no per-request config is needed. + */ + @SuppressWarnings("unchecked") + private String resolveConfigJson(ExtractionRequest request, String pwd) { + Map config = new LinkedHashMap<>(); + if (request.tikaServerConfigJson != null && !request.tikaServerConfigJson.isBlank()) { + Object parsed; + try { + parsed = Utils.fromJSONString(request.tikaServerConfigJson); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Invalid JSON in " + ExtractingParams.TIKASERVER_CONFIG_JSON + ": " + e.getMessage(), + e); + } + if (!(parsed instanceof Map)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + ExtractingParams.TIKASERVER_CONFIG_JSON + " must be a JSON object"); + } + config.putAll((Map) parsed); + } + if (pwd != null && !config.containsKey("simple-password-provider")) { + config.put("simple-password-provider", Map.of("password", pwd)); + } + return config.isEmpty() ? null : Utils.toJSONString(config); + } + private static class LimitingInputStream extends InputStream { private final InputStream in; private final long max; diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java index e34514040b6e..41f9b83364c1 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java @@ -87,11 +87,14 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe for (Object o : list) { if (!(o instanceof Map)) continue; Map map = (Map) o; + // Tika 4.x renamed X-TIKA:content to tk:content (TIKA-4816); accept either so this also + // works against a Tika Server 3.x still on the old key. + String contentKey = map.containsKey("tk:content") ? "tk:content" : "X-TIKA:content"; // Copy metadata for (Object k : map.keySet()) { String key = String.valueOf(k); Object val = map.get(k); - if ("X-TIKA:content".equalsIgnoreCase(key)) { + if (contentKey.equalsIgnoreCase(key)) { // handled below continue; } @@ -103,7 +106,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe md.add(key, String.valueOf(val)); } } - Object content = map.get("X-TIKA:content"); + Object content = map.get(contentKey); if (content != null) { String xhtml = String.valueOf(content); if (!xhtml.isEmpty() && handler != null) { diff --git a/solr/modules/extraction/src/test-files/extraction/tika-server-config.json b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json new file mode 100644 index 000000000000..63a1e4fcbd77 --- /dev/null +++ b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json @@ -0,0 +1,5 @@ +{ + "server": { + "allowPerRequestConfig": true + } +} diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java index c9c872bc99de..f6f4cef080a1 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java @@ -440,17 +440,20 @@ public void testLiterals() throws Exception { "one", "literal.extractionLiteral", "two", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-parser-override", + "ignored_parser", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.Last-Modified", "extractedDate"); @@ -480,17 +483,20 @@ public void testLiterals() throws Exception { "extractedLanguage", "literal.extractionLiteral", "one", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.Last-Modified", "extractedDate"); @@ -596,21 +602,26 @@ public void testPlainTextSpecifyingMimeType() throws Exception { "one", "fmap.language", "extractedLanguage", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:detected-encoding", + "ignored_parser", + "fmap.tk:encoding-detector", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:encoding-detection-trace", "ignored_parser", - "fmap.X-TIKA:detectedEncoding", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:encodingDetector", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.content", "extractedContent", @@ -644,21 +655,28 @@ public void testPlainTextSpecifyingResourceName() throws Exception { "one", "fmap.language", "extractedLanguage", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:detected-encoding", + "ignored_parser", + "fmap.tk:encoding-detector", + "ignored_parser", + "fmap.tk:encoding-detection-trace", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:detectedEncoding", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:encodingDetector", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-magic-detected", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:resource-name", "ignored_parser", "fmap.content", "extractedContent", diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java index 23470049c985..1c76fe6e668a 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; @ThreadLeakFilters(filters = {SolrIgnoredThreadsFilter.class, QuickPatchThreadsFilter.class}) public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHandlerTestAbstract { @@ -43,9 +44,15 @@ public static void beforeClassTika() { String baseUrl; try { + // allowPerRequestConfig is off by default (it lets a client inject arbitrary parser + // config, e.g. for encrypted-document passwords); enabling it here is test-only. tika = - new GenericContainer<>("apache/tika:3.2.3.0-full") + new GenericContainer<>("apache/tika:4.0.0-full") .withExposedPorts(9998) + .withCopyFileToContainer( + MountableFile.forHostPath(getFile("extraction/tika-server-config.json")), + "/tika-config.json") + .withCommand("-c", "/tika-config.json") .waitingFor(Wait.forListeningPort()); tika.start(); baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998); diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index b7723da94ff9..7bce4376b512 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -19,8 +19,14 @@ import com.carrotsearch.randomizedtesting.ThreadFilter; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import java.io.ByteArrayInputStream; +import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -28,12 +34,26 @@ import org.apache.solr.SolrIgnoredThreadsFilter; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.Utils; import org.apache.solr.handler.extraction.fromtika.ToXMLContentHandler; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.io.Content; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.Callback; import org.junit.AfterClass; import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; /** * Integration tests for TikaServerExtractionBackend using a real Tika Server via Testcontainers. @@ -45,6 +65,7 @@ TikaServerExtractionBackendTest.TestcontainersThreadsFilter.class }) public class TikaServerExtractionBackendTest extends SolrTestCaseJ4 { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); // Ignore known non-daemon threads spawned by Testcontainers and Java HttpClient in this test @SuppressWarnings("NewClassNamingConvention") @@ -70,7 +91,16 @@ public static void startTikaServer() { "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); try { - tika = new GenericContainer<>("apache/tika:3.2.3.0-full").withExposedPorts(9998); + // allowPerRequestConfig is off by default (it lets a client inject arbitrary parser + // config, e.g. for encrypted-document passwords or tikaserver.config); enabling it here is + // test-only. + tika = + new GenericContainer<>("apache/tika:4.0.0-full") + .withExposedPorts(9998) + .withCopyFileToContainer( + MountableFile.forHostPath(getFile("extraction/tika-server-config.json")), + "/tika-config.json") + .withCommand("-c", "/tika-config.json"); tika.start(); baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998); } catch (Throwable t) { @@ -90,6 +120,110 @@ public static void stopTikaServer() { } } + private static Server embeddingsServer; + private static GenericContainer tikaChunks; + private static String chunksBaseUrl; + + /** + * A minimal, embedded-Jetty, OpenAI-compatible embeddings endpoint standing in for a real + * embeddings API. Each embedding is a small deterministic function of the input text. + */ + private static class EmbeddingsHandler extends Handler.Abstract { + @Override + public boolean handle(Request request, Response response, Callback callback) throws Exception { + String body = Content.Source.asString(request, StandardCharsets.UTF_8); + @SuppressWarnings("unchecked") + Map req = (Map) Utils.fromJSONString(body); + List inputs = (List) req.get("input"); + List data = new ArrayList<>(); + for (int i = 0; i < inputs.size(); i++) { + int total = 0; + for (byte b : String.valueOf(inputs.get(i)).getBytes(StandardCharsets.UTF_8)) { + total += (b & 0xFF); + } + List vector = new ArrayList<>(); + for (int d = 0; d < 4; d++) { + vector.add(((total + d) % 10) / 10.0); + } + Map embedding = new LinkedHashMap<>(); + embedding.put("object", "embedding"); + embedding.put("index", i); + embedding.put("embedding", vector); + data.add(embedding); + } + Map resp = new LinkedHashMap<>(); + resp.put("object", "list"); + resp.put("data", data); + resp.put("model", req.getOrDefault("model", "mock-embed")); + byte[] respBytes = Utils.toJSONString(resp).getBytes(StandardCharsets.UTF_8); + response.setStatus(200); + response.getHeaders().put(HttpHeader.CONTENT_TYPE, "application/json"); + response.write(true, ByteBuffer.wrap(respBytes), callback); + return true; + } + } + + @SuppressWarnings("resource") + @BeforeClass + public static void startChunksTikaServer() { + Assume.assumeFalse( + "Skipping on s390x", "s390x".equalsIgnoreCase(System.getProperty("os.arch"))); + try { + embeddingsServer = new Server(); + ServerConnector connector = new ServerConnector(embeddingsServer); + connector.setPort(0); + embeddingsServer.addConnector(connector); + embeddingsServer.setHandler(new EmbeddingsHandler()); + embeddingsServer.start(); + int embeddingsPort = connector.getLocalPort(); + + // Tika 4.x requires a top-level "server" element even when there's nothing to configure + // in it. + String config = + "{\"server\":{},\"metadata-filters\":[{\"openai-embedding-filter\":" + + "{\"baseUrl\":\"http://host.docker.internal:" + + embeddingsPort + + "\",\"model\":\"mock-embed\"}}]}"; + Path configFile = Files.createTempFile("tika-chunks-config", ".json"); + Files.writeString(configFile, config); + // Files.createTempFile defaults to owner-only (0600) permissions; the TikaServer container + // process runs as a different uid and needs read access to the bind-mounted file. + Files.setPosixFilePermissions(configFile, PosixFilePermissions.fromString("rw-r--r--")); + + tikaChunks = + new GenericContainer<>("apache/tika:4.0.0-full") + .withExposedPorts(9998) + .withExtraHost("host.docker.internal", "host-gateway") + .withCopyFileToContainer(MountableFile.forHostPath(configFile), "/tika-config.json") + .withCommand("-c", "/tika-config.json") + .withLogConsumer(new Slf4jLogConsumer(log)) + .waitingFor(Wait.forListeningPort()); + tikaChunks.start(); + chunksBaseUrl = "http://" + tikaChunks.getHost() + ":" + tikaChunks.getMappedPort(9998); + } catch (Throwable t) { + // Skip tests if Docker/Testcontainers are not available in the environment + Assume.assumeNoException("Docker/Testcontainers not available; skipping chunk tests", t); + } + } + + @AfterClass + public static void stopChunksTikaServer() { + if (tikaChunks != null) { + try { + tikaChunks.stop(); + } catch (Throwable ignore) { + } + tikaChunks = null; + } + if (embeddingsServer != null) { + try { + embeddingsServer.stop(); + } catch (Throwable ignore) { + } + embeddingsServer = null; + } + } + private static ExtractionRequest newRequest( String resourceName, String contentType, @@ -154,14 +288,15 @@ public void testPdfWithImageRecursive() throws Exception { Assume.assumeTrue("Tika server container not started", tika != null); try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); - // Enable recursive extraction and set header to extract images from PDF + // Tika 4.x removed the X-Tika-* header family entirely (see resolveConfigJson's javadoc); + // there is no replacement for this combination. Per-request config now requires the + // multipart /config endpoints, but Tika 4.x has no XML-output variant of /rmeta/config, so + // per-request PDF options (e.g. explicit inline-image extraction) cannot be requested + // together with tikaserver.recursive=true. The PDF's embedded image still gets OCR'd into + // the main document's content by default, just not exposed as a separate embedded + // resource entry the way the pre-4.x X-Tika-PDFextractInlineImages header used to. ExtractionRequest request = - newRequest( - "pdf-with-image.pdf", - "application/pdf", - "xml", - true, - Map.of("X-Tika-PDFextractInlineImages", "true")); + newRequest("pdf-with-image.pdf", "application/pdf", "xml", true, Map.of()); try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { ToXMLContentHandler xmlHandler = new ToXMLContentHandler(); ExtractionMetadata md = backend.buildMetadataFromRequest(request); @@ -169,9 +304,8 @@ public void testPdfWithImageRecursive() throws Exception { String c = xmlHandler.toString(); assertNotNull(c); assertTrue(c.contains("Puppet Apply")); - assertTrue(c.contains("embedded:image0.jpg")); - assertEquals( - "org.apache.tika.parser.DefaultParser", md.getFirst("X-TIKA:Parsed-By-Full-Set")); + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + assertEquals("org.apache.tika.parser.DefaultParser", md.getFirst("tk:parsed-by-full-set")); } } } @@ -227,4 +361,151 @@ public void testMaxCharsLimitEnforcedWithSaxHandler() throws Exception { } } } + + private static ExtractionRequest newRequestWithConfig( + String resourceName, String contentType, String extractFormat, String configJson) { + return ExtractionRequest.builder() + .streamType(contentType) + .resourceName(resourceName) + .contentType(contentType) + .streamName(resourceName) + .extractFormat(extractFormat) + .tikaServerConfigJson(configJson) + .build(); + } + + @Test + public void testConfigJsonDisablesOcr() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); + // With no config, the PDF's embedded image gets OCR'd and "Puppet Apply" (from the image) + // appears in the extracted content. Disabling OCR via tikaserver.config should suppress it. + ExtractionRequest request = + newRequestWithConfig( + "pdf-with-image.pdf", + "application/pdf", + "xml", + "{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + ExtractionResult res = backend.extract(in, request); + assertNotNull(res.getContent()); + assertFalse( + "Expected tikaserver.config's NO_OCR strategy to suppress the OCR'd image text", + res.getContent().contains("Puppet Apply")); + } + } + } + + @Test + public void testConfigJsonMergesWithPassword() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = Files.readAllBytes(getFile("extraction/encrypted-password-is-solrRules.pdf")); + ExtractionRequest request = + ExtractionRequest.builder() + .streamType("application/pdf") + .resourceName("encrypted-password-is-solrRules.pdf") + .contentType("application/pdf") + .streamName("encrypted-password-is-solrRules.pdf") + .extractFormat("xml") + .resourcePassword("solrRules") + .tikaServerConfigJson("{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}") + .build(); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + ExtractionResult res = backend.extract(in, request); + assertNotNull(res); + assertTrue( + "Expected the password-unlocked content to still be present alongside the merged" + + " tikaserver.config", + res.getContent().contains("This is a test of PDF and Word extraction")); + } + } + } + + @Test + public void testInvalidConfigJsonRejected() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = "hello".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = + newRequestWithConfig("test.txt", "text/plain", "xml", "not valid json"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_CONFIG_JSON)); + } + } + } + + @Test + public void testConfigJsonRejectedForRecursive() throws Exception { + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = "hello".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = + ExtractionRequest.builder() + .streamType("text/plain") + .resourceName("test.txt") + .contentType("text/plain") + .streamName("test.txt") + .extractFormat("xml") + .tikaServerRecursive(true) + .tikaServerConfigJson("{\"parse-context\":{}}") + .build(); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_RECURSIVE)); + } + } + } + + private static ExtractionRequest newMarkdownRequest(String resourceName) { + return ExtractionRequest.builder() + .resourceName(resourceName) + .contentType("text/markdown") + .streamName(resourceName) + .build(); + } + + @Test + public void testExtractChunks() throws Exception { + Assume.assumeTrue("Chunks TikaServer container not started", tikaChunks != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(chunksBaseUrl)) { + String markdown = + "# Report\n\nRevenue grew 15% year over year in the last quarter.\n\n" + + "# Costs\n\nOperating costs remained flat compared to prior periods and did not" + + " change much.\n"; + byte[] data = markdown.getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = newMarkdownRequest("sample.md"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + List chunks = backend.extractChunks(in, request); + assertFalse("Expected at least one chunk", chunks.isEmpty()); + assertTrue( + "Expected more than one chunk from a two-heading Markdown document", chunks.size() > 1); + for (TikaServerExtractionBackend.Chunk chunk : chunks) { + assertNotNull(chunk.text); + assertFalse(chunk.text.isBlank()); + assertEquals( + "Expected the mock embedding server's 4-dimensional vectors", 4, chunk.vector.length); + } + } + } + } + + @Test + public void testExtractChunksThrowsWithoutEmbeddingFilterConfigured() throws Exception { + // Reuses the plain `tika` container from startTikaServer(), which has no embedding filter. + Assume.assumeTrue("Tika server container not started", tika != null); + try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(baseUrl)) { + byte[] data = "# Heading\n\nSome text.".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = newMarkdownRequest("sample.md"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = + expectThrows(SolrException.class, () -> backend.extractChunks(in, request)); + assertTrue(e.getMessage().contains("tk:chunks")); + } + } + } } diff --git a/solr/packaging/test/mock_embeddings_server.py b/solr/packaging/test/mock_embeddings_server.py new file mode 100644 index 000000000000..69334e5cc457 --- /dev/null +++ b/solr/packaging/test/mock_embeddings_server.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Minimal, stdlib-only, OpenAI-compatible embeddings endpoint used by +# test_extraction.bats to exercise Solr Cell's tikaserver.chunks feature without +# depending on a real embeddings API. Each embedding is a small deterministic +# function of the input text, so the resulting vectors are reproducible. + +import json +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +DIMENSIONS = 4 + + +def embed(text): + total = sum(text.encode("utf-8")) + return [float((total + i) % 10) / 10.0 for i in range(DIMENSIONS)] + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"{}") + inputs = body.get("input", []) + data = [ + {"object": "embedding", "index": i, "embedding": embed(text)} + for i, text in enumerate(inputs) + ] + response = { + "object": "list", + "data": data, + "model": body.get("model", "mock-embed"), + "usage": {"prompt_tokens": 0, "total_tokens": 0}, + } + payload = json.dumps(response).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, format, *args): + pass + + +if __name__ == "__main__": + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8077 + HTTPServer(("0.0.0.0", port), Handler).serve_forever() diff --git a/solr/packaging/test/test_extraction.bats b/solr/packaging/test/test_extraction.bats index 4b8e62ae59cc..8e76843678fc 100644 --- a/solr/packaging/test/test_extraction.bats +++ b/solr/packaging/test/test_extraction.bats @@ -18,10 +18,13 @@ load bats_helper -# Apply the ExtractingRequestHandler via Config API and print error body on failure +# Apply the ExtractingRequestHandler via Config API and print error body on failure. +# Defaults to the shared bats_tika container; pass a second arg to point at a different +# TikaServer port (e.g. a test-specific container). apply_extract_handler() { local collection="$1" - local json="{\"add-requesthandler\":{\"name\":\"/update/extract\",\"class\":\"org.apache.solr.handler.extraction.ExtractingRequestHandler\",\"tikaserver.url\":\"http://localhost:${TIKA_PORT}\",\"defaults\":{\"lowernames\":\"true\",\"captureAttr\":\"true\"}}}" + local tika_port="${2:-${TIKA_PORT}}" + local json="{\"add-requesthandler\":{\"name\":\"/update/extract\",\"class\":\"org.apache.solr.handler.extraction.ExtractingRequestHandler\",\"tikaserver.url\":\"http://localhost:${tika_port}\",\"defaults\":{\"lowernames\":\"true\",\"captureAttr\":\"true\"}}}" local url="http://localhost:${SOLR_PORT}/solr/${collection}/config" # Capture body and status code local resp code body @@ -40,7 +43,7 @@ apply_extract_handler() { setup_file() { if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then export TIKA_PORT=$((SOLR_PORT+5)) - docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:3.2.3.0-full >/dev/null 2>&1 || true + docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:4.0.0-full >/dev/null 2>&1 || true echo "Waiting for Tika Server to be ready on port ${TIKA_PORT}" >&3 if ! wait_for 120 3 curl -s -f "http://localhost:${TIKA_PORT}/tika" -o /dev/null; then export DOCKER_UNAVAILABLE=1 @@ -71,6 +74,10 @@ teardown() { delete_all_collections SOLR_STOP_WAIT=1 solr stop --all >/dev/null 2>&1 + + # Defensive cleanup in case the chunks test failed before reaching its own cleanup + pkill -f mock_embeddings_server.py >/dev/null 2>&1 || true + docker stop bats_tika_chunks >/dev/null 2>&1 || true } @test "using curl to extract a single pdf file" { @@ -143,3 +150,80 @@ teardown() { run curl "http://localhost:${SOLR_PORT}/solr/website_extraction/select?q=*:*" assert_output --partial '"numFound":1' } + +@test "using tikaserver.chunks to index Tika 4.x chunk embeddings into a dense_vector field" { + + if [ -n "${DOCKER_UNAVAILABLE:-}" ]; then + skip "Docker is not available" + fi + + # Disable security manager to allow extraction + # This appears to be a bug. + export SOLR_SECURITY_MANAGER_ENABLED=false + + # A minimal, stdlib-only, OpenAI-compatible embeddings endpoint standing in for a real + # embeddings API, so this test has no external dependency. + local embed_port=$((SOLR_PORT+6)) + python3 "${BATS_TEST_DIRNAME}/mock_embeddings_server.py" "${embed_port}" & + local embed_pid=$! + + # A dedicated TikaServer, distinct from the shared bats_tika container, configured with an + # openai-embedding-filter pointed at the mock embeddings endpoint above. Chunking requires + # the Markdown content handler, which is Tika 4.x's default. + local chunks_tika_port=$((SOLR_PORT+7)) + local tika_config="${BATS_TEST_TMPDIR}/tika-chunks-config.json" + cat > "$tika_config" </dev/null 2>&1 + + wait_for 15 1 curl -s -f -X POST -H 'Content-Type: application/json' \ + -d '{"model":"mock-embed","input":["ping"]}' "http://localhost:${embed_port}/v1/embeddings" -o /dev/null + wait_for 60 3 curl -s -f "http://localhost:${chunks_tika_port}/tika" -o /dev/null + + solr start -Dsolr.modules=extraction + + solr create -c chunks_extraction -d _default + wait_for 30 3 curl -s -S -f "http://localhost:${SOLR_PORT}/solr/chunks_extraction/select?q=*:*" -o /dev/null + apply_extract_handler chunks_extraction "${chunks_tika_port}" + + curl -s -X POST -H 'Content-type:application/json' \ + -d '{"add-field-type":{"name":"knn_vector","class":"solr.DenseVectorField","vectorDimension":4,"similarityFunction":"cosine"}}' \ + "http://localhost:${SOLR_PORT}/solr/chunks_extraction/schema" + curl -s -X POST -H 'Content-type:application/json' \ + -d '{"add-field":{"name":"vector","type":"knn_vector","indexed":true,"stored":true}}' \ + "http://localhost:${SOLR_PORT}/solr/chunks_extraction/schema" + curl -s -X POST -H 'Content-type:application/json' \ + -d '{"add-field":{"name":"chunk_parent_id","type":"string","indexed":true,"stored":true}}' \ + "http://localhost:${SOLR_PORT}/solr/chunks_extraction/schema" + + local sample="${BATS_TEST_TMPDIR}/sample.md" + printf '# Report\n\nRevenue grew 15%% year over year in the last quarter.\n\n# Costs\n\nOperating costs remained flat compared to prior periods and did not change much.\n' > "$sample" + + # resource.name is required so TikaServer detects this upload as Markdown (curl's multipart + # Content-Type guess for .md is unreliable); chunking only splits on headings for Markdown content. + curl -s "http://localhost:${SOLR_PORT}/solr/chunks_extraction/update/extract?tikaserver.chunks=true&resource.name=sample.md&literal.id=doc1&commit=true" \ + -F "myfile=@${sample}" + + run curl -s "http://localhost:${SOLR_PORT}/solr/chunks_extraction/select?q=chunk_parent_id:doc1" + assert_output --partial '"numFound":2' + + run curl -s "http://localhost:${SOLR_PORT}/solr/chunks_extraction/select?q=%7B!knn+f=vector+topK=3%7D%5B0.7,0.8,0.9,0.0%5D&fl=id,vector" + assert_output --partial '"numFound":2' + refute_output --partial '"numFound":0' + + kill "$embed_pid" >/dev/null 2>&1 || true + docker stop bats_tika_chunks >/dev/null 2>&1 || true +} diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc index db34f483b69f..d58038d509e3 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc @@ -39,7 +39,7 @@ Indexing binary files (PDF, DOCX, PPTX, etc.) with the Post Tool requires the So .Start a Tika Server quickly using Docker (exposes port 9998 on localhost) [,bash] ---- -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full ---- === Start Solr with the extraction module enabled diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index 1a50f8f14c6e..56ca389a90c8 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -78,7 +78,7 @@ The quickest way to run Tika Server for development is using Docker. The example [,bash] ---- -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full ---- NOTE: If Solr runs in Docker too, ensure both containers share a network and use the Tika container name as the host in `tikaserver.url`. @@ -92,7 +92,7 @@ First we start a tika server on port 9998, using Docker. [source,bash] ---- # Start Tika Server in the background -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full # To stop the server when done, run `docker stop tika` ---- @@ -384,6 +384,61 @@ Example: `passwordsFile=/path/to/passwords.txt` // + // Only applicable for `tikaserver` backend. Can only be set in `solrconfig.xml`, not per request. +`tikaserver.chunks`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `false` +|=== ++ +If `true`, splits the extracted document into Tika 4.x's `tk:chunks` -- text chunks carrying precomputed embedding vectors, produced by a server-side embedding metadata filter such as `openai-embedding-filter` -- and indexes one Solr document per chunk instead of one document for the whole source file. +See <> below. ++ +Example: `tikaserver.chunks=true` + +`tikaserver.chunks.contentField`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `content` +|=== ++ +The Solr field that holds a chunk's text when `tikaserver.chunks=true`. +This field must already exist in the schema. + +`tikaserver.chunks.vectorField`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `vector` +|=== ++ +The Solr `DenseVectorField`-typed field that holds a chunk's embedding when `tikaserver.chunks=true`. +This field must already exist in the schema, with a `vectorDimension` matching the embedding filter's output dimension. + +`tikaserver.chunks.parentField`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `chunk_parent_id` +|=== ++ +The Solr field that holds the source document's ID on every chunk document when `tikaserver.chunks=true`. +Chunks are indexed as plain sibling documents referencing their parent by this field, not as Solr nested/block-join child documents. +This field must already exist in the schema. + +`tikaserver.config`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A raw JSON object sent as the per-request parser configuration for Tika Server (e.g., `{"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}`). +See <> below for details and an important security note: this requires `allowPerRequestConfig=true` on the Tika Server, which is off by default. ++ +Example: `tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}` + `tikaserver.maxChars`:: + [%autowidth,frame=none] @@ -540,9 +595,103 @@ So you can use the other URPs without worrying about unexpected field additions. === Parser-Specific Properties -Parser-specific properties for Tika must be configured directly on your Tika Server instance. Consult the https://tika.apache.org/[Apache Tika documentation] for details. +Server-wide parser properties (things that should apply to every request) must be configured directly on your Tika Server instance, via its own JSON configuration file. Consult the https://tika.apache.org/[Apache Tika documentation] for details. + +For a single request, you can instead pass parser-specific options through Solr using the `tikaserver.config` parameter, whose value is a raw JSON object matching Tika Server's per-request configuration format. +For example, to disable OCR for one request: + +[,console] +---- +$ bin/solr post -c gettingstarted example/exampledocs/solr-word.pdf --params 'literal.id=doc1&tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' +---- + +[IMPORTANT] +==== +`tikaserver.config` requires your Tika Server to be started with `allowPerRequestConfig: true` in its own JSON configuration (under the `server` section). +This is off by default, and Tika Server logs a warning when it is enabled, because it lets any client that can reach `/update/extract` inject arbitrary parser configuration, including options that spawn external processes such as OCR. +Only enable it if you need per-request configuration, and treat access to your Solr instance's extraction endpoint accordingly. + +[source,json] +---- +{ + "server": { + "allowPerRequestConfig": true + } +} +---- +==== + +`tikaserver.config` is combined with any password resolved from `resource.password` or `passwordsFile` (see <>) into a single request to Tika Server, and is only supported for non-recursive extraction (`tikaserver.recursive=false`, the default); Tika Server has no way to accept per-request configuration for recursive extraction while also returning the XHTML content Solr Cell needs. + +=== Indexing Chunks with Embeddings + +Tika 4.x's `tika-inference` module adds optional server-side embedding metadata filters (e.g. `openai-embedding-filter`) that split a parsed document's content into semantically-bounded chunks -- splitting on Markdown heading and paragraph boundaries -- and attach a precomputed embedding vector to each chunk. +Setting `tikaserver.chunks=true` tells Solr Cell to read those chunks back out and index one Solr document per chunk, each with its own text and embedding vector, instead of a single document for the whole source file. +This turns Solr Cell into a path straight from "binary document in" to "kNN-searchable chunks out," using Solr's existing xref:query-guide:dense-vector-search.adoc[dense vector search] support. -NOTE: In earlier versions of Solr Cell you could supply Tika configuration directly to Solr. This is no longer possible. +Chunking happens entirely on the Tika Server side; Solr Cell does not call an embeddings API itself. +You must configure the embedding filter directly in Tika Server's own JSON configuration file (the same mechanism used to start Tika Server with `-c`, see <>), for example: + +[source,json] +---- +{ + "server": {}, + "metadata-filters": [ + { + "openai-embedding-filter": { + "baseUrl": "https://api.openai.com", + "model": "text-embedding-3-small", + "apiKey": "${OPENAI_API_KEY}" + } + } + ] +} +---- + +Consult the https://tika.apache.org/[Apache Tika documentation] for the full set of embedding filter options. + +Before indexing, your collection's schema must already define the three fields `tikaserver.chunks` writes to: a text field for the chunk content (default `content`), a `DenseVectorField` for the embedding (default `vector`, with `vectorDimension` matching your embedding model's output), and a string field for the parent document ID (default `chunk_parent_id`). +For example: + +[source,bash] +---- +curl -X POST -H 'Content-type:application/json' -d '{ + "add-field-type": { + "name": "knn_vector", + "class": "solr.DenseVectorField", + "vectorDimension": 1536, + "similarityFunction": "cosine" + } +}' 'http://localhost:8983/solr/gettingstarted/schema' + +curl -X POST -H 'Content-type:application/json' -d '{ + "add-field": {"name": "vector", "type": "knn_vector", "indexed": true, "stored": true}, + "add-field": {"name": "chunk_parent_id", "type": "string", "indexed": true, "stored": true} +}' 'http://localhost:8983/solr/gettingstarted/schema' +---- + +With the schema in place, index a document with `tikaserver.chunks=true`: + +[source,bash] +---- +curl "http://localhost:8983/solr/gettingstarted/update/extract?tikaserver.chunks=true&literal.id=doc1&commit=true" \ + -F "myfile=@report.md" +---- + +Each resulting chunk document gets an ID of the form `-chunk-` (`doc1-chunk-0`, `doc1-chunk-1`, ...), a `chunk_parent_id` of `doc1`, and its own `content`/`vector`. +Any `literal.__fieldname__` parameters are copied onto every chunk document. +You can then run a kNN query scoped to (or across) a document's chunks: + +[source,bash] +---- +curl "http://localhost:8983/solr/gettingstarted/select?q={!knn+f=vector+topK=5}%5B0.01,...%5D&fq=chunk_parent_id:doc1" +---- + +[NOTE] +==== +This is a v1 feature: chunk documents are indexed as plain sibling documents (matching the "standard RAG pattern" used by LangChain, LlamaIndex, and Haystack), not as Solr nested/block-join child documents, and the `fmap`/`capture`/`captureAttr`/`xpath` parameters -- which operate on Tika's XHTML output -- have no effect in chunk mode. +If `tikaserver.chunks=true` but the Tika Server response carries no `tk:chunks` (for example, because no embedding filter is configured), Solr Cell raises an error rather than silently indexing nothing. +==== === Indexing Encrypted Documents diff --git a/solr/ui/gradle.lockfile b/solr/ui/gradle.lockfile index 2a0144af5101..140406144c3c 100644 --- a/solr/ui/gradle.lockfile +++ b/solr/ui/gradle.lockfile @@ -343,26 +343,26 @@ org.jetbrains.compose.ui:ui-wasm-js:1.11.1=wasmJsCompileClasspath,wasmJsNpmAggre org.jetbrains.compose.ui:ui:1.10.0=composeHotReloadDevTools org.jetbrains.compose.ui:ui:1.11.1=allDevSourceSetsCompileDependenciesMetadata,allSourceSetsCompileDependenciesMetadata,allTestSourceSetsCompileDependenciesMetadata,commonMainResolvableDependenciesMetadata,commonTestResolvableDependenciesMetadata,composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,jvmCompileClasspath,jvmDevCompileClasspath,jvmDevResolvableDependenciesMetadata,jvmDevRuntimeClasspath,jvmMainCompileClasspath,jvmMainResolvableDependenciesMetadata,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath,metadataCommonMainCompileClasspath,metadataCompileClasspath,wasmJsCompileClasspath,wasmJsMainResolvableDependenciesMetadata,wasmJsNpmAggregated,wasmJsRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestResolvableDependenciesMetadata,wasmJsTestRuntimeClasspath,webMainResolvableDependenciesMetadata,webTestResolvableDependenciesMetadata org.jetbrains.kotlin:kotlin-build-tools-api:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-build-tools-api:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-build-tools-api:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-build-tools-compat:2.4.0=kotlinAbiValidationCompatClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath org.jetbrains.kotlin:kotlin-build-tools-cri-impl:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-build-tools-cri-impl:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-build-tools-cri-impl:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-build-tools-impl:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-build-tools-impl:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-build-tools-impl:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-compiler-embeddable:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-compiler-embeddable:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-compiler-embeddable:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-compiler-runner:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-compiler-runner:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-compiler-runner:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-compose-compiler-plugin-embeddable:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest org.jetbrains.kotlin:kotlin-daemon-client:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-daemon-client:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-daemon-client:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-daemon-embeddable:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-daemon-embeddable:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-daemon-embeddable:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-klib-commonizer-embeddable:2.4.0=kotlinKlibCommonizerClasspath org.jetbrains.kotlin:kotlin-reflect:1.6.10=kotlinAbiValidationCompatClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinKlibCommonizerClasspath org.jetbrains.kotlin:kotlin-reflect:2.3.21=allDevSourceSetsCompileDependenciesMetadata,allSourceSetsCompileDependenciesMetadata,allTestSourceSetsCompileDependenciesMetadata,composeHotReloadDevDesktopDevRuntimeClasspath,composeHotReloadDevDesktopRuntimeClasspath,composeHotReloadDevDesktopTestRuntimeClasspath,composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,desktopCompileClasspath,desktopDevCompileClasspath,desktopDevResolvableDependenciesMetadata,desktopDevRuntimeClasspath,desktopMainCompileClasspath,desktopMainResolvableDependenciesMetadata,desktopMainRuntimeClasspath,desktopRuntimeClasspath,desktopTestCompileClasspath,desktopTestResolvableDependenciesMetadata,desktopTestRuntimeClasspath,jvmCompileClasspath,jvmDevCompileClasspath,jvmDevResolvableDependenciesMetadata,jvmDevRuntimeClasspath,jvmMainCompileClasspath,jvmMainResolvableDependenciesMetadata,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath org.jetbrains.kotlin:kotlin-script-runtime:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest,kotlinKlibCommonizerClasspath -org.jetbrains.kotlin:kotlin-script-runtime:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-script-runtime:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-scripting-common:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable:2.4.0=kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest @@ -375,12 +375,12 @@ org.jetbrains.kotlin:kotlin-stdlib-wasm-js:2.4.0=wasmJsCompileClasspath,wasmJsNp org.jetbrains.kotlin:kotlin-stdlib:2.1.21=composeHotReloadRuntime org.jetbrains.kotlin:kotlin-stdlib:2.2.21=composeHotReloadDevTools org.jetbrains.kotlin:kotlin-stdlib:2.4.0=allDevSourceSetsCompileDependenciesMetadata,allSourceSetsCompileDependenciesMetadata,allTestSourceSetsCompileDependenciesMetadata,commonMainResolvableDependenciesMetadata,commonTestResolvableDependenciesMetadata,composeHotReloadDevDesktopDevRuntimeClasspath,composeHotReloadDevDesktopRuntimeClasspath,composeHotReloadDevDesktopTestRuntimeClasspath,composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,desktopCompileClasspath,desktopDevCompileClasspath,desktopDevResolvableDependenciesMetadata,desktopDevRuntimeClasspath,desktopMainCompileClasspath,desktopMainResolvableDependenciesMetadata,desktopMainRuntimeClasspath,desktopRuntimeClasspath,desktopTestCompileClasspath,desktopTestResolvableDependenciesMetadata,desktopTestRuntimeClasspath,jvmCompileClasspath,jvmDevCompileClasspath,jvmDevResolvableDependenciesMetadata,jvmDevRuntimeClasspath,jvmMainCompileClasspath,jvmMainResolvableDependenciesMetadata,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath,kotlinBuildToolsApiClasspath,kotlinCompilerClasspath,kotlinCompilerPluginClasspathDesktopDev,kotlinCompilerPluginClasspathDesktopMain,kotlinCompilerPluginClasspathDesktopTest,kotlinCompilerPluginClasspathJvmDev,kotlinCompilerPluginClasspathJvmMain,kotlinCompilerPluginClasspathJvmTest,kotlinCompilerPluginClasspathMetadataCommonMain,kotlinCompilerPluginClasspathMetadataMain,kotlinCompilerPluginClasspathWasmJsMain,kotlinCompilerPluginClasspathWasmJsTest,kotlinKlibCommonizerClasspath,metadataCommonMainCompileClasspath,metadataCompileClasspath,wasmJsCompileClasspath,wasmJsMainResolvableDependenciesMetadata,wasmJsNpmAggregated,wasmJsRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestResolvableDependenciesMetadata,wasmJsTestRuntimeClasspath,webMainResolvableDependenciesMetadata,webTestResolvableDependenciesMetadata -org.jetbrains.kotlin:kotlin-stdlib:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-stdlib:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlin:kotlin-test-junit:2.4.0=composeHotReloadDevDesktopTestRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,desktopTestCompileClasspath,desktopTestRuntimeClasspath,jvmTestCompileClasspath,jvmTestRuntimeClasspath org.jetbrains.kotlin:kotlin-test-wasm-js:2.4.0=wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestRuntimeClasspath org.jetbrains.kotlin:kotlin-test:2.4.0=allTestSourceSetsCompileDependenciesMetadata,commonTestResolvableDependenciesMetadata,composeHotReloadDevJvmTestRuntimeClasspath,jvmTestCompileClasspath,jvmTestResolvableDependenciesMetadata,jvmTestRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestResolvableDependenciesMetadata,wasmJsTestRuntimeClasspath,webTestResolvableDependenciesMetadata org.jetbrains.kotlin:kotlin-tooling-core:2.4.0=kotlinBuildToolsApiClasspath,kotlinCompilerClasspath -org.jetbrains.kotlin:kotlin-tooling-core:2.4.20-RC=kotlinAbiValidationCompatClasspath +org.jetbrains.kotlin:kotlin-tooling-core:2.4.20-RC2=kotlinAbiValidationCompatClasspath org.jetbrains.kotlinx:atomicfu-jvm:0.23.2=composeHotReloadDevTools org.jetbrains.kotlinx:atomicfu-jvm:0.33.0=composeHotReloadDevJvmDevRuntimeClasspath,composeHotReloadDevJvmRuntimeClasspath,composeHotReloadDevJvmTestRuntimeClasspath,jvmDevRuntimeClasspath,jvmMainRuntimeClasspath,jvmRuntimeClasspath,jvmTestRuntimeClasspath org.jetbrains.kotlinx:atomicfu-wasm-js:0.33.0=wasmJsCompileClasspath,wasmJsNpmAggregated,wasmJsRuntimeClasspath,wasmJsTestCompileClasspath,wasmJsTestNpmAggregated,wasmJsTestRuntimeClasspath