diff --git a/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml b/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml new file mode 100644 index 000000000000..bf8bad38c896 --- /dev/null +++ b/changelog/unreleased/PR#4853-update-org-apache-tika-tika-core.yml @@ -0,0 +1,7 @@ +title: Update org.apache.tika:tika-core to v4 +type: dependency_update +authors: +- name: solrbot +links: +- name: PR#4853 + url: https://github.com/apache/solr/pull/4853 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d14b00312287..77af71b1a6b7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -42,7 +42,7 @@ apache-log4j = "2.26.1" apache-lucene = "10.4.0" apache-opennlp = "2.5.11" apache-rat = "0.18" -apache-tika = "3.3.2" +apache-tika = "4.0.0" apache-tomcat = "6.0.53" apache-zookeeper = "3.9.5" # @keep for version alignment diff --git a/solr/licenses/commonmark-0.30.0.jar.sha1 b/solr/licenses/commonmark-0.30.0.jar.sha1 new file mode 100644 index 000000000000..ef17e4e5f72b --- /dev/null +++ b/solr/licenses/commonmark-0.30.0.jar.sha1 @@ -0,0 +1 @@ +fc611136b3191a08d2b84c5d5a26116e3531afee diff --git a/solr/licenses/commonmark-LICENSE-BSD.txt b/solr/licenses/commonmark-LICENSE-BSD.txt new file mode 100644 index 000000000000..604b777d32c7 --- /dev/null +++ b/solr/licenses/commonmark-LICENSE-BSD.txt @@ -0,0 +1,23 @@ +Copyright (c) 2015, Robin Stocker +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/solr/licenses/commonmark-NOTICE.txt b/solr/licenses/commonmark-NOTICE.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 b/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 new file mode 100644 index 000000000000..79f99ad6fa50 --- /dev/null +++ b/solr/licenses/commonmark-ext-gfm-strikethrough-0.30.0.jar.sha1 @@ -0,0 +1 @@ +e9addacba704ffce26d6b8bcd1c50e235ec2c991 diff --git a/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 b/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 new file mode 100644 index 000000000000..8f580d0ef23c --- /dev/null +++ b/solr/licenses/commonmark-ext-gfm-tables-0.30.0.jar.sha1 @@ -0,0 +1 @@ +84560a4af5afe4808543bd9c5587a660ce148b14 diff --git a/solr/licenses/tika-core-4.0.0.jar.sha1 b/solr/licenses/tika-core-4.0.0.jar.sha1 new file mode 100644 index 000000000000..bab6900e1458 --- /dev/null +++ b/solr/licenses/tika-core-4.0.0.jar.sha1 @@ -0,0 +1 @@ +f370d8619de9e19329b2cc117943c83468c2ff9f diff --git a/solr/modules/extraction/gradle.lockfile b/solr/modules/extraction/gradle.lockfile index 40fd886cdca3..401e6ca1a3a6 100644 --- a/solr/modules/extraction/gradle.lockfile +++ b/solr/modules/extraction/gradle.lockfile @@ -117,11 +117,14 @@ org.apache.lucene:lucene-spatial-extras:10.4.0=jarValidation,runtimeClasspath,ru org.apache.lucene:lucene-spatial3d:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.lucene:lucene-suggest:10.4.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath org.apache.lucene:lucene-test-framework:10.4.0=jarValidation,testCompileClasspath,testRuntimeClasspath -org.apache.tika:tika-core:3.3.2=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.apache.tika:tika-core:4.0.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.apache.zookeeper:zookeeper-jute:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apache.zookeeper:zookeeper:3.9.5=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.apiguardian:apiguardian-api:1.1.2=jarValidation,testRuntimeClasspath org.codehaus.woodstox:stax2-api:4.3.0=jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testRuntimeClasspath +org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.commonmark:commonmark-ext-gfm-tables:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath +org.commonmark:commonmark:0.30.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.compression:jetty-compression-common:12.1.12=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.compression:jetty-compression-gzip:12.1.12=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,solrPlatformLibs,testCompileClasspath,testRuntimeClasspath org.eclipse.jetty.ee10:jetty-ee10-servlet:12.1.12=jarValidation,testRuntimeClasspath diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java index d4fd1c4cbd1b..d5d343846834 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java @@ -143,6 +143,7 @@ public void load( .tikaServerRecursive(tikaserverRecursive) .tikaServerTimeoutSeconds(tikaTimeoutSecs) .tikaServerRequestHeaders(Map.of()) + .tikaServerConfigJson(params.get(ExtractingParams.TIKASERVER_CONFIG_JSON)) .build(); boolean captureAttr = params.getBool(ExtractingParams.CAPTURE_ATTRIBUTES, false); diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java index a99025127b19..5a52f2ac02aa 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java @@ -157,4 +157,14 @@ public interface ExtractingParams { /** Default or per-request timeout in seconds for TikaServer HTTP calls. */ String TIKASERVER_TIMEOUT_SECS = "tikaserver.timeoutSeconds"; + + /** + * Optional raw JSON object sent as the "config" part of a per-request TikaServer configuration + * call (e.g. {@code {"pdf-parser":{"ocr":{"strategy":"no_ocr"}}}}). Tika Server 4.x removed its + * X-Tika-* configuration headers in favor of this JSON mechanism; the server must additionally + * have {@code allowPerRequestConfig=true} set, or the request is rejected with 403. Ignored for + * recursive (tikaserver.recursive) requests, since TikaServer has no XML-output variant of + * /rmeta/config. + */ + String TIKASERVER_CONFIG_JSON = "tikaserver.config"; } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java index 79068816271a..d34496772ce3 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractionRequest.java @@ -38,6 +38,7 @@ public class ExtractionRequest { public final boolean tikaServerRecursive; public final Integer tikaServerTimeoutSeconds; // optional per-request override public final Map tikaServerRequestHeaders = new HashMap<>(); + public final String tikaServerConfigJson; // optional raw JSON "config" part; TikaServer 4.x only /** * Constructs an ExtractionRequest object containing metadata and configurations for extraction @@ -59,6 +60,8 @@ public class ExtractionRequest { * only). If null or ≤ 0, the default timeout will be used * @param tikaServerRequestHeaders optional headers to be included in requests to the extraction * service. TikaServer only + * @param tikaServerConfigJson optional raw JSON object sent as the per-request "config" part. + * TikaServer 4.x only; requires allowPerRequestConfig=true on the server */ private ExtractionRequest( String streamType, @@ -73,7 +76,8 @@ private ExtractionRequest( String extractFormat, boolean tikaServerRecursive, Integer tikaServerTimeoutSeconds, - Map tikaServerRequestHeaders) { + Map tikaServerRequestHeaders, + String tikaServerConfigJson) { this.streamType = streamType; this.resourceName = resourceName; this.contentType = contentType; @@ -89,6 +93,7 @@ private ExtractionRequest( if (tikaServerRequestHeaders != null) { this.tikaServerRequestHeaders.putAll(tikaServerRequestHeaders); } + this.tikaServerConfigJson = tikaServerConfigJson; } /** Creates a new Builder for constructing ExtractionRequest instances. */ @@ -111,6 +116,7 @@ public static class Builder { private boolean tikaServerRecursive = false; private Integer tikaServerTimeoutSeconds; private Map tikaServerRequestHeaders; + private String tikaServerConfigJson; private Builder() {} @@ -179,6 +185,11 @@ public Builder tikaServerRequestHeaders(Map tikaServerRequestHea return this; } + public Builder tikaServerConfigJson(String tikaServerConfigJson) { + this.tikaServerConfigJson = tikaServerConfigJson; + return this; + } + public ExtractionRequest build() { return new ExtractionRequest( streamType, @@ -193,7 +204,8 @@ public ExtractionRequest build() { extractFormat, tikaServerRecursive, tikaServerTimeoutSeconds, - tikaServerRequestHeaders); + tikaServerRequestHeaders, + tikaServerConfigJson); } } } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java index a6d08bb11f77..395345087c9f 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerExtractionBackend.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PushbackInputStream; +import java.lang.invoke.MethodHandles; import java.net.ConnectException; import java.net.MalformedURLException; import java.net.SocketTimeoutException; @@ -37,15 +39,23 @@ import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.Utils; import org.apache.solr.util.RefCounted; import org.apache.tika.sax.BodyContentHandler; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.InputStreamRequestContent; import org.eclipse.jetty.client.InputStreamResponseListener; +import org.eclipse.jetty.client.MultiPartRequestContent; import org.eclipse.jetty.client.Request; import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.client.StringRequestContent; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.MultiPart; import org.eclipse.jetty.io.EofException; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.helpers.DefaultHandler; /** @@ -53,6 +63,8 @@ * import of org.apache.tika.sax.BodyContentHandler; */ public class TikaServerExtractionBackend implements ExtractionBackend { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + /** * Default maximum response size (100MB) to prevent excessive memory usage from large documents */ @@ -173,26 +185,81 @@ public void extractWithSaxHandler( * request.tikaserverRecursive */ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) throws Exception { - String url = baseUrl + (request.tikaServerRecursive ? "/rmeta" : "/tika"); + ExtractionMetadata md = buildMetadataFromRequest(request); + String pwd = resolvePassword(request, md); + String configJson = resolveConfigJson(request, pwd); HttpClient client = acquiredResourcesRef.get().client; - - Request req = client.newRequest(url).method("PUT"); Duration effectiveTimeout = (request.tikaServerTimeoutSeconds != null && request.tikaServerTimeoutSeconds > 0) ? Duration.ofSeconds(request.tikaServerTimeoutSeconds) : defaultTimeout; - req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); - // Also set idle timeout in case of heavy server side work like OCR - req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); - - // Headers - String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); - req.headers(h -> h.add("Accept", accept)); String contentType = (request.streamType != null) ? request.streamType : request.contentType; - if (contentType != null) { - req.headers(h -> h.add("Content-Type", contentType)); + + String url; + Request req; + if (configJson != null && request.tikaServerRecursive) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Per-request TikaServer config (password or " + + ExtractingParams.TIKASERVER_CONFIG_JSON + + ") is not supported together with " + + ExtractingParams.TIKASERVER_RECURSIVE + + "=true: TikaServer 4.x has no XML-output variant of /rmeta/config."); + } + if (configJson != null) { + // Tika 4.x dropped its X-Tika-* configuration headers (including Password) in favor of a + // per-request JSON "config" part on a multipart request; the server must additionally opt + // in with allowPerRequestConfig=true. There is no XML content-handler variant of + // /rmeta/config in Tika 4.x, so this path only covers non-recursive extraction (checked + // above). + url = baseUrl + "/tika/config/xml"; + req = client.newRequest(url).method("POST"); + req.headers(h -> h.add("Accept", "text/xml")); + + HttpFields.Mutable fileFields = HttpFields.build(); + if (contentType != null) { + fileFields.add(HttpHeader.CONTENT_TYPE, contentType); + } + try (MultiPartRequestContent multiPart = new MultiPartRequestContent()) { + multiPart.addPart( + new MultiPart.ContentSourcePart( + "file", + request.resourceName, + fileFields, + new InputStreamRequestContent(inputStream))); + multiPart.addPart( + new MultiPart.ContentSourcePart( + "config", + null, + HttpFields.build().add(HttpHeader.CONTENT_TYPE, "application/json"), + new StringRequestContent(configJson))); + req.body(multiPart); + } + } else { + // Tika 4.x's default content handler is Markdown, not XHTML/XML (TIKA-4663); Solr's SAX-based + // content handling needs the previous XHTML/XML output, requested via the /xml path variants. + url = baseUrl + (request.tikaServerRecursive ? "/rmeta/xml" : "/tika/xml"); + req = client.newRequest(url).method("PUT"); + String accept = (request.tikaServerRecursive ? "application/json" : "text/xml"); + req.headers(h -> h.add("Accept", accept)); + if (contentType != null) { + req.headers(h -> h.add("Content-Type", contentType)); + } + if (request.resourceName != null) { + req.headers( + h -> + h.add( + "Content-Disposition", + "attachment; filename=\"" + request.resourceName + "\"")); + } + if (contentType != null) { + req.body(new InputStreamRequestContent(contentType, inputStream)); + } else { + req.body(new InputStreamRequestContent(inputStream)); + } } + if (!request.tikaServerRequestHeaders.isEmpty()) { req.headers( h -> @@ -202,32 +269,9 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t })); } - ExtractionMetadata md = buildMetadataFromRequest(request); - if (request.resourcePassword != null || request.passwordsMap != null) { - RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider(); - if (request.resourcePassword != null) { - passwordProvider.setExplicitPassword(request.resourcePassword); - } - if (request.passwordsMap != null) { - passwordProvider.setPasswordMap(request.passwordsMap); - } - String pwd = passwordProvider.getPassword(md); - if (pwd != null) { - req.headers(h -> h.add("Password", pwd)); // Tika Server expects this header if provided - } - } - if (request.resourceName != null) { - req.headers( - h -> - h.add( - "Content-Disposition", "attachment; filename=\"" + request.resourceName + "\"")); - } - - if (contentType != null) { - req.body(new InputStreamRequestContent(contentType, inputStream)); - } else { - req.body(new InputStreamRequestContent(inputStream)); - } + req.timeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); + // Also set idle timeout in case of heavy server side work like OCR + req.idleTimeout(effectiveTimeout.toMillis(), TimeUnit.MILLISECONDS); InputStreamResponseListener listener = new InputStreamResponseListener(); req.send(listener); @@ -273,7 +317,33 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t } int code = response.getStatus(); - if (code < 200 || code >= 300) { + InputStream responseStream = listener.getInputStream(); + // Tika 4.x's raw /tika* endpoints (non-recursive) return 422 whenever a container-level + // exception occurred during parsing -- including a non-aborting one like a writeLimit + // truncation -- but the body still carries whatever content was successfully extracted + // (there's no envelope to carry the exception itself on these endpoints; use /rmeta for + // that). A request that extracted nothing at all (e.g. a wrong password) also gets 422, but + // with an empty body -- treat that case as the failure it is instead of a silent empty + // "success". Peek the first byte to tell the two apart. + if (code == 422 && !request.tikaServerRecursive) { + PushbackInputStream peekable = new PushbackInputStream(responseStream, 1); + int firstByte = peekable.read(); + if (firstByte == -1) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "TikaServer " + + url + + " returned status 422 (Unprocessable Entity) with no content -- the document" + + " could not be parsed at all (check the password, if one was required)."); + } + peekable.unread(firstByte); + log.warn( + "TikaServer {} returned 422 (a container-level exception occurred during parsing); " + + "using the partial content it still returned. Use tikaserver.recursive=true " + + "against /rmeta for the exception detail.", + url); + responseStream = peekable; + } else if (code < 200 || code >= 300) { SolrException.ErrorCode errorCode = SolrException.ErrorCode.getErrorCode(code); String reason = response.getReason(); String msg = @@ -285,11 +355,56 @@ InputStream callTikaServer(InputStream inputStream, ExtractionRequest request) t throw new SolrException(errorCode, msg); } - InputStream responseStream = listener.getInputStream(); // Bound the amount of data we read from Tika Server to avoid excessive memory/CPU usage return new LimitingInputStream(responseStream, maxCharsLimit); } + /** Resolves the password to use for an encrypted document, or null if none applies. */ + private String resolvePassword(ExtractionRequest request, ExtractionMetadata md) { + if (request.resourcePassword == null && request.passwordsMap == null) { + return null; + } + RegexRulesPasswordProvider passwordProvider = new RegexRulesPasswordProvider(); + if (request.resourcePassword != null) { + passwordProvider.setExplicitPassword(request.resourcePassword); + } + if (request.passwordsMap != null) { + passwordProvider.setPasswordMap(request.passwordsMap); + } + return passwordProvider.getPassword(md); + } + + /** + * Builds the per-request TikaServer JSON "config" payload, merging any resolved password with any + * caller-supplied {@link ExtractingParams#TIKASERVER_CONFIG_JSON}. Returns null if neither + * applies, meaning no per-request config is needed. + */ + @SuppressWarnings("unchecked") + private String resolveConfigJson(ExtractionRequest request, String pwd) { + Map config = new LinkedHashMap<>(); + if (request.tikaServerConfigJson != null && !request.tikaServerConfigJson.isBlank()) { + Object parsed; + try { + parsed = Utils.fromJSONString(request.tikaServerConfigJson); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Invalid JSON in " + ExtractingParams.TIKASERVER_CONFIG_JSON + ": " + e.getMessage(), + e); + } + if (!(parsed instanceof Map)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + ExtractingParams.TIKASERVER_CONFIG_JSON + " must be a JSON object"); + } + config.putAll((Map) parsed); + } + if (pwd != null && !config.containsKey("simple-password-provider")) { + config.put("simple-password-provider", Map.of("password", pwd)); + } + return config.isEmpty() ? null : Utils.toJSONString(config); + } + private static class LimitingInputStream extends InputStream { private final InputStream in; private final long max; diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java index e34514040b6e..41f9b83364c1 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/TikaServerParser.java @@ -87,11 +87,14 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe for (Object o : list) { if (!(o instanceof Map)) continue; Map map = (Map) o; + // Tika 4.x renamed X-TIKA:content to tk:content (TIKA-4816); accept either so this also + // works against a Tika Server 3.x still on the old key. + String contentKey = map.containsKey("tk:content") ? "tk:content" : "X-TIKA:content"; // Copy metadata for (Object k : map.keySet()) { String key = String.valueOf(k); Object val = map.get(k); - if ("X-TIKA:content".equalsIgnoreCase(key)) { + if (contentKey.equalsIgnoreCase(key)) { // handled below continue; } @@ -103,7 +106,7 @@ void parseRmetaJson(InputStream jsonStream, DefaultHandler handler, ExtractionMe md.add(key, String.valueOf(val)); } } - Object content = map.get("X-TIKA:content"); + Object content = map.get(contentKey); if (content != null) { String xhtml = String.valueOf(content); if (!xhtml.isEmpty() && handler != null) { diff --git a/solr/modules/extraction/src/test-files/extraction/tika-server-config.json b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json new file mode 100644 index 000000000000..63a1e4fcbd77 --- /dev/null +++ b/solr/modules/extraction/src/test-files/extraction/tika-server-config.json @@ -0,0 +1,5 @@ +{ + "server": { + "allowPerRequestConfig": true + } +} diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java index c9c872bc99de..f6f4cef080a1 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTestAbstract.java @@ -440,17 +440,20 @@ public void testLiterals() throws Exception { "one", "literal.extractionLiteral", "two", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-parser-override", + "ignored_parser", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.Last-Modified", "extractedDate"); @@ -480,17 +483,20 @@ public void testLiterals() throws Exception { "extractedLanguage", "literal.extractionLiteral", "one", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.Last-Modified", "extractedDate"); @@ -596,21 +602,26 @@ public void testPlainTextSpecifyingMimeType() throws Exception { "one", "fmap.language", "extractedLanguage", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:detected-encoding", + "ignored_parser", + "fmap.tk:encoding-detector", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:encoding-detection-trace", "ignored_parser", - "fmap.X-TIKA:detectedEncoding", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:encodingDetector", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:content-type-magic-detected", "ignored_parser", "fmap.content", "extractedContent", @@ -644,21 +655,28 @@ public void testPlainTextSpecifyingResourceName() throws Exception { "one", "fmap.language", "extractedLanguage", - "fmap.X-Parsed-By", + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + "fmap.tk:parsed-by", + "ignored_parser", + "fmap.tk:detected-encoding", + "ignored_parser", + "fmap.tk:encoding-detector", + "ignored_parser", + "fmap.tk:encoding-detection-trace", "ignored_parser", - "fmap.X-TIKA:Parsed-By", + "fmap.tk:parsed-by-full-set", "ignored_parser", - "fmap.X-TIKA:detectedEncoding", + "fmap.tk:content-handler-type", "ignored_parser", - "fmap.X-TIKA:encodingDetector", + "fmap.tk:parse-time-millis", "ignored_parser", - "fmap.X-TIKA:Parsed-By-Full-Set", + "fmap.tk:embedded-depth", "ignored_parser", - "fmap.X-TIKA:content_handler", + "fmap.tk:content-type-parser-override", "ignored_parser", - "fmap.X-TIKA:parse_time_millis", + "fmap.tk:content-type-magic-detected", "ignored_parser", - "fmap.X-TIKA:embedded_depth", + "fmap.tk:resource-name", "ignored_parser", "fmap.content", "extractedContent", diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java index f32dbb5289c8..7f31e8fcd6d6 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTikaServerTest.java @@ -31,7 +31,8 @@ public class ExtractingRequestHandlerTikaServerTest extends ExtractingRequestHan private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @ClassRule - public static final TikaServerContainerRule tikaContainer = new TikaServerContainerRule(); + public static final TikaServerContainerRule tikaContainer = + new TikaServerContainerRule(getFile("extraction/tika-server-config.json")); @BeforeClass public static void beforeClassTika() throws Exception { diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java index 7a6ae3937998..368bdb0c75e1 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerContainerRule.java @@ -17,6 +17,7 @@ package org.apache.solr.handler.extraction; import java.lang.invoke.MethodHandles; +import java.nio.file.Path; import org.junit.Assume; import org.junit.rules.ExternalResource; import org.slf4j.Logger; @@ -24,6 +25,7 @@ import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; /** * JUnit rule that manages a single Apache Tika Server Testcontainer. Declare as a @@ -38,11 +40,21 @@ public class TikaServerContainerRule extends ExternalResource { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String TIKA_DOCKER_IMAGE = "apache/tika:3.2.3.0-full"; + public static final String TIKA_DOCKER_IMAGE = "apache/tika:4.0.0-full"; + private final Path serverConfigFile; private GenericContainer tika; private String baseUrl; + /** + * @param serverConfigFile optional Tika Server JSON config file to mount and start the container + * with (via {@code -c}), e.g. to set {@code allowPerRequestConfig: true}. Null for the + * container's default configuration. + */ + public TikaServerContainerRule(Path serverConfigFile) { + this.serverConfigFile = serverConfigFile; + } + @Override @SuppressWarnings("resource") protected void before() { @@ -56,6 +68,10 @@ protected void before() { new GenericContainer<>(TIKA_DOCKER_IMAGE) .withExposedPorts(9998) .waitingFor(Wait.forListeningPort()); + if (serverConfigFile != null) { + tika.withCopyFileToContainer(MountableFile.forHostPath(serverConfigFile), "/tika-config.json") + .withCommand("-c", "/tika-config.json"); + } tika.start(); baseUrl = "http://" + tika.getHost() + ":" + tika.getMappedPort(9998); } diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java index 326ab818596c..1a677bfc3877 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TikaServerExtractionBackendTest.java @@ -58,7 +58,8 @@ public boolean reject(Thread t) { } @ClassRule - public static final TikaServerContainerRule tikaContainer = new TikaServerContainerRule(); + public static final TikaServerContainerRule tikaContainer = + new TikaServerContainerRule(getFile("extraction/tika-server-config.json")); private static ExtractionRequest newRequest( String resourceName, @@ -124,14 +125,15 @@ public void testPdfWithImageRecursive() throws Exception { try (TikaServerExtractionBackend backend = new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); - // Enable recursive extraction and set header to extract images from PDF + // Tika 4.x removed the X-Tika-* header family entirely (see resolveConfigJson's javadoc); + // there is no replacement for this combination. Per-request config now requires the + // multipart /config endpoints, but Tika 4.x has no XML-output variant of /rmeta/config, so + // per-request PDF options (e.g. explicit inline-image extraction) cannot be requested + // together with tikaserver.recursive=true. The PDF's embedded image still gets OCR'd into + // the main document's content by default, just not exposed as a separate embedded + // resource entry the way the pre-4.x X-Tika-PDFextractInlineImages header used to. ExtractionRequest request = - newRequest( - "pdf-with-image.pdf", - "application/pdf", - "xml", - true, - Map.of("X-Tika-PDFextractInlineImages", "true")); + newRequest("pdf-with-image.pdf", "application/pdf", "xml", true, Map.of()); try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { ToXMLContentHandler xmlHandler = new ToXMLContentHandler(); ExtractionMetadata md = backend.buildMetadataFromRequest(request); @@ -139,9 +141,8 @@ public void testPdfWithImageRecursive() throws Exception { String c = xmlHandler.toString(); assertNotNull(c); assertTrue(c.contains("Puppet Apply")); - assertTrue(c.contains("embedded:image0.jpg")); - assertEquals( - "org.apache.tika.parser.DefaultParser", md.getFirst("X-TIKA:Parsed-By-Full-Set")); + // Tika 4.x renamed its metadata keys under a single lowercase tk: prefix (TIKA-4816) + assertEquals("org.apache.tika.parser.DefaultParser", md.getFirst("tk:parsed-by-full-set")); } } } @@ -195,4 +196,103 @@ public void testMaxCharsLimitEnforcedWithSaxHandler() throws Exception { } } } + + private static ExtractionRequest newRequestWithConfig( + String resourceName, String contentType, String extractFormat, String configJson) { + return ExtractionRequest.builder() + .streamType(contentType) + .resourceName(resourceName) + .contentType(contentType) + .streamName(resourceName) + .extractFormat(extractFormat) + .tikaServerConfigJson(configJson) + .build(); + } + + @Test + public void testConfigJsonDisablesOcr() throws Exception { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { + byte[] data = Files.readAllBytes(getFile("extraction/pdf-with-image.pdf")); + // With no config, the PDF's embedded image gets OCR'd and "Puppet Apply" (from the image) + // appears in the extracted content. Disabling OCR via tikaserver.config should suppress it. + ExtractionRequest request = + newRequestWithConfig( + "pdf-with-image.pdf", + "application/pdf", + "xml", + "{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + ExtractionResult res = backend.extract(in, request); + assertNotNull(res.getContent()); + assertFalse( + "Expected tikaserver.config's NO_OCR strategy to suppress the OCR'd image text", + res.getContent().contains("Puppet Apply")); + } + } + } + + @Test + public void testConfigJsonMergesWithPassword() throws Exception { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { + byte[] data = Files.readAllBytes(getFile("extraction/encrypted-password-is-solrRules.pdf")); + ExtractionRequest request = + ExtractionRequest.builder() + .streamType("application/pdf") + .resourceName("encrypted-password-is-solrRules.pdf") + .contentType("application/pdf") + .streamName("encrypted-password-is-solrRules.pdf") + .extractFormat("xml") + .resourcePassword("solrRules") + .tikaServerConfigJson("{\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}}") + .build(); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + ExtractionResult res = backend.extract(in, request); + assertNotNull(res); + assertTrue( + "Expected the password-unlocked content to still be present alongside the merged" + + " tikaserver.config", + res.getContent().contains("This is a test of PDF and Word extraction")); + } + } + } + + @Test + public void testInvalidConfigJsonRejected() throws Exception { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { + byte[] data = "hello".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = + newRequestWithConfig("test.txt", "text/plain", "xml", "not valid json"); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_CONFIG_JSON)); + } + } + } + + @Test + public void testConfigJsonRejectedForRecursive() throws Exception { + try (TikaServerExtractionBackend backend = + new TikaServerExtractionBackend(tikaContainer.getBaseUrl())) { + byte[] data = "hello".getBytes(StandardCharsets.UTF_8); + ExtractionRequest request = + ExtractionRequest.builder() + .streamType("text/plain") + .resourceName("test.txt") + .contentType("text/plain") + .streamName("test.txt") + .extractFormat("xml") + .tikaServerRecursive(true) + .tikaServerConfigJson("{\"parse-context\":{}}") + .build(); + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + SolrException e = expectThrows(SolrException.class, () -> backend.extract(in, request)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + assertTrue(e.getMessage().contains(ExtractingParams.TIKASERVER_RECURSIVE)); + } + } + } } diff --git a/solr/packaging/test/test_extraction.bats b/solr/packaging/test/test_extraction.bats index 4b8e62ae59cc..0823377452c6 100644 --- a/solr/packaging/test/test_extraction.bats +++ b/solr/packaging/test/test_extraction.bats @@ -40,7 +40,7 @@ apply_extract_handler() { setup_file() { if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then export TIKA_PORT=$((SOLR_PORT+5)) - docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:3.2.3.0-full >/dev/null 2>&1 || true + docker run --rm -p ${TIKA_PORT}:9998 --name bats_tika -d apache/tika:4.0.0-full >/dev/null 2>&1 || true echo "Waiting for Tika Server to be ready on port ${TIKA_PORT}" >&3 if ! wait_for 120 3 curl -s -f "http://localhost:${TIKA_PORT}/tika" -o /dev/null; then export DOCKER_UNAVAILABLE=1 diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc index db34f483b69f..d58038d509e3 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-diy.adoc @@ -39,7 +39,7 @@ Indexing binary files (PDF, DOCX, PPTX, etc.) with the Post Tool requires the So .Start a Tika Server quickly using Docker (exposes port 9998 on localhost) [,bash] ---- -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full ---- === Start Solr with the extraction module enabled diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index bc9e1905ec61..e1da9b61d227 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -78,7 +78,7 @@ The quickest way to run Tika Server for development is using Docker. The example [,bash] ---- -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full ---- NOTE: If Solr runs in Docker too, ensure both containers share a network and use the Tika container name as the host in `tikaserver.url`. @@ -92,7 +92,7 @@ First we start a tika server on port 9998, using Docker. [source,bash] ---- # Start Tika Server in the background -docker run --rm -p 9998:9998 --name tika -d apache/tika:3.2.3.0-full +docker run --rm -p 9998:9998 --name tika -d apache/tika:4.0.0-full # To stop the server when done, run `docker stop tika` ---- @@ -384,6 +384,18 @@ Example: `passwordsFile=/path/to/passwords.txt` // + // Only applicable for `tikaserver` backend. Can only be set in `solrconfig.xml`, not per request. +`tikaserver.config`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A raw JSON object sent as the per-request parser configuration for Tika Server (e.g., `{"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}`). +See <> below for details and an important security note: this requires `allowPerRequestConfig=true` on the Tika Server, which is off by default. ++ +Example: `tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}` + `tikaserver.maxChars`:: + [%autowidth,frame=none] @@ -540,9 +552,33 @@ So you can use the other URPs without worrying about unexpected field additions. === Parser-Specific Properties -Parser-specific properties for Tika must be configured directly on your Tika Server instance. Consult the https://tika.apache.org/[Apache Tika documentation] for details. +Server-wide parser properties (things that should apply to every request) must be configured directly on your Tika Server instance, via its own JSON configuration file. Consult the https://tika.apache.org/[Apache Tika documentation] for details. + +For a single request, you can instead pass parser-specific options through Solr using the `tikaserver.config` parameter, whose value is a raw JSON object matching Tika Server's per-request configuration format. +For example, to disable OCR for one request: + +[,console] +---- +$ bin/solr post -c gettingstarted example/exampledocs/solr-word.pdf --params 'literal.id=doc1&tikaserver.config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' +---- + +[IMPORTANT] +==== +`tikaserver.config` requires your Tika Server to be started with `allowPerRequestConfig: true` in its own JSON configuration (under the `server` section). +This is off by default, and Tika Server logs a warning when it is enabled, because it lets any client that can reach `/update/extract` inject arbitrary parser configuration, including options that spawn external processes such as OCR. +Only enable it if you need per-request configuration, and treat access to your Solr instance's extraction endpoint accordingly. + +[source,json] +---- +{ + "server": { + "allowPerRequestConfig": true + } +} +---- +==== -NOTE: In earlier versions of Solr Cell you could supply Tika configuration directly to Solr. This is no longer possible. +`tikaserver.config` is combined with any password resolved from `resource.password` or `passwordsFile` (see <>) into a single request to Tika Server, and is only supported for non-recursive extraction (`tikaserver.recursive=false`, the default); Tika Server has no way to accept per-request configuration for recursive extraction while also returning the XHTML content Solr Cell needs. === Indexing Encrypted Documents