diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index 46179df97..7d4663e81 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -162,11 +162,43 @@ public void execute(Tuple tuple) { // check that the mimetype is in the whitelist if (!mimeTypeWhiteList.isEmpty()) { boolean mt_match = false; - // see if a mimetype was guessed in JSOUPBolt + // parse.Content-Type is assumed byte-detected (JSoupParserBolt uses Tika detection, + // not the raw server header). A custom upstream writing a header-copied value bypasses + // this check — that is a caller responsibility. String mimeType = metadata.getFirstValue("parse.Content-Type"); - // otherwise rely on what could have been obtained from HTTP if (mimeType == null) { - mimeType = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); + // parse.Content-Type is absent: detect from content bytes so that + // the whitelist is evaluated against the same type Tika will use + // to select a parser, not the server-declared HTTP header which is + // untrusted and may differ from what the bytes actually are. + String httpCTHint = + metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix); + org.apache.tika.metadata.Metadata detectionMd = + new org.apache.tika.metadata.Metadata(); + if (StringUtils.isNotBlank(httpCTHint)) { + // pass the header as a hint only — detect() weighs it but + // content bytes take precedence + detectionMd.set(org.apache.tika.metadata.Metadata.CONTENT_TYPE, httpCTHint); + } + // pass the filename so detection matches what the parser dispatches on; + // without it, an ambiguous byte sequence (e.g. plain text with a .html + // extension) can resolve differently here than at parse time + try { + URL _url = URLUtil.toURL(url); + detectionMd.set(TikaCoreProperties.RESOURCE_NAME_KEY, _url.getFile()); + } catch (MalformedURLException e1) { + throw new IllegalStateException("Malformed URL", e1); + } + try { + mimeType = tika.detect(new ByteArrayInputStream(content), detectionMd); + } catch (IOException e) { + LOG.warn("Failed to detect MIME type for {}: {}", url, e.getMessage()); + } + if (mimeType != null) { + // write back so downstream code and metadata consumers see + // the same value (avoids a second detection pass) + metadata.setValue("parse.Content-Type", mimeType); + } } if (mimeType != null) { for (Pattern mt : mimeTypeWhiteList) { diff --git a/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java new file mode 100644 index 000000000..30faaae2c --- /dev/null +++ b/external/tika/src/test/java/org/apache/stormcrawler/tika/ParserBoltWhitelistDetectionTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.tika; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.http.HttpHeaders; +import org.apache.storm.task.OutputCollector; +import org.apache.stormcrawler.Constants; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.TestUtil; +import org.apache.stormcrawler.parse.ParsingTester; +import org.apache.stormcrawler.persistence.Status; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Regression test for: when no parse.Content-Type is present, ParserBolt must evaluate + * parser.mimetype.whitelist against the byte-detected MIME type (via tika.detect()), not the + * server-declared HTTP Content-Type header. Previously the whitelist checked the header while + * Tika's AutoDetectParser dispatched on the bytes, allowing a server to claim a whitelisted type + * while serving arbitrary content. + */ +class ParserBoltWhitelistDetectionTest extends ParsingTester { + + @BeforeEach + void setupParserBolt() { + bolt = new ParserBolt(); + setupParserBolt(bolt); + } + + /** + * The whitelist allows Word documents (application/.+word.*). The server header claims Word, + * but the body bytes are plain HTML. After the fix, detection on bytes yields text/html which + * does NOT match the whitelist, so the document must be rejected with ERROR. + */ + @Test + void whitelistAppliesToTheDetectedType() throws IOException { + Map conf = new HashMap<>(); + // the whitelist shipped by the archetypes + conf.put("parser.mimetype.whitelist", "application/.+word.*"); + conf.put(ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "http."); + bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + + // no parse.Content-Type: no JSoupParserBolt upstream, or detect.mimetype disabled + Metadata metadata = new Metadata(); + metadata.addValue( + "http." + HttpHeaders.CONTENT_TYPE, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + + // the body is NOT a word document + byte[] content = + "

not a word document

" + .getBytes(StandardCharsets.UTF_8); + parse("https://example.org/doc.docx", content, metadata); + + System.out.println("detected type: " + metadata.getFirstValue("parse.Content-Type")); + System.out.println("emitted documents: " + output.getEmitted().size()); + + List> status = output.getEmitted(Constants.StatusStreamName); + Assertions.assertEquals( + 1, status.size(), "content not matching the whitelist should be rejected"); + Assertions.assertEquals( + Status.ERROR, + status.get(0).get(2), + "status should be ERROR for mismatched content type"); + } + + /** + * Sanity check: when parse.Content-Type IS already present (e.g. set by JSoupParserBolt), the + * whitelist check must still use it directly and not re-detect. + */ + @Test + void whitelistUsesPreexistingParsedContentType() throws IOException { + Map conf = new HashMap<>(); + conf.put("parser.mimetype.whitelist", "text/html.*"); + conf.put(ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "http."); + bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + + Metadata metadata = new Metadata(); + // simulate JSoupParserBolt having detected the type already + metadata.addValue("parse.Content-Type", "text/html; charset=UTF-8"); + metadata.addValue("http." + HttpHeaders.CONTENT_TYPE, "text/html; charset=UTF-8"); + + byte[] content = "

hello

".getBytes(StandardCharsets.UTF_8); + parse("https://example.org/index.html", content, metadata); + + // document should pass the whitelist and be emitted (no ERROR on status stream) + List> status = output.getEmitted(Constants.StatusStreamName); + boolean hasError = + status != null && status.stream().anyMatch(row -> Status.ERROR.equals(row.get(2))); + Assertions.assertFalse(hasError, "whitelisted HTML document should not be rejected"); + } + + /** + * Plain-text bytes with a .html URL extension. Without the filename hint, Tika resolves the + * ambiguous bytes as text/plain; with it, the extension pushes detection to text/html. The + * whitelist is set to text/html.*, so the document must be accepted — verifying that the same + * RESOURCE_NAME_KEY hint is passed to both the whitelist check and the parser dispatch. + */ + @Test + void filenameHintInfluencesDetection() throws IOException { + Map conf = new HashMap<>(); + conf.put("parser.mimetype.whitelist", "text/html.*"); + conf.put(ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "http."); + bolt.prepare(conf, TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + + // no parse.Content-Type, no Content-Type header — detection relies on bytes + filename + Metadata metadata = new Metadata(); + + // plain text bytes: no HTML magic, ambiguous without the filename hint + byte[] content = "just some plain text, no html tags".getBytes(StandardCharsets.UTF_8); + + // .html extension should push detection to text/html + parse("https://example.org/page.html", content, metadata); + + System.out.println("detected type: " + metadata.getFirstValue("parse.Content-Type")); + + List> status = output.getEmitted(Constants.StatusStreamName); + boolean hasError = + status != null && status.stream().anyMatch(row -> Status.ERROR.equals(row.get(2))); + Assertions.assertFalse( + hasError, "document with .html URL should be accepted by text/html.* whitelist"); + } +}