From df104ab49d93a3586ed39c5a72cd4520014ada12 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 29 Jul 2026 19:25:13 +0200 Subject: [PATCH 1/2] WARC writer: WARC-Protocol header to follow WARC field proposals (fixes #1998) OkHttp protocol: add protocol response header key `_cipher_suites_` to hold the SSL/TLS Cipher suite separate from `_protocol_versions_`. WARC writer: 1. add WARC header `WARC-Cipher-Suite` 2. split multiple values in `WARC-Protocol` header and repeat header --- .../stormcrawler/protocol/ProtocolResponse.java | 6 ++++++ .../stormcrawler/protocol/okhttp/HttpProtocol.java | 4 +++- .../apache/stormcrawler/warc/WARCRecordFormat.java | 9 ++++++++- .../apache/stormcrawler/warc/WARCHdfsBoltTest.java | 11 ++++++++++- .../stormcrawler/warc/WARCRecordFormatTest.java | 14 ++++++++++---- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index c465ff000..31e78cd9c 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -45,6 +45,12 @@ public class ProtocolResponse { */ public static final String PROTOCOL_VERSIONS_KEY = "_protocol_versions_"; + /** + * Key which holds the SSL/TLS cipher suites. For requests sent over http:// the value may be + * null. + */ + public static final String CIPHER_SUITES_KEY = "_cipher_suites_"; + /** * Metadata key which holds a boolean value in metadata whether the response content is trimmed * or not. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 7717be59c..8f052563d 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -905,10 +905,11 @@ public Response intercept(Interceptor.Chain chain) throws IOException { .getBytes(StandardCharsets.ISO_8859_1)); final StringBuilder protocols = new StringBuilder(response.protocol().toString()); + String cipherSuite = null; final Handshake handshake = connection.handshake(); if (handshake != null) { protocols.append(',').append(handshake.tlsVersion()); - protocols.append(',').append(handshake.cipherSuite()); + cipherSuite = handshake.cipherSuite().toString(); } // returns a modified version of the response @@ -922,6 +923,7 @@ public Response intercept(Interceptor.Chain chain) throws IOException { .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress) .header(ProtocolResponse.REQUEST_TIME_KEY, Long.toString(startFetchTime)) .header(ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString()) + .header(ProtocolResponse.CIPHER_SUITES_KEY, cipherSuite) .build(); } } diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index 473c0f137..00224169f 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -616,7 +616,14 @@ public byte[] format(Tuple tuple) { metadata.getFirstValue( ProtocolResponse.PROTOCOL_VERSIONS_KEY, this.protocolMDprefix); if (protocolVersions != null) { - buffer.append("WARC-Protocol: ").append(protocolVersions).append(CRLF); + for (String val : StringUtils.split(protocolVersions, ',')) { + buffer.append("WARC-Protocol: ").append(val).append(CRLF); + } + } + final String cipherSuites = + metadata.getFirstValue(ProtocolResponse.CIPHER_SUITES_KEY, this.protocolMDprefix); + if (cipherSuites != null) { + buffer.append("WARC-Cipher-Suite: ").append(cipherSuites).append(CRLF); } buffer.append("WARC-Payload-Digest").append(": ").append(payloadDigest).append(CRLF); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index 2bc8340f2..dee09592a 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -130,6 +130,13 @@ void testHttp2() throws IOException { assertTrue( response.headers().first("WARC-Protocol").isPresent(), "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertEquals( + 2, + response.headers().all("WARC-Protocol").size(), + "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertTrue( + response.headers().first("WARC-Cipher-Suite").isPresent(), + "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); assertTrue( response.headers().first("WARC-IP-Address").isPresent(), "WARC response record is expected to include WARC header \"WARC-IP-Address\""); @@ -241,7 +248,9 @@ private Tuple getPage(String httpVersionString) { + "Connection: close\r\n\r\n"); metadata.addValue( protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - httpVersionString + ",TLS_1_3,TLS_AES_256_GCM_SHA384"); + httpVersionString + ",TLS_1_3"); + metadata.addValue( + protocolMDprefix + ProtocolResponse.CIPHER_SUITES_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index 224deb74e..e529ba6e8 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -252,9 +252,9 @@ void testReplaceHttpVersion() { + "Content-Encoding: gzip\r\n" + "Content-Length: 26\r\n" + "Connection: close"); + metadata.addValue(protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, "h2,TLS_1_3"); metadata.addValue( - protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - "h2,TLS_1_3,TLS_AES_256_GCM_SHA384"); + protocolMDprefix + ProtocolResponse.CIPHER_SUITES_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); @@ -273,8 +273,14 @@ void testReplaceHttpVersion() { statusLine.matches("^HTTP/1\\.[01] .*"), "WARC response record: HTTP status line must start with HTTP/1.1 or HTTP/1.0"); assertTrue( - headersPayload[0].contains("\r\nWARC-Protocol: "), - "WARC response record is expected to include WARC header \"WARC-Protocol\""); + headersPayload[0].contains("\r\nWARC-Protocol: h2\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: h2\""); + assertTrue( + headersPayload[0].contains("\r\nWARC-Protocol: TLS_1_3\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: TLS_1_3\""); + assertTrue( + headersPayload[0].contains("\r\nWARC-Cipher-Suite: "), + "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); assertTrue( headersPayload[0].contains("\r\nWARC-IP-Address: "), "WARC response record is expected to include WARC header \"WARC-IP-Address\""); From b738a51ee217aaace51ad91876b716795e1a729d Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Sat, 12 Sep 2026 15:07:14 +0200 Subject: [PATCH 2/2] WARC writer: WARC-Protocol header to follow WARC field proposals (fixes #1998) Address PR review comments: - `_cipher_suites_` -> `_cipher_suite_` because it holds only a single value, also adapt Javadoc accordingly - do not add Cipher Suite to protocol metadata if it does not exist (http://, no TLS) - normalize OkHttp internal SSL/TLS version strings following the WARC-Protocol field proposal - add unit test for the OkHttp protocol response interceptor (contributed by @rzo1) - WARC writer: trim values of the WARC-Protocol field - updata WARC module unit tests --- .../protocol/ProtocolResponse.java | 6 +- .../protocol/okhttp/HttpProtocol.java | 59 ++++++++++++---- .../okhttp/HttpProtocolHeadersTest.java | 67 +++++++++++++++++++ .../stormcrawler/warc/WARCRecordFormat.java | 6 +- .../stormcrawler/warc/WARCHdfsBoltTest.java | 17 ++--- .../warc/WARCRecordFormatTest.java | 8 +-- 6 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index 31e78cd9c..566ba7418 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -46,10 +46,10 @@ public class ProtocolResponse { public static final String PROTOCOL_VERSIONS_KEY = "_protocol_versions_"; /** - * Key which holds the SSL/TLS cipher suites. For requests sent over http:// the value may be - * null. + * Key which holds the SSL/TLS cipher suite. Not set if the request was sent over an unencrypted + * connection (http://). */ - public static final String CIPHER_SUITES_KEY = "_cipher_suites_"; + public static final String CIPHER_SUITE_KEY = "_cipher_suite_"; /** * Metadata key which holds a boolean value in metadata whether the response content is trimmed diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 8f052563d..abc2ad096 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -65,6 +65,7 @@ import okhttp3.Response; import okhttp3.ResponseBody; import okhttp3.Route; +import okhttp3.TlsVersion; import okhttp3.brotli.Brotli; import okhttp3.zstd.Zstd; import okio.BufferedSource; @@ -820,7 +821,7 @@ public Response intercept(Interceptor.Chain chain) throws IOException { static class HTTPHeadersInterceptor implements Interceptor { - private String getNormalizedProtocolName(Protocol protocol) { + private static String getNormalizedProtocolName(Protocol protocol) { String name = protocol.toString().toUpperCase(Locale.ROOT); if ("H2".equals(name)) { // back-ward compatible protocol version name @@ -829,6 +830,30 @@ private String getNormalizedProtocolName(Protocol protocol) { return name; } + /** + * Maps a {@link TlsVersion} to the protocol identifier used in the WARC-Protocol + * header, see the WARC field proposal. The + * enum names of {@link TlsVersion} (e.g. TLS_1_3) are not part of the + * registered values (e.g. tls/1.3). + */ + private static String getProtocolIdentifier(TlsVersion tlsVersion) { + switch (tlsVersion) { + case SSL_3_0: + return "ssl/3.0"; + case TLS_1_0: + return "tls/1.0"; + case TLS_1_1: + return "tls/1.1"; + case TLS_1_2: + return "tls/1.2"; + case TLS_1_3: + return "tls/1.3"; + default: + return tlsVersion.javaName().toLowerCase(Locale.ROOT); + } + } + @NotNull @Override public Response intercept(Interceptor.Chain chain) throws IOException { @@ -904,27 +929,33 @@ public Response intercept(Interceptor.Chain chain) throws IOException { .toString() .getBytes(StandardCharsets.ISO_8859_1)); + Response.Builder respBuilder = + response.newBuilder() + .header( + ProtocolResponse.REQUEST_HEADERS_KEY, + new String(encodedBytesRequest, StandardCharsets.ISO_8859_1)) + .header( + ProtocolResponse.RESPONSE_HEADERS_KEY, + new String(encodedBytesResponse, StandardCharsets.ISO_8859_1)) + .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress) + .header( + ProtocolResponse.REQUEST_TIME_KEY, + Long.toString(startFetchTime)); + final StringBuilder protocols = new StringBuilder(response.protocol().toString()); String cipherSuite = null; final Handshake handshake = connection.handshake(); if (handshake != null) { - protocols.append(',').append(handshake.tlsVersion()); + protocols.append(',').append(getProtocolIdentifier(handshake.tlsVersion())); cipherSuite = handshake.cipherSuite().toString(); + respBuilder = respBuilder.header(ProtocolResponse.CIPHER_SUITE_KEY, cipherSuite); } + respBuilder = + respBuilder.header( + ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString()); // returns a modified version of the response - return response.newBuilder() - .header( - ProtocolResponse.REQUEST_HEADERS_KEY, - new String(encodedBytesRequest, StandardCharsets.ISO_8859_1)) - .header( - ProtocolResponse.RESPONSE_HEADERS_KEY, - new String(encodedBytesResponse, StandardCharsets.ISO_8859_1)) - .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress) - .header(ProtocolResponse.REQUEST_TIME_KEY, Long.toString(startFetchTime)) - .header(ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString()) - .header(ProtocolResponse.CIPHER_SUITES_KEY, cipherSuite) - .build(); + return respBuilder.build(); } } diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java new file mode 100644 index 000000000..98a01f942 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.okhttp; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.AbstractProtocolTest; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.Test; + +/** Tests the protocol metadata collected by the response interceptor. */ +class HttpProtocolHeadersTest extends AbstractProtocolTest { + + /** + * Over an unencrypted connection there is no handshake, hence no TLS version and no cipher + * suite. The cipher suite header must be skipped entirely: OkHttp's {@code + * Response.Builder.header(...)} does not accept a null value. + */ + @Test + void plainHttpRequestStoresProtocolVersionButNoCipherSuite() throws Exception { + HttpProtocol protocol = new HttpProtocol(); + Config conf = protocolConfig(); + conf.put("http.store.headers", true); + protocol.configure(conf); + + ProtocolResponse response = + protocol.getProtocolOutput("http://localhost:" + HTTP_PORT, Metadata.empty); + + assertEquals(200, response.getStatusCode()); + Metadata metadata = response.getMetadata(); + assertEquals( + "http/1.1", + metadata.getFirstValue(ProtocolResponse.PROTOCOL_VERSIONS_KEY), + "The protocol version is expected to be stored for plain HTTP requests"); + assertNull( + metadata.getFirstValue(ProtocolResponse.CIPHER_SUITE_KEY), + "No cipher suite is expected without a TLS handshake"); + } + + private Config protocolConfig() { + Config conf = new Config(); + conf.put("http.agent.name", "test"); + conf.put("http.agent.version", "1.0"); + conf.put("http.agent.description", "test"); + conf.put("http.agent.url", "http://test.example.com"); + conf.put("http.agent.email", "test@example.com"); + return conf; + } +} diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index 00224169f..a7111f98d 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -616,12 +616,12 @@ public byte[] format(Tuple tuple) { metadata.getFirstValue( ProtocolResponse.PROTOCOL_VERSIONS_KEY, this.protocolMDprefix); if (protocolVersions != null) { - for (String val : StringUtils.split(protocolVersions, ',')) { - buffer.append("WARC-Protocol: ").append(val).append(CRLF); + for (String protocolVersion : StringUtils.split(protocolVersions, ',')) { + buffer.append("WARC-Protocol: ").append(protocolVersion.trim()).append(CRLF); } } final String cipherSuites = - metadata.getFirstValue(ProtocolResponse.CIPHER_SUITES_KEY, this.protocolMDprefix); + metadata.getFirstValue(ProtocolResponse.CIPHER_SUITE_KEY, this.protocolMDprefix); if (cipherSuites != null) { buffer.append("WARC-Cipher-Suite: ").append(cipherSuites).append(CRLF); } diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index dee09592a..fe23d9312 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -105,9 +105,10 @@ void test() throws IOException { assertEquals("response", records.get(2).type()); WarcResponse response = (WarcResponse) records.get(2); assertEquals(MessageVersion.HTTP_1_1, response.http().version()); - assertTrue( - response.headers().first("WARC-Protocol").isPresent(), - "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertEquals( + List.of("HTTP/1.1", "tls/1.3"), + response.headers().all("WARC-Protocol"), + "WARC response record is expected to repeat the WARC header \"WARC-Protocol\" for every protocol layer"); assertTrue( response.headers().first("WARC-IP-Address").isPresent(), "WARC response record is expected to include WARC header \"WARC-IP-Address\""); @@ -131,9 +132,9 @@ void testHttp2() throws IOException { response.headers().first("WARC-Protocol").isPresent(), "WARC response record is expected to include WARC header \"WARC-Protocol\""); assertEquals( - 2, - response.headers().all("WARC-Protocol").size(), - "WARC response record is expected to include WARC header \"WARC-Protocol\""); + List.of("HTTP/2", "tls/1.3"), + response.headers().all("WARC-Protocol"), + "WARC response record is expected to repeat the WARC header \"WARC-Protocol\" for every protocol layer"); assertTrue( response.headers().first("WARC-Cipher-Suite").isPresent(), "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); @@ -248,9 +249,9 @@ private Tuple getPage(String httpVersionString) { + "Connection: close\r\n\r\n"); metadata.addValue( protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - httpVersionString + ",TLS_1_3"); + httpVersionString + ",tls/1.3"); metadata.addValue( - protocolMDprefix + ProtocolResponse.CIPHER_SUITES_KEY, "TLS_AES_256_GCM_SHA384"); + protocolMDprefix + ProtocolResponse.CIPHER_SUITE_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index e529ba6e8..11f9682b1 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -252,9 +252,9 @@ void testReplaceHttpVersion() { + "Content-Encoding: gzip\r\n" + "Content-Length: 26\r\n" + "Connection: close"); - metadata.addValue(protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, "h2,TLS_1_3"); + metadata.addValue(protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, "h2,tls/1.3"); metadata.addValue( - protocolMDprefix + ProtocolResponse.CIPHER_SUITES_KEY, "TLS_AES_256_GCM_SHA384"); + protocolMDprefix + ProtocolResponse.CIPHER_SUITE_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); @@ -276,8 +276,8 @@ void testReplaceHttpVersion() { headersPayload[0].contains("\r\nWARC-Protocol: h2\r\n"), "WARC response record is expected to include a WARC header \"WARC-Protocol: h2\""); assertTrue( - headersPayload[0].contains("\r\nWARC-Protocol: TLS_1_3\r\n"), - "WARC response record is expected to include a WARC header \"WARC-Protocol: TLS_1_3\""); + headersPayload[0].contains("\r\nWARC-Protocol: tls/1.3\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: tls/1.3\""); assertTrue( headersPayload[0].contains("\r\nWARC-Cipher-Suite: "), "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\"");