diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index c465ff000..566ba7418 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -45,6 +45,12 @@ public class ProtocolResponse { */ public static final String PROTOCOL_VERSIONS_KEY = "_protocol_versions_"; + /** + * Key which holds the SSL/TLS cipher suite. Not set if the request was sent over an unencrypted + * connection (http://). + */ + public static final String CIPHER_SUITE_KEY = "_cipher_suite_"; + /** * Metadata key which holds a boolean value in metadata whether the response content is trimmed * or not. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 7717be59c..abc2ad096 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -65,6 +65,7 @@ import okhttp3.Response; import okhttp3.ResponseBody; import okhttp3.Route; +import okhttp3.TlsVersion; import okhttp3.brotli.Brotli; import okhttp3.zstd.Zstd; import okio.BufferedSource; @@ -820,7 +821,7 @@ public Response intercept(Interceptor.Chain chain) throws IOException { static class HTTPHeadersInterceptor implements Interceptor { - private String getNormalizedProtocolName(Protocol protocol) { + private static String getNormalizedProtocolName(Protocol protocol) { String name = protocol.toString().toUpperCase(Locale.ROOT); if ("H2".equals(name)) { // back-ward compatible protocol version name @@ -829,6 +830,30 @@ private String getNormalizedProtocolName(Protocol protocol) { return name; } + /** + * Maps a {@link TlsVersion} to the protocol identifier used in the WARC-Protocol + * header, see the WARC field proposal. The + * enum names of {@link TlsVersion} (e.g. TLS_1_3) are not part of the + * registered values (e.g. tls/1.3). + */ + private static String getProtocolIdentifier(TlsVersion tlsVersion) { + switch (tlsVersion) { + case SSL_3_0: + return "ssl/3.0"; + case TLS_1_0: + return "tls/1.0"; + case TLS_1_1: + return "tls/1.1"; + case TLS_1_2: + return "tls/1.2"; + case TLS_1_3: + return "tls/1.3"; + default: + return tlsVersion.javaName().toLowerCase(Locale.ROOT); + } + } + @NotNull @Override public Response intercept(Interceptor.Chain chain) throws IOException { @@ -904,25 +929,33 @@ public Response intercept(Interceptor.Chain chain) throws IOException { .toString() .getBytes(StandardCharsets.ISO_8859_1)); + Response.Builder respBuilder = + response.newBuilder() + .header( + ProtocolResponse.REQUEST_HEADERS_KEY, + new String(encodedBytesRequest, StandardCharsets.ISO_8859_1)) + .header( + ProtocolResponse.RESPONSE_HEADERS_KEY, + new String(encodedBytesResponse, StandardCharsets.ISO_8859_1)) + .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress) + .header( + ProtocolResponse.REQUEST_TIME_KEY, + Long.toString(startFetchTime)); + final StringBuilder protocols = new StringBuilder(response.protocol().toString()); + String cipherSuite = null; final Handshake handshake = connection.handshake(); if (handshake != null) { - protocols.append(',').append(handshake.tlsVersion()); - protocols.append(',').append(handshake.cipherSuite()); + protocols.append(',').append(getProtocolIdentifier(handshake.tlsVersion())); + cipherSuite = handshake.cipherSuite().toString(); + respBuilder = respBuilder.header(ProtocolResponse.CIPHER_SUITE_KEY, cipherSuite); } + respBuilder = + respBuilder.header( + ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString()); // returns a modified version of the response - return response.newBuilder() - .header( - ProtocolResponse.REQUEST_HEADERS_KEY, - new String(encodedBytesRequest, StandardCharsets.ISO_8859_1)) - .header( - ProtocolResponse.RESPONSE_HEADERS_KEY, - new String(encodedBytesResponse, StandardCharsets.ISO_8859_1)) - .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress) - .header(ProtocolResponse.REQUEST_TIME_KEY, Long.toString(startFetchTime)) - .header(ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString()) - .build(); + return respBuilder.build(); } } diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java new file mode 100644 index 000000000..98a01f942 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.okhttp; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.AbstractProtocolTest; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.Test; + +/** Tests the protocol metadata collected by the response interceptor. */ +class HttpProtocolHeadersTest extends AbstractProtocolTest { + + /** + * Over an unencrypted connection there is no handshake, hence no TLS version and no cipher + * suite. The cipher suite header must be skipped entirely: OkHttp's {@code + * Response.Builder.header(...)} does not accept a null value. + */ + @Test + void plainHttpRequestStoresProtocolVersionButNoCipherSuite() throws Exception { + HttpProtocol protocol = new HttpProtocol(); + Config conf = protocolConfig(); + conf.put("http.store.headers", true); + protocol.configure(conf); + + ProtocolResponse response = + protocol.getProtocolOutput("http://localhost:" + HTTP_PORT, Metadata.empty); + + assertEquals(200, response.getStatusCode()); + Metadata metadata = response.getMetadata(); + assertEquals( + "http/1.1", + metadata.getFirstValue(ProtocolResponse.PROTOCOL_VERSIONS_KEY), + "The protocol version is expected to be stored for plain HTTP requests"); + assertNull( + metadata.getFirstValue(ProtocolResponse.CIPHER_SUITE_KEY), + "No cipher suite is expected without a TLS handshake"); + } + + private Config protocolConfig() { + Config conf = new Config(); + conf.put("http.agent.name", "test"); + conf.put("http.agent.version", "1.0"); + conf.put("http.agent.description", "test"); + conf.put("http.agent.url", "http://test.example.com"); + conf.put("http.agent.email", "test@example.com"); + return conf; + } +} diff --git a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java index 473c0f137..a7111f98d 100644 --- a/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java +++ b/external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java @@ -616,7 +616,14 @@ public byte[] format(Tuple tuple) { metadata.getFirstValue( ProtocolResponse.PROTOCOL_VERSIONS_KEY, this.protocolMDprefix); if (protocolVersions != null) { - buffer.append("WARC-Protocol: ").append(protocolVersions).append(CRLF); + for (String protocolVersion : StringUtils.split(protocolVersions, ',')) { + buffer.append("WARC-Protocol: ").append(protocolVersion.trim()).append(CRLF); + } + } + final String cipherSuites = + metadata.getFirstValue(ProtocolResponse.CIPHER_SUITE_KEY, this.protocolMDprefix); + if (cipherSuites != null) { + buffer.append("WARC-Cipher-Suite: ").append(cipherSuites).append(CRLF); } buffer.append("WARC-Payload-Digest").append(": ").append(payloadDigest).append(CRLF); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java index 2bc8340f2..fe23d9312 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java @@ -105,9 +105,10 @@ void test() throws IOException { assertEquals("response", records.get(2).type()); WarcResponse response = (WarcResponse) records.get(2); assertEquals(MessageVersion.HTTP_1_1, response.http().version()); - assertTrue( - response.headers().first("WARC-Protocol").isPresent(), - "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertEquals( + List.of("HTTP/1.1", "tls/1.3"), + response.headers().all("WARC-Protocol"), + "WARC response record is expected to repeat the WARC header \"WARC-Protocol\" for every protocol layer"); assertTrue( response.headers().first("WARC-IP-Address").isPresent(), "WARC response record is expected to include WARC header \"WARC-IP-Address\""); @@ -130,6 +131,13 @@ void testHttp2() throws IOException { assertTrue( response.headers().first("WARC-Protocol").isPresent(), "WARC response record is expected to include WARC header \"WARC-Protocol\""); + assertEquals( + List.of("HTTP/2", "tls/1.3"), + response.headers().all("WARC-Protocol"), + "WARC response record is expected to repeat the WARC header \"WARC-Protocol\" for every protocol layer"); + assertTrue( + response.headers().first("WARC-Cipher-Suite").isPresent(), + "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); assertTrue( response.headers().first("WARC-IP-Address").isPresent(), "WARC response record is expected to include WARC header \"WARC-IP-Address\""); @@ -241,7 +249,9 @@ private Tuple getPage(String httpVersionString) { + "Connection: close\r\n\r\n"); metadata.addValue( protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - httpVersionString + ",TLS_1_3,TLS_AES_256_GCM_SHA384"); + httpVersionString + ",tls/1.3"); + metadata.addValue( + protocolMDprefix + ProtocolResponse.CIPHER_SUITE_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); diff --git a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java index 224deb74e..11f9682b1 100644 --- a/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java +++ b/external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java @@ -252,9 +252,9 @@ void testReplaceHttpVersion() { + "Content-Encoding: gzip\r\n" + "Content-Length: 26\r\n" + "Connection: close"); + metadata.addValue(protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, "h2,tls/1.3"); metadata.addValue( - protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, - "h2,TLS_1_3,TLS_AES_256_GCM_SHA384"); + protocolMDprefix + ProtocolResponse.CIPHER_SUITE_KEY, "TLS_AES_256_GCM_SHA384"); metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123"); Tuple tuple = mock(Tuple.class); when(tuple.getBinaryByField("content")).thenReturn(content); @@ -273,8 +273,14 @@ void testReplaceHttpVersion() { statusLine.matches("^HTTP/1\\.[01] .*"), "WARC response record: HTTP status line must start with HTTP/1.1 or HTTP/1.0"); assertTrue( - headersPayload[0].contains("\r\nWARC-Protocol: "), - "WARC response record is expected to include WARC header \"WARC-Protocol\""); + headersPayload[0].contains("\r\nWARC-Protocol: h2\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: h2\""); + assertTrue( + headersPayload[0].contains("\r\nWARC-Protocol: tls/1.3\r\n"), + "WARC response record is expected to include a WARC header \"WARC-Protocol: tls/1.3\""); + assertTrue( + headersPayload[0].contains("\r\nWARC-Cipher-Suite: "), + "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\""); assertTrue( headersPayload[0].contains("\r\nWARC-IP-Address: "), "WARC response record is expected to include WARC header \"WARC-IP-Address\"");