#1998 WARC writer: WARC-Protocol header to follow WARC field proposals

core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java

@@ -45,6 +45,12 @@ public class ProtocolResponse {
      */
     public static final String PROTOCOL_VERSIONS_KEY = "_protocol_versions_";
 
+    /**
+     * Key which holds the SSL/TLS cipher suite. Not set if the request was sent over an unencrypted
+     * connection (http://).
+     */
+    public static final String CIPHER_SUITE_KEY = "_cipher_suite_";
+
     /**
      * Metadata key which holds a boolean value in metadata whether the response content is trimmed
      * or not.

core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java

@@ -65,6 +65,7 @@
 import okhttp3.Response;
 import okhttp3.ResponseBody;
 import okhttp3.Route;
+import okhttp3.TlsVersion;
 import okhttp3.brotli.Brotli;
 import okhttp3.zstd.Zstd;
 import okio.BufferedSource;
@@ -820,7 +821,7 @@ public Response intercept(Interceptor.Chain chain) throws IOException {
 
     static class HTTPHeadersInterceptor implements Interceptor {
 
-        private String getNormalizedProtocolName(Protocol protocol) {
+        private static String getNormalizedProtocolName(Protocol protocol) {
             String name = protocol.toString().toUpperCase(Locale.ROOT);
             if ("H2".equals(name)) {
                 // back-ward compatible protocol version name
@@ -829,6 +830,30 @@ private String getNormalizedProtocolName(Protocol protocol) {
             return name;
         }
 
+        /**
+         * Maps a {@link TlsVersion} to the protocol identifier used in the <code>WARC-Protocol
+         * </code> header, see the <a
+         * href="https://github.com/iipc/warc-specifications/issues/42">WARC field proposal</a>. The
+         * enum names of {@link TlsVersion} (e.g. <code>TLS_1_3</code>) are not part of the
+         * registered values (e.g. <code>tls/1.3</code>).
+         */
+        private static String getProtocolIdentifier(TlsVersion tlsVersion) {
+            switch (tlsVersion) {
+                case SSL_3_0:
+                    return "ssl/3.0";
+                case TLS_1_0:
+                    return "tls/1.0";
+                case TLS_1_1:
+                    return "tls/1.1";
+                case TLS_1_2:
+                    return "tls/1.2";
+                case TLS_1_3:
+                    return "tls/1.3";
+                default:
+                    return tlsVersion.javaName().toLowerCase(Locale.ROOT);
+            }
+        }
+
         @NotNull
         @Override
         public Response intercept(Interceptor.Chain chain) throws IOException {
@@ -904,25 +929,33 @@ public Response intercept(Interceptor.Chain chain) throws IOException {
                                             .toString()
                                             .getBytes(StandardCharsets.ISO_8859_1));
 
+            Response.Builder respBuilder =
+                    response.newBuilder()
+                            .header(
+                                    ProtocolResponse.REQUEST_HEADERS_KEY,
+                                    new String(encodedBytesRequest, StandardCharsets.ISO_8859_1))
+                            .header(
+                                    ProtocolResponse.RESPONSE_HEADERS_KEY,
+                                    new String(encodedBytesResponse, StandardCharsets.ISO_8859_1))
+                            .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress)
+                            .header(
+                                    ProtocolResponse.REQUEST_TIME_KEY,
+                                    Long.toString(startFetchTime));
+
             final StringBuilder protocols = new StringBuilder(response.protocol().toString());
+            String cipherSuite = null;
             final Handshake handshake = connection.handshake();
             if (handshake != null) {
-                protocols.append(',').append(handshake.tlsVersion());
-                protocols.append(',').append(handshake.cipherSuite());
+                protocols.append(',').append(getProtocolIdentifier(handshake.tlsVersion()));
+                cipherSuite = handshake.cipherSuite().toString();
+                respBuilder = respBuilder.header(ProtocolResponse.CIPHER_SUITE_KEY, cipherSuite);
             }
+            respBuilder =
+                    respBuilder.header(
+                            ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString());
 
             // returns a modified version of the response
-            return response.newBuilder()
-                    .header(
-                            ProtocolResponse.REQUEST_HEADERS_KEY,
-                            new String(encodedBytesRequest, StandardCharsets.ISO_8859_1))
-                    .header(
-                            ProtocolResponse.RESPONSE_HEADERS_KEY,
-                            new String(encodedBytesResponse, StandardCharsets.ISO_8859_1))
-                    .header(ProtocolResponse.RESPONSE_IP_KEY, ipAddress)
-                    .header(ProtocolResponse.REQUEST_TIME_KEY, Long.toString(startFetchTime))
-                    .header(ProtocolResponse.PROTOCOL_VERSIONS_KEY, protocols.toString())
-                    .build();
+            return respBuilder.build();
         }
     }
 

core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolHeadersTest.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol.okhttp;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.apache.storm.Config;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.protocol.AbstractProtocolTest;
+import org.apache.stormcrawler.protocol.ProtocolResponse;
+import org.junit.jupiter.api.Test;
+
+/** Tests the protocol metadata collected by the response interceptor. */
+class HttpProtocolHeadersTest extends AbstractProtocolTest {
+
+    /**
+     * Over an unencrypted connection there is no handshake, hence no TLS version and no cipher
+     * suite. The cipher suite header must be skipped entirely: OkHttp's {@code
+     * Response.Builder.header(...)} does not accept a null value.
+     */
+    @Test
+    void plainHttpRequestStoresProtocolVersionButNoCipherSuite() throws Exception {
+        HttpProtocol protocol = new HttpProtocol();
+        Config conf = protocolConfig();
+        conf.put("http.store.headers", true);
+        protocol.configure(conf);
+
+        ProtocolResponse response =
+                protocol.getProtocolOutput("http://localhost:" + HTTP_PORT, Metadata.empty);
+
+        assertEquals(200, response.getStatusCode());
+        Metadata metadata = response.getMetadata();
+        assertEquals(
+                "http/1.1",
+                metadata.getFirstValue(ProtocolResponse.PROTOCOL_VERSIONS_KEY),
+                "The protocol version is expected to be stored for plain HTTP requests");
+        assertNull(
+                metadata.getFirstValue(ProtocolResponse.CIPHER_SUITE_KEY),
+                "No cipher suite is expected without a TLS handshake");
+    }
+
+    private Config protocolConfig() {
+        Config conf = new Config();
+        conf.put("http.agent.name", "test");
+        conf.put("http.agent.version", "1.0");
+        conf.put("http.agent.description", "test");
+        conf.put("http.agent.url", "http://test.example.com");
+        conf.put("http.agent.email", "test@example.com");
+        return conf;
+    }
+}

external/warc/src/main/java/org/apache/stormcrawler/warc/WARCRecordFormat.java

@@ -616,7 +616,14 @@ public byte[] format(Tuple tuple) {
                 metadata.getFirstValue(
                         ProtocolResponse.PROTOCOL_VERSIONS_KEY, this.protocolMDprefix);
         if (protocolVersions != null) {
-            buffer.append("WARC-Protocol: ").append(protocolVersions).append(CRLF);
+            for (String protocolVersion : StringUtils.split(protocolVersions, ',')) {
+                buffer.append("WARC-Protocol: ").append(protocolVersion.trim()).append(CRLF);
+            }
+        }
+        final String cipherSuites =
+                metadata.getFirstValue(ProtocolResponse.CIPHER_SUITE_KEY, this.protocolMDprefix);
+        if (cipherSuites != null) {
+            buffer.append("WARC-Cipher-Suite: ").append(cipherSuites).append(CRLF);
         }
 
         buffer.append("WARC-Payload-Digest").append(": ").append(payloadDigest).append(CRLF);

external/warc/src/test/java/org/apache/stormcrawler/warc/WARCHdfsBoltTest.java

@@ -105,9 +105,10 @@ void test() throws IOException {
         assertEquals("response", records.get(2).type());
         WarcResponse response = (WarcResponse) records.get(2);
         assertEquals(MessageVersion.HTTP_1_1, response.http().version());
-        assertTrue(
-                response.headers().first("WARC-Protocol").isPresent(),
-                "WARC response record is expected to include WARC header \"WARC-Protocol\"");
+        assertEquals(
+                List.of("HTTP/1.1", "tls/1.3"),
+                response.headers().all("WARC-Protocol"),
+                "WARC response record is expected to repeat the WARC header \"WARC-Protocol\" for every protocol layer");
         assertTrue(
                 response.headers().first("WARC-IP-Address").isPresent(),
                 "WARC response record is expected to include WARC header \"WARC-IP-Address\"");
@@ -130,6 +131,13 @@ void testHttp2() throws IOException {
         assertTrue(
                 response.headers().first("WARC-Protocol").isPresent(),
                 "WARC response record is expected to include WARC header \"WARC-Protocol\"");
+        assertEquals(
+                List.of("HTTP/2", "tls/1.3"),
+                response.headers().all("WARC-Protocol"),
+                "WARC response record is expected to repeat the WARC header \"WARC-Protocol\" for every protocol layer");
+        assertTrue(
+                response.headers().first("WARC-Cipher-Suite").isPresent(),
+                "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\"");
         assertTrue(
                 response.headers().first("WARC-IP-Address").isPresent(),
                 "WARC response record is expected to include WARC header \"WARC-IP-Address\"");
@@ -241,7 +249,9 @@ private Tuple getPage(String httpVersionString) {
                         + "Connection: close\r\n\r\n");
         metadata.addValue(
                 protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY,
-                httpVersionString + ",TLS_1_3,TLS_AES_256_GCM_SHA384");
+                httpVersionString + ",tls/1.3");
+        metadata.addValue(
+                protocolMDprefix + ProtocolResponse.CIPHER_SUITE_KEY, "TLS_AES_256_GCM_SHA384");
         metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123");
         Tuple tuple = mock(Tuple.class);
         when(tuple.getBinaryByField("content")).thenReturn(content);

external/warc/src/test/java/org/apache/stormcrawler/warc/WARCRecordFormatTest.java

@@ -252,9 +252,9 @@ void testReplaceHttpVersion() {
                         + "Content-Encoding: gzip\r\n"
                         + "Content-Length: 26\r\n"
                         + "Connection: close");
+        metadata.addValue(protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY, "h2,tls/1.3");
         metadata.addValue(
-                protocolMDprefix + ProtocolResponse.PROTOCOL_VERSIONS_KEY,
-                "h2,TLS_1_3,TLS_AES_256_GCM_SHA384");
+                protocolMDprefix + ProtocolResponse.CIPHER_SUITE_KEY, "TLS_AES_256_GCM_SHA384");
         metadata.addValue(protocolMDprefix + ProtocolResponse.RESPONSE_IP_KEY, "123.123.123.123");
         Tuple tuple = mock(Tuple.class);
         when(tuple.getBinaryByField("content")).thenReturn(content);
@@ -273,8 +273,14 @@ void testReplaceHttpVersion() {
                 statusLine.matches("^HTTP/1\\.[01] .*"),
                 "WARC response record: HTTP status line must start with HTTP/1.1 or HTTP/1.0");
         assertTrue(
-                headersPayload[0].contains("\r\nWARC-Protocol: "),
-                "WARC response record is expected to include WARC header \"WARC-Protocol\"");
+                headersPayload[0].contains("\r\nWARC-Protocol: h2\r\n"),
+                "WARC response record is expected to include a WARC header \"WARC-Protocol: h2\"");
+        assertTrue(
+                headersPayload[0].contains("\r\nWARC-Protocol: tls/1.3\r\n"),
+                "WARC response record is expected to include a WARC header \"WARC-Protocol: tls/1.3\"");
+        assertTrue(
+                headersPayload[0].contains("\r\nWARC-Cipher-Suite: "),
+                "WARC response record is expected to include WARC header \"WARC-Cipher-Suite\"");
         assertTrue(
                 headersPayload[0].contains("\r\nWARC-IP-Address: "),
                 "WARC response record is expected to include WARC header \"WARC-IP-Address\"");