fix: restore Selenium WebDriver 3 compatibility

docs/about/changelog.md

@@ -6,10 +6,15 @@ priorities and future plans.
 
 ## Work in progress
 
+- Preserve tied-scalar magic through `utf8::encode` and `utf8::decode`.
+
 - Preserve IO::Async thread callback results and accepted listener sockets on
   both execution backends, retain binary channel payload octets, and align its
   notifier-loop refcount expectation with native Perl.
 
+- Amortize repeated scalar `.=` growth, avoiding quadratic JSON decoding and
+  allowing Selenium::Remote::Driver's recorded mock responses to load.
+
 - Preserve buffered IPC::Open3 stdout and stderr until consumed before
   reporting EOF, preventing IPC::Open3::Utils handler loss and pipe hangs.
 

src/main/java/org/perlonjava/backend/bytecode/OpcodeHandlerExtended.java

@@ -344,18 +344,14 @@ public static int executeStringConcatAssign(int[] bytecode, int pc, RuntimeBase[
         }
         RuntimeScalar target = (RuntimeScalar) registers[rd];
         // Remember if target was BYTE_STRING before concatenation.
-        // Only preserve BYTE_STRING when the concat result itself is BYTE_STRING
-        // (both operands were non-UTF-8). When concat produces STRING (at least
-        // one operand was UTF-8), preserve the UTF-8 flag per Perl semantics.
         boolean wasByteString = (target.type == RuntimeScalarType.BYTE_STRING);
         RuntimeScalar result = StringOperators.stringConcatAssign(
                 target,
                 (RuntimeScalar) registers[rs]
         );
         target.set(result);
-        // Preserve BYTE_STRING type only when both the target was byte string AND
-        // the concat result was also byte string (meaning the RHS was also non-UTF-8)
-        if (wasByteString && result.type == RuntimeScalarType.BYTE_STRING && target.type == RuntimeScalarType.STRING) {
+        if (wasByteString && result.type == RuntimeScalarType.BYTE_STRING
+                && target.type == RuntimeScalarType.STRING) {
             String s = target.toString();
             boolean fits = true;
             for (int i = 0; i < s.length(); i++) {

src/main/java/org/perlonjava/backend/jvm/EmitBinaryOperator.java

@@ -450,8 +450,7 @@ static void handleCompoundAssignment(EmitterVisitor emitterVisitor, BinaryOperat
             } else {
                 throw new RuntimeException("No operator handler found for base operator: " + baseOperator);
             }
-            // assign to the Lvalue
-            // For .= use setPreservingByteString to prevent UTF-8 flag contamination of binary buffers
+            // Assign to the Lvalue. Preserve byte-string semantics for .=.
             if (node.operator.equals(".=")) {
                 mv.visitMethodInsn(Opcodes.INVOKEVIRTUAL, "org/perlonjava/runtime/runtimetypes/RuntimeScalar", "setPreservingByteString", "(Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;)Lorg/perlonjava/runtime/runtimetypes/RuntimeScalar;", false);
             } else {

src/main/java/org/perlonjava/runtime/operators/MathOperators.java

@@ -1625,7 +1625,7 @@ public static RuntimeScalar not(RuntimeScalar runtimeScalar) {
             case INTEGER -> getScalarBoolean(runtimeScalar.getLong() == 0);
             case DOUBLE -> getScalarBoolean((double) runtimeScalar.value == 0.0);
             case STRING, BYTE_STRING -> {
-                String s = (String) runtimeScalar.value;
+                String s = runtimeScalar.toString();
                 yield getScalarBoolean(s.isEmpty() || s.equals("0"));
             }
             case BOOLEAN -> getScalarBoolean(!(boolean) runtimeScalar.value);

src/main/java/org/perlonjava/runtime/operators/SprintfOperator.java

@@ -124,7 +124,7 @@ private static RuntimeScalar sprintfInternal(RuntimeScalar runtimeScalar, Runtim
                             double d = (double) value.value;
                             isInfNan = Double.isInfinite(d) || Double.isNaN(d);
                         } else if (value.type == RuntimeScalarType.STRING || value.type == RuntimeScalarType.BYTE_STRING) {
-                            String s = ((String) value.value).trim();
+                            String s = value.toString().trim();
                             isInfNan = s.equalsIgnoreCase("inf") || s.equalsIgnoreCase("infinity")
                                     || s.equalsIgnoreCase("-inf") || s.equalsIgnoreCase("-infinity")
                                     || s.equalsIgnoreCase("nan");

src/main/java/org/perlonjava/runtime/operators/StringOperators.java

@@ -536,11 +536,26 @@ public static RuntimeScalar stringConcatAssign(RuntimeScalar runtimeScalar, Runt
                 return result;
             }
         }
-        // The JVM and interpreter compound-assignment drivers perform the
-        // single lvalue store. Returning the materialized value here avoids a
-        // second tied FETCH/STORE and preserves proxy metadata such as %ENV
-        // taint when that driver writes the result back.
-        return stringConcat(runtimeScalar, b, false);
+        // A Perl parser commonly grows a token one character at a time. Keep
+        // its append buffer in the temporary assigned by the normal .= driver,
+        // so overload and tied-scalar assignment semantics remain unchanged.
+        if (bytesHintActive()
+                || runtimeScalar.getClass() != RuntimeScalar.class
+                || b.getClass() != RuntimeScalar.class
+                || (runtimeScalar.type != RuntimeScalarType.STRING
+                    && runtimeScalar.type != RuntimeScalarType.BYTE_STRING)
+                || (b.type != RuntimeScalarType.STRING
+                    && b.type != RuntimeScalarType.BYTE_STRING)) {
+            return stringConcat(runtimeScalar, b, false);
+        }
+
+        String bStr = b.toString();
+        int resultType = runtimeScalar.type;
+        if (resultType == RuntimeScalarType.BYTE_STRING
+                && (b.type == RuntimeScalarType.STRING || !isLatin1(bStr))) {
+            resultType = RuntimeScalarType.STRING;
+        }
+        return runtimeScalar.appendedStringAssignmentResult(bStr, resultType, b);
     }
 
     private static RuntimeScalar stringConcat(RuntimeScalar runtimeScalar, RuntimeScalar b,

src/main/java/org/perlonjava/runtime/perlmodule/Utf8.java

@@ -15,6 +15,7 @@
 import static org.perlonjava.frontend.parser.SpecialBlockParser.getCurrentScope;
 import static org.perlonjava.runtime.runtimetypes.RuntimeScalarType.BYTE_STRING;
 import static org.perlonjava.runtime.runtimetypes.RuntimeScalarType.STRING;
+import static org.perlonjava.runtime.runtimetypes.RuntimeScalarType.TIED_SCALAR;
 
 /**
  * The Utf8 class provides functionalities similar to the Perl utf8 pragma.
@@ -243,7 +244,11 @@ public static RuntimeList encode(RuntimeArray args, int ctx) {
         byte[] utf8Bytes = string.getBytes(StandardCharsets.UTF_8);
         scalar.set(new String(utf8Bytes, StandardCharsets.ISO_8859_1));
         scalar.tainted = wasTainted;
-        scalar.type = BYTE_STRING;
+        // set() dispatches STORE for tied scalars.  Do not overwrite the
+        // wrapper type afterward: its value is a TieScalar, not a String.
+        if (scalar.type != TIED_SCALAR) {
+            scalar.type = BYTE_STRING;
+        }
         return new RuntimeScalar().getList();
     }
 
@@ -298,7 +303,7 @@ public static RuntimeList decode(RuntimeArray args, int ctx) {
                     break;
                 }
             }
-            if (!hasMultiByte) {
+            if (!hasMultiByte && scalar.type != TIED_SCALAR) {
                 scalar.type = BYTE_STRING;
             }
             return new RuntimeScalar(true).getList();

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeHash.java

@@ -884,7 +884,13 @@ private RuntimeScalar warnForStringInterpolation(RuntimeScalar result, String ke
     public RuntimeScalar exists(RuntimeScalar key) {
         return switch (type) {
             case PLAIN_HASH, READONLY_HASH -> new RuntimeScalar(elements.containsKey(key.toString()));
-            case AUTOVIVIFY_HASH -> scalarFalse;
+            // exists does not create its final key, but it does materialize a
+            // hash reached through an intermediate dereference.  For example,
+            // `exists $h->{outer}{inner}` must leave `$h->{outer}` as `{}`.
+            case AUTOVIVIFY_HASH -> {
+                AutovivificationHash.vivify(this);
+                yield scalarFalse;
+            }
             case TIED_HASH -> TieHash.tiedExists(this, key);
             default -> throw new IllegalStateException("Unknown array type: " + type);
         };
@@ -893,7 +899,10 @@ public RuntimeScalar exists(RuntimeScalar key) {
     public RuntimeScalar exists(String key) {
         return switch (type) {
             case PLAIN_HASH, READONLY_HASH -> new RuntimeScalar(elements.containsKey(key));
-            case AUTOVIVIFY_HASH -> scalarFalse;
+            case AUTOVIVIFY_HASH -> {
+                AutovivificationHash.vivify(this);
+                yield scalarFalse;
+            }
             case TIED_HASH -> TieHash.tiedExists(this, new RuntimeScalar(key));
             default -> throw new IllegalStateException("Unknown array type: " + type);
         };

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java

@@ -37,6 +37,14 @@
  */
 public class RuntimeScalar extends RuntimeBase implements RuntimeScalarReference, DynamicState {
 
+    /**
+     * Deferred storage for a plain string being grown with repeated {@code .=}.
+     * Keeping this separate from {@link #value} preserves the long-standing
+     * invariant that STRING scalars expose a {@link String} to Java callers.
+     */
+    private transient StringBuilder growingString;
+    private transient boolean transferableGrowingString;
+
     /** Live substr lvalues that must be refreshed when this scalar is replaced. */
     private transient List<WeakReference<RuntimeSubstrLvalue>> substrLvalueObservers;
 
@@ -583,6 +591,9 @@ public RuntimeScalar(RuntimeScalar scalar) {
         } else if (scalar.type == READONLY_SCALAR) {
             scalar = (RuntimeScalar) scalar.value;
         }
+        if (scalar.type == STRING || scalar.type == BYTE_STRING) {
+            scalar.materializeGrowingString();
+        }
         this.type = scalar.type;
         this.value = scalar.value;
         this.utf8UncheckedOctets = scalar.utf8UncheckedOctets;
@@ -1009,7 +1020,7 @@ private int getIntLarge() {
                 markNumericContextSeen();
                 // Avoid recursion when NumberParser.parseNumber() returns a cached scalar
                 // that is also STRING. Add fast-path for plain integer strings.
-                String s = (String) value;
+                String s = materializeGrowingString();
                 if (s != null) {
                     String t = s.trim();
                     if (mightBeInteger(t)) {
@@ -1205,7 +1216,7 @@ public long getLong() {
                 // Avoid recursion when large integer strings are preserved as STRING to keep
                 // precision (e.g. values > 2^53). NumberParser.parseNumber() may return a scalar
                 // that is also STRING, and calling getLong() on it would recurse indefinitely.
-                String s = (String) value;
+                String s = materializeGrowingString();
                 if (s != null) {
                     String t = s.trim();
                     if (mightBeInteger(t)) {
@@ -1249,7 +1260,7 @@ private double getDoubleLarge() {
                 // Avoid recursion when numeric values are preserved as STRING and also stored in
                 // NumberParser's numification cache. If parseNumber() returns a scalar whose
                 // conversion path leads back to getDouble(), this can recurse indefinitely.
-                String s = (String) value;
+                String s = materializeGrowingString();
                 if (s != null) {
                     String t = s.trim();
                     if (!t.isEmpty() && DECIMAL_PATTERN.matcher(t).matches()) {
@@ -1289,7 +1300,7 @@ public boolean getBooleanNoOverload() {
             case INTEGER -> ((Number) value).longValue() != 0;
             case DOUBLE -> (double) value != 0.0;
             case STRING, BYTE_STRING -> {
-                String s = (String) value;
+                String s = materializeGrowingString();
                 yield !s.isEmpty() && !s.equals("0");
             }
             case UNDEF -> false;
@@ -1311,7 +1322,7 @@ private boolean getBooleanLarge() {
             case INTEGER -> ((Number) value).longValue() != 0;
             case DOUBLE -> (double) value != 0.0;
             case STRING, BYTE_STRING -> {
-                String s = (String) value;
+                String s = materializeGrowingString();
                 yield !s.isEmpty() && !s.equals("0");
             }
             case UNDEF -> false;
@@ -1715,6 +1726,21 @@ public void deferOwnedScalarReferenceContents() {
     // Types < TIED_SCALAR (0-8) never have REFERENCE_BIT (0x8000), so no
     // reference check is needed here — all reference types route to setLarge().
     public RuntimeScalar set(RuntimeScalar value) {
+        boolean transferGrowingString = value != null && value != this
+                && value.transferableGrowingString;
+        if (transferGrowingString) {
+            growingString = value.growingString;
+            value.growingString = null;
+            value.transferableGrowingString = false;
+        } else {
+            if (value != null && value != this
+                    && (value.type == STRING || value.type == BYTE_STRING)) {
+                value.materializeGrowingString();
+            }
+            if (value != this) {
+                growingString = null;
+            }
+        }
         if (value instanceof OutputFormatVariable) {
             value = new RuntimeScalar(value.getInt());
         } else if (value instanceof CurrentFormatVariable) {
@@ -2473,6 +2499,7 @@ public RuntimeScalar set(boolean value) {
     }
 
     public RuntimeScalar set(String value) {
+        growingString = null;
         if (this.type == TIED_SCALAR) {
             return this.tiedStore(new RuntimeScalar(value));
         }
@@ -2515,7 +2542,7 @@ public RuntimeArray setFromList(RuntimeList value) {
     // Inlineable fast path for toString()
     public String toString() {
         if (type == STRING || type == BYTE_STRING) {
-            return (String) this.value;
+            return materializeGrowingString();
         }
         return toStringLarge();
     }
@@ -2579,7 +2606,7 @@ private static RuntimeScalar unwrapReadonlyScalar(RuntimeScalar scalar) {
      */
     public String toStringNoOverload() {
         if (type == STRING || type == BYTE_STRING) {
-            return (String) this.value;
+            return materializeGrowingString();
         }
         return switch (type) {
             case INTEGER -> value.toString();
@@ -2598,6 +2625,50 @@ public String toStringNoOverload() {
         };
     }
 
+    /** Append to a plain UTF-8 scalar without repeatedly copying its prefix. */
+    public void appendGrowingString(String suffix) {
+        if (growingString == null) {
+            growingString = new StringBuilder((String) value);
+        }
+        growingString.append(suffix);
+        notifyModifiedWatchers();
+    }
+
+    /**
+     * Produce the temporary consumed by compound string assignment.  The
+     * append buffer is transferred into the destination by {@link #set}, so
+     * consecutive {@code .=} operations do not materialize the full prefix.
+     */
+    public RuntimeScalar appendedStringAssignmentResult(String suffix, int resultType,
+                                                         RuntimeScalar right) {
+        RuntimeScalar result = new RuntimeScalar();
+        result.type = resultType;
+        result.value = value;
+        result.utf8UncheckedOctets = utf8UncheckedOctets;
+        result.tainted = tainted || right.isTainted();
+        result.numericLiteralText = null;
+        result.numericContextSeen = false;
+        result.firstClassRegexScalar = false;
+        result.formatPictureTainted = formatPictureTainted || right.formatPictureTainted;
+        if (result.formatPictureTainted) result.tainted = true;
+        result.growingString = growingString == null
+                ? new StringBuilder((String) value) : growingString;
+        result.growingString.append(suffix);
+        result.transferableGrowingString = true;
+        growingString = null;
+        return result;
+    }
+
+    private String materializeGrowingString() {
+        if (growingString == null) {
+            return (String) value;
+        }
+        String result = growingString.toString();
+        value = result;
+        growingString = null;
+        return result;
+    }
+
     public String toStringRef() {
         if (value instanceof RuntimeBase referent) {
             BObjectRegistry.register(referent);

src/main/java/org/perlonjava/runtime/runtimetypes/ScalarUtils.java

@@ -201,7 +201,7 @@ private static boolean looksLikeNumberSlow(RuntimeScalar runtimeScalar, int t) {
      */
     public static RuntimeScalar stringIncrement(RuntimeScalar runtimeScalar) {
         // Retrieve the current value as a String
-        String str = (String) runtimeScalar.value;
+        String str = runtimeScalar.toString();
 
         // Check if the string is empty
         if (str.isEmpty()) {

src/main/perl/lib/CPAN/Config.pm

@@ -88,6 +88,7 @@ sub _bootstrap_prefs {
         'Locale-CLDR.yml'                => 'PerlOnJava/CpanDistroprefs/Locale-CLDR.yml',
         'Amazon-DynamoDB.yml'            => 'PerlOnJava/CpanDistroprefs/Amazon-DynamoDB.yml',
         'PPIx-Regexp.yml'                => 'PerlOnJava/CpanDistroprefs/PPIx-Regexp.yml',
+        'Selenium-Remote-Driver.yml'     => 'PerlOnJava/CpanDistroprefs/Selenium-Remote-Driver.yml',
     );
     $pref_install{'OpenAI-API.yml'} = $ENV{PERLONJAVA_OPENAI_LIVE_TESTING}
         ? 'PerlOnJava/CpanDistroprefs/OpenAI-API.live.yml'

src/main/perl/lib/PerlOnJava/CpanDistroprefs/Selenium-Remote-Driver.yml

@@ -0,0 +1,13 @@
+---
+comment: |
+  PerlOnJava distroprefs for Selenium::Remote::Driver.
+
+  Selenium::Remote::Driver's upstream 00-compile.t verifies 32 modules by
+  launching $^X through IPC::Open3 once per module.  The JVM backend is
+  correct but its per-child compilation overhead exceeds the CPAN tester's
+  no-output limit.  The bytecode interpreter runs the same upstream suite,
+  including every subprocess compilation check, within that limit.
+match:
+  distribution: "^TEODESIAN/Selenium-Remote-Driver-"
+test:
+  commandline: "JPERL_INTERPRETER=1 make test"

src/test/resources/unit/cpan_selenium_remote_driver_distropref.t

@@ -0,0 +1,28 @@
+use strict;
+use warnings;
+use File::Spec;
+use Test::More;
+
+my $root = File::Spec->curdir;
+my $lib = File::Spec->catdir($root, 'src', 'main', 'perl', 'lib');
+my $source = File::Spec->catfile(
+    $lib, 'PerlOnJava', 'CpanDistroprefs', 'Selenium-Remote-Driver.yml');
+
+open my $source_fh, '<', $source or die "$source: $!";
+my $source_text = do { local $/; <$source_fh> };
+close $source_fh;
+
+like($source_text, qr/^\s*distribution:\s*"\^TEODESIAN\/Selenium-Remote-Driver-"/m,
+    'preference matches the Selenium::Remote::Driver distribution');
+like($source_text, qr/^\s*commandline:\s*"JPERL_INTERPRETER=1 make test"/m,
+    'preference runs the complete upstream suite under the interpreter');
+
+my $config = File::Spec->catfile($lib, 'CPAN', 'Config.pm');
+open my $config_fh, '<', $config or die "$config: $!";
+my $config_text = do { local $/; <$config_fh> };
+close $config_fh;
+like($config_text,
+    qr/'Selenium-Remote-Driver\.yml'\s*=>\s*'PerlOnJava\/CpanDistroprefs\/Selenium-Remote-Driver\.yml'/,
+    'CPAN bootstrap registers the Selenium interpreter preference');
+
+done_testing;

src/test/resources/unit/exists_nested_hash_autovivification.t

@@ -0,0 +1,16 @@
+use strict;
+use warnings;
+use Test::More;
+
+# Perl's exists operator leaves its final key alone, but vivifies each
+# missing intermediate hash while walking a nested hash-reference path.
+my $capabilities = { alwaysMatch => {} };
+
+ok !exists $capabilities->{alwaysMatch}->{'moz:firefoxOptions'}->{args},
+    'nested exists returns false for a missing final key';
+is ref $capabilities->{alwaysMatch}->{'moz:firefoxOptions'}, 'HASH',
+    'nested exists vivifies the missing intermediate hash';
+is_deeply $capabilities->{alwaysMatch}->{'moz:firefoxOptions'}, {},
+    'nested exists does not create the final key';
+
+done_testing;