Skip to content

fix: preserve UTF-8 HTML parser octets #1281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fglock merged 2 commits into from
Sep 7, 2026
Merged

fix: preserve UTF-8 HTML parser octets #1281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
93da168
fix: preserve UTF-8 HTML parser octets
fglock Sep 7, 2026
7044ab8
Merge branch 'master' into fix/1245-html-formatter-thai-utf8
fglock Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/about/changelog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ priorities and future plans.

## Work in progress

- Preserve UTF-8 HTML octets through HTML::Parser and no-op entity decoding,
restoring complete Thai text in HTML::Formatter output.
- Preserve caller-owned array and hash lifetimes across generated coercion
callbacks, so `Types::Const` freezes cloned values without modifying the
original reference.
Expand Down
88 changes: 63 additions & 25 deletions src/main/java/org/perlonjava/runtime/perlmodule/HTMLParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ public static RuntimeList _alloc_pstate(RuntimeArray args, int ctx) {
pstate.put("_eof", scalarFalse);
pstate.put("_started", scalarFalse);
pstate.put("_buf", new RuntimeScalar(""));
// Tracks the representation of parser input while parseHtml() turns
// slices into callback values. File input is normally an octet
// string; turning its slices into STRING corrupts later regexes that
// correctly interpret STRING as Unicode.
pstate.put("_input_is_byte_string", scalarFalse);
pstate.put("_literal_mode", new RuntimeScalar(""));
pstate.put("_pending_end_tag", new RuntimeScalar(""));
pstate.put("_bool_attr_val", scalarUndef);
Expand Down Expand Up @@ -174,6 +179,10 @@ public static RuntimeList parse(RuntimeArray args, int ctx) {
fireEvent(self, selfHash, pstate, "start_document");
}
String chunkStr = chunk.toString();
RuntimeScalar buffered = pstate.get("_buf");
boolean inputIsByteString = !buffered.toString().isEmpty()
? buffered.type == RuntimeScalarType.BYTE_STRING
: chunk.type == RuntimeScalarType.BYTE_STRING;

// When utf8_mode is set and the input is a BYTE_STRING, try to
// decode UTF-8 byte sequences to characters. If decoding fails
Expand All @@ -190,13 +199,15 @@ public static RuntimeList parse(RuntimeArray args, int ctx) {
.onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPORT);
try {
chunkStr = decoder.decode(java.nio.ByteBuffer.wrap(bytes)).toString();
inputIsByteString = false;
} catch (java.nio.charset.CharacterCodingException e) {
// Not valid UTF-8; keep original string (Latin-1 identity mapping)
}
}

String html = pstate.get("_buf").toString() + chunkStr;
pstate.put("_buf", new RuntimeScalar(""));
pstate.put("_input_is_byte_string", inputIsByteString ? scalarTrue : scalarFalse);
String html = buffered.toString() + chunkStr;
pstate.put("_buf", parsedScalar(pstate, ""));
parseHtml(self, selfHash, pstate, html);
}
}
Expand Down Expand Up @@ -346,15 +357,22 @@ public static RuntimeList decode_entities(RuntimeArray args, int ctx) {
for (int i = 0; i < items; i++) {
RuntimeScalar sv = args.get(i);
String decoded = decodeEntitiesString(sv.toString(), entity2char, false);
sv.set(decoded);
// A no-op entity decode must not change a byte string into a
// Unicode string. HTML::TreeBuilder performs this operation
// on every text node before handing it to HTML::Formatter.
if (!decoded.equals(sv.toString())) {
sv.set(decoded);
}
}
return new RuntimeList();
} else {
// Scalar/list context: return decoded copies
RuntimeList result = new RuntimeList();
for (int i = 0; i < items; i++) {
String decoded = decodeEntitiesString(args.get(i).toString(), entity2char, false);
result.add(new RuntimeScalar(decoded));
RuntimeScalar source = args.get(i);
String decoded = decodeEntitiesString(source.toString(), entity2char, false);
result.add(decoded.equals(source.toString())
? new RuntimeScalar(source) : new RuntimeScalar(decoded));
}
return result;
}
Expand Down Expand Up @@ -388,7 +406,9 @@ public static RuntimeList _decode_entities(RuntimeArray args, int ctx) {
}

String decoded = decodeEntitiesString(stringSv.toString(), entityHash, expandPrefix);
stringSv.set(decoded);
if (!decoded.equals(stringSv.toString())) {
stringSv.set(decoded);
}

return new RuntimeList();
}
Expand Down Expand Up @@ -431,6 +451,19 @@ private static RuntimeHash getPstate(RuntimeHash selfHash) {
return ref.hashDeref();
}

/**
* Creates a scalar for a slice of the current parser input without
* changing its Perl byte/Unicode representation.
*/
private static RuntimeScalar parsedScalar(RuntimeHash pstate, String value) {
RuntimeScalar scalar = new RuntimeScalar(value);
RuntimeScalar byteInput = pstate.get("_input_is_byte_string");
if (byteInput != null && byteInput.getBoolean()) {
scalar.type = RuntimeScalarType.BYTE_STRING;
}
return scalar;
}

/**
* Fire a parser event by calling the registered handler.
* Supports three callback types:
Expand Down Expand Up @@ -641,7 +674,12 @@ private static RuntimeArray buildEventDataFromArgspec(String argspec, String eve
String rawText = eventArgs[0].toString();
RuntimeHash entity2char = GlobalVariable.getGlobalHash("HTML::Entities::entity2char");
String decoded = decodeEntitiesString(rawText, entity2char, false);
RuntimeArray.push(result, new RuntimeScalar(decoded));
// Retain the original scalar when no entity changed the
// text. In particular, an HTML document read as UTF-8
// octets must not acquire Perl's UTF-8 flag merely by
// passing through a dtext handler.
RuntimeArray.push(result, decoded.equals(rawText)
? eventArgs[0] : new RuntimeScalar(decoded));
} else if (eventArgs.length > 0) {
RuntimeArray.push(result, eventArgs[eventArgs.length - 1]);
} else {
Expand Down Expand Up @@ -849,19 +887,19 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
if (!literalMode.isEmpty()) {
int endIdx = findLiteralEnd(pstate, html, literalMode, 0);
if (endIdx < 0) {
pstate.put("_buf", new RuntimeScalar(html));
pstate.put("_buf", parsedScalar(pstate, html));
return;
}

int endTagEnd = html.indexOf('>', endIdx);
if (endTagEnd < 0) {
pstate.put("_buf", new RuntimeScalar(html));
pstate.put("_buf", parsedScalar(pstate, html));
return;
}

if (endIdx > 0) {
fireEvent(self, selfHash, pstate, "text",
new RuntimeScalar(html.substring(0, endIdx)));
parsedScalar(pstate, html.substring(0, endIdx)));
}
endTagEnd++;
fireEvent(self, selfHash, pstate, "end",
Expand All @@ -877,15 +915,15 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
// Flush pending text
if (i > textStart) {
fireEvent(self, selfHash, pstate, "text",
new RuntimeScalar(html.substring(textStart, i)));
parsedScalar(pstate, html.substring(textStart, i)));
}

int tagStart = i;
i++; // skip '<'

// If we're at end of input, buffer the '<' for next parse() call
if (i >= len) {
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}

Expand All @@ -900,7 +938,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
while (i < len && html.charAt(i) != '>') i++;
if (i >= len) {
// Incomplete end tag - buffer for next parse() call
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}
if (i < len) i++; // skip '>'
Expand Down Expand Up @@ -931,7 +969,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH

if (endIdx < 0) {
// Unterminated marked section - buffer
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}

Expand All @@ -943,7 +981,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
// Emit as text with is_cdata=true
pstate.put("_in_cdata", scalarTrue);
fireEvent(self, selfHash, pstate, "text",
new RuntimeScalar(content));
parsedScalar(pstate, content));
pstate.put("_in_cdata", scalarFalse);
break;
case "IGNORE":
Expand All @@ -968,7 +1006,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
fireEvent(self, selfHash, pstate, "declaration",
new RuntimeScalar(decl));
} else {
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}
}
Expand All @@ -981,7 +1019,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
fireEvent(self, selfHash, pstate, "comment",
new RuntimeScalar(comment));
} else {
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}
}
Expand All @@ -997,7 +1035,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
new RuntimeScalar(comment));
} else {
// Unterminated comment - buffer it
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}
} else {
Expand All @@ -1023,7 +1061,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
fireEvent(self, selfHash, pstate, "process",
new RuntimeScalar(pi));
} else {
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}
textStart = i;
Expand Down Expand Up @@ -1101,7 +1139,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
i++;
} else if (i >= len) {
// Incomplete tag - buffer for next parse() call
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
pstate.put("_buf", parsedScalar(pstate, html.substring(tagStart)));
return;
}

Expand Down Expand Up @@ -1144,14 +1182,14 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
i = endTagEnd;
} else {
// Incomplete end tag - buffer for next parse()
pstate.put("_buf", new RuntimeScalar(html.substring(endIdx)));
pstate.put("_buf", parsedScalar(pstate, html.substring(endIdx)));
return;
}
} else {
// The start event has already fired. Preserve literal mode and only
// buffer its content so a later chunk cannot emit the start twice.
pstate.put("_literal_mode", new RuntimeScalar(tagName));
pstate.put("_buf", new RuntimeScalar(html.substring(i)));
pstate.put("_buf", parsedScalar(pstate, html.substring(i)));
return;
}
}
Expand All @@ -1166,7 +1204,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
// Flush remaining text
if (textStart < len) {
fireEvent(self, selfHash, pstate, "text",
new RuntimeScalar(html.substring(textStart)));
parsedScalar(pstate, html.substring(textStart)));
}
}

Expand All @@ -1182,7 +1220,7 @@ private static void flushBufferedAtEof(RuntimeScalar self, RuntimeHash selfHash,
while (true) {
String remaining = pstate.get("_buf").toString();
String literalMode = pstate.get("_literal_mode").toString();
pstate.put("_buf", new RuntimeScalar(""));
pstate.put("_buf", parsedScalar(pstate, ""));
pstate.put("_literal_mode", new RuntimeScalar(""));

if (literalMode.isEmpty()) {
Expand All @@ -1205,7 +1243,7 @@ private static void flushBufferedAtEof(RuntimeScalar self, RuntimeHash selfHash,
// HTML::Parser treats unterminated textarea/xmp/iframe/plaintext
// literal content as text at EOF. Keep the existing listing mode
// on that same non-markup path.
fireEvent(self, selfHash, pstate, "text", new RuntimeScalar(remaining));
fireEvent(self, selfHash, pstate, "text", parsedScalar(pstate, remaining));
}

if (pstate.get("_buf").toString().isEmpty()
Expand Down
38 changes: 38 additions & 0 deletions src/test/resources/unit/html_parser_utf8_byte_slices.t
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
use strict;
use warnings;
use Encode qw(encode);
use HTML::Entities;
use HTML::Parser;
use Test::More tests => 4;

# HTML::Parser receives file content as octets. Its text callbacks must retain
# that representation: marking an octet slice as Unicode makes a later /\s/
# see UTF-8 continuation byte C2 A0 as a non-breaking space.
my $thai = join '', map { chr } (
0x0E04, 0x0E33, 0x0E23, 0x0E49, 0x0E2D, 0x0E07, 0x0E02, 0x0E2D,
0x0E04, 0x0E37, 0x0E19, 0x0E40, 0x0E07, 0x0E34, 0x0E19, 0x0E20,
0x0E32, 0x0E29, 0x0E35, 0x0E2D, 0x0E32, 0x0E01, 0x0E23,
);
my $thai_octets = encode('UTF-8', $thai);
my $octets = "Thai: $thai_octets";
my @text;

my $parser = HTML::Parser->new(
api_version => 3,
handlers => { text => [ sub { push @text, $_[0] }, 'dtext' ] },
);
$parser->parse("<p>$octets</p>");
$parser->eof;

is_deeply(\@text, [ $octets ], 'parser text callback preserves UTF-8 octets');

my @parts = split /(\s+)/, $text[0];
is_deeply(\@parts, [ 'Thai:', ' ', $thai_octets ],
'split does not treat a UTF-8 continuation byte as whitespace');
is(join('', @parts), $octets, 'split round trip preserves the complete Thai suffix');

my $tree_text = $octets;
HTML::Entities::decode($tree_text); # HTML::TreeBuilder decodes every non-CDATA text node this way.
my @tree_parts = split /(\s+)/, $tree_text;
is_deeply(\@tree_parts, [ 'Thai:', ' ', $thai_octets ],
'a no-op entity decode preserves UTF-8 octets for TreeBuilder');
Loading