diff --git a/src/Language/Lexer.php b/src/Language/Lexer.php index 841132271..59667f4d6 100644 --- a/src/Language/Lexer.php +++ b/src/Language/Lexer.php @@ -551,6 +551,12 @@ private function readString(int $line, int $col, Token $prev): Token throw new SyntaxError($this->source, $this->position, 'Unterminated string.'); } + /** Compares bytes, since decoding one byte at a time would land inside multibyte characters. */ + private function isTripleQuoteAhead(): bool + { + return substr($this->source->body, $this->byteStreamPosition, 3) === '"""'; + } + /** * Reads a block string token from the source file. * @@ -571,52 +577,31 @@ private function readBlockString(int $line, int $col, Token $prev): Token while ($code !== null) { // Closing Triple-Quote (""") - if ($code === 34) { - // Move 2 quotes - [, $nextCode] = $this->moveStringCursor(1, 1)->readChar(); - [, $nextNextCode] = $this->moveStringCursor(1, 1)->readChar(); + if ($code === 34 && $this->isTripleQuoteAhead()) { + $value .= $chunk; - if ($nextCode === 34 && $nextNextCode === 34) { - $value .= $chunk; - - $this->moveStringCursor(1, 1); - - return new Token( - Token::BLOCK_STRING, - $start, - $this->position, - $line, - $col, - $prev, - BlockString::dedentBlockStringLines($value) - ); - } + $this->moveStringCursor(3, 3); - // move cursor back to before the first quote - $this->moveStringCursor(-2, -2); + return new Token( + Token::BLOCK_STRING, + $start, + $this->position, + $line, + $col, + $prev, + BlockString::dedentBlockStringLines($value) + ); } $this->assertValidBlockStringCharacterCode($code, $this->position); $this->moveStringCursor(1, $bytes); - [, $nextCode] = $this->readChar(); - [, $nextNextCode] = $this->moveStringCursor(1, 1)->readChar(); - [, $nextNextNextCode] = $this->moveStringCursor(1, 1)->readChar(); - // Escape Triple-Quote (\""") - if ( - $code === 92 - && $nextCode === 34 - && $nextNextCode === 34 - && $nextNextNextCode === 34 - ) { - $this->moveStringCursor(1, 1); + if ($code === 92 && $this->isTripleQuoteAhead()) { + $this->moveStringCursor(3, 3); $value .= $chunk . '"""'; $chunk = ''; } else { - // move cursor back to before the first quote - $this->moveStringCursor(-2, -2); - if ($code === 10) { // new line ++$this->line; $this->lineStart = $this->position; diff --git a/tests/Language/LexerTest.php b/tests/Language/LexerTest.php index b1fc0ddc6..3ca52dc1d 100644 --- a/tests/Language/LexerTest.php +++ b/tests/Language/LexerTest.php @@ -50,7 +50,14 @@ private function lexOne(string $body): Token { $lexer = new Lexer(new Source($body)); - return $lexer->advance(); + set_error_handler(static function (int $severity, string $message): bool { + throw new \ErrorException($message, 0, $severity); + }); + try { + return $lexer->advance(); + } finally { + restore_error_handler(); + } } private static function loc(int $line, int $column): SourceLocation @@ -334,6 +341,14 @@ public function testLexesStringsWithUnescapedTabCharacter(): void ); } + public function testLexesBlockStringsWithMultibyteCharactersAfterQuotes(): void + { + self::assertArraySubset( + ['kind' => Token::BLOCK_STRING, 'start' => 0, 'end' => 11, 'value' => 'a"é\\ü'], + (array) $this->lexOne('"""a"é\\ü"""') + ); + } + /** @see it('lexes block strings') */ public function testLexesBlockString(): void { @@ -479,6 +494,9 @@ public static function reportsUsefulBlockStringErrors(): iterable { yield ['"""', 'Unterminated string.', self::loc(1, 4)]; yield ['"""no end quote', 'Unterminated string.', self::loc(1, 16)]; + yield ['"""a"é', 'Unterminated string.', self::loc(1, 7)]; + yield ['""""é', 'Unterminated string.', self::loc(1, 6)]; + yield ['"""\\é', 'Unterminated string.', self::loc(1, 6)]; yield [ '"""contains unescaped ' . json_decode('"\u0007"') . ' control char"""', 'Invalid character within String: "\\u0007"',