Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/rbgo/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func run(src, name string) error {
if err != nil {
return err
}
iseq, err := compiler.Compile(prog)
iseq, err := compiler.CompileWithEncoding(prog, compiler.MagicSourceEncoding(src))
if err != nil {
return err
}
Expand Down
94 changes: 91 additions & 3 deletions internal/compiler/compiler.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"math/big"
"strconv"
"strings"
"unicode/utf8"

"github.com/go-embedded-ruby/ruby/internal/bytecode"
"github.com/go-embedded-ruby/ruby/internal/object"
Expand Down Expand Up @@ -167,24 +168,111 @@ type Compiler struct {
ctxs []*loopCtx // innermost-last stack of break/next targets
retryTargets []int // innermost-last stack of begin-body PCs for `retry`
stack []*builder
// srcEnc is the source file's encoding (from a `# encoding:` magic comment),
// canonicalised to a registry name, or "" for the UTF-8 default. When set, a
// string literal is tagged with it (see newStrLit) so a literal in a
// `# encoding: binary` file is BINARY, matching MRI.
srcEnc string
}

// Compile lowers a Program into the top-level ISeq.
// Compile lowers a Program into the top-level ISeq, treating string literals as
// UTF-8 (the default source encoding).
func Compile(prog *ast.Program) (iseq *bytecode.ISeq, err error) {
return CompileWithEncoding(prog, "")
}

// CompileWithEncoding lowers a Program whose source declared srcEnc (a canonical
// encoding name, or "" for UTF-8) via a `# encoding:` magic comment, so string
// literals carry that encoding.
func CompileWithEncoding(prog *ast.Program, srcEnc string) (iseq *bytecode.ISeq, err error) {
defer func() {
if r := recover(); r != nil {
// Unchecked: a non-compileError is an internal bug and re-panics as a
// conversion error rather than leaving an uncovered re-panic branch.
iseq, err = nil, r.(compileError)
}
}()
c := &Compiler{}
c := &Compiler{srcEnc: srcEnc}
c.push(newBuilder("<main>", nil))
c.compileBody(prog.Body)
c.cur().emit(bytecode.OpReturn, 0, 0)
return c.pop().build(), nil
}

// newStrLit builds the String constant for a source literal, tagging it with the
// source file's encoding when one was declared. A literal whose bytes are valid
// multi-byte UTF-8 is left UTF-8 regardless: such content comes from a `\u` escape
// (or literal Unicode), which MRI always encodes as UTF-8 even in a non-UTF-8
// source. Pure-ASCII and byte-oriented (`\x`) content take the source encoding.
func (c *Compiler) newStrLit(s string) object.Value {
if c.srcEnc != "" {
b := []byte(s)
if !(utf8.Valid(b) && !asciiOnlyBytes(b)) {
return object.NewStringBytesEnc(b, c.srcEnc)
}
}
return object.NewString(s)
}

// asciiOnlyBytes reports whether every byte is 7-bit ASCII.
func asciiOnlyBytes(b []byte) bool {
for _, x := range b {
if x >= 0x80 {
return false
}
}
return true
}

// MagicSourceEncoding returns the canonical registry encoding name declared by a
// `# encoding:` / `# coding:` magic comment on the first line of src (or the
// second when the first is a shebang), or "" for UTF-8 or no declaration. Only the
// encodings the negotiation cares about are recognised by canonical spelling;
// anything else (including plain UTF-8) yields "" so literals keep the default.
func MagicSourceEncoding(src string) string {
line, rest, _ := strings.Cut(src, "\n")
if strings.HasPrefix(strings.TrimSpace(line), "#!") {
line, _, _ = strings.Cut(rest, "\n")
}
name := magicEncodingName(line)
switch strings.ToLower(name) {
case "binary", "ascii-8bit":
return "ASCII-8BIT"
case "us-ascii", "ascii":
return "US-ASCII"
}
return ""
}

// magicEncodingName extracts the value of a `coding:`/`encoding:` field from a
// comment line (e.g. `# -*- coding: binary -*-` or `# encoding: binary`), or "".
func magicEncodingName(line string) string {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "#") {
return ""
}
low := strings.ToLower(line)
i := strings.Index(low, "coding")
if i < 0 {
return ""
}
rest := line[i+len("coding"):]
rest = strings.TrimLeft(rest, ":= \t")
// The value ends at the first whitespace or a `-*-` terminator.
end := len(rest)
for j, r := range rest {
if r == ' ' || r == '\t' || r == ';' {
end = j
break
}
if strings.HasPrefix(rest[j:], "-*-") {
end = j
break
}
}
return strings.TrimSpace(rest[:end])
}

// CompileWithLocals lowers prog for a Binding eval: it compiles in a child scope
// whose parent holds the binding's locals (by slot), so references to them
// resolve at depth 1 — reads and writes reach the binding's environment when the
Expand Down Expand Up @@ -254,7 +342,7 @@ func (c *Compiler) compileNode(n ast.Node) {
// Integer/Float, or a Rational for the combined `ri` form).
b.emit(bytecode.OpPushConst, b.addConst(&object.Complex{Re: object.IntValue(0), Im: numericValue(v.Value)}), 0)
case *ast.StringLit:
b.emit(bytecode.OpPushConst, b.addConst(object.NewString(v.Value)), 0)
b.emit(bytecode.OpPushConst, b.addConst(c.newStrLit(v.Value)), 0)
case *ast.StrInterp:
// Concatenate each part coerced with to_s onto a growing string.
b.emit(bytecode.OpPushConst, b.addConst(object.NewString("")), 0)
Expand Down
82 changes: 82 additions & 0 deletions internal/compiler/magic_encoding_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package compiler

import (
"testing"

"github.com/go-embedded-ruby/ruby/internal/object"
"github.com/go-ruby-parser/parser"
)

func TestMagicSourceEncoding(t *testing.T) {
cases := []struct{ src, want string }{
{"# encoding: binary\nx = 1\n", "ASCII-8BIT"},
{"# encoding: BINARY\n", "ASCII-8BIT"},
{"# coding: ascii-8bit\n", "ASCII-8BIT"},
{"# -*- coding: binary -*-\n", "ASCII-8BIT"},
{"# encoding: us-ascii\n", "US-ASCII"},
{"# encoding: ascii\n", "US-ASCII"},
{"# encoding: utf-8\n", ""}, // UTF-8 is the default: no tag
{"# encoding: euc-jp\n", ""}, // unrecognised for tagging: no tag
{"x = 1\n", ""}, // no magic comment
{"puts 1 # encoding: binary\n", ""}, // must be a comment line, not trailing
{"# just a plain comment\n", ""}, // a comment without a coding: field
{"# coding:binary-*-\n", "ASCII-8BIT"}, // value terminated by -*- with no space
{"# coding: binary;\n", "ASCII-8BIT"}, // value terminated by a semicolon
{"#!/usr/bin/env ruby\n# encoding: binary\nx = 1\n", "ASCII-8BIT"}, // 2nd line after shebang
{"", ""},
}
for _, tc := range cases {
if got := MagicSourceEncoding(tc.src); got != tc.want {
t.Errorf("MagicSourceEncoding(%q) = %q, want %q", tc.src, got, tc.want)
}
}
}

// firstStringConst returns the encoding name of the first String constant in a
// program compiled with the given source encoding.
func firstStringConst(t *testing.T, src, srcEnc string) string {
t.Helper()
prog, err := parser.Parse(src)
if err != nil {
t.Fatalf("parse: %v", err)
}
iseq, err := CompileWithEncoding(prog, srcEnc)
if err != nil {
t.Fatalf("compile: %v", err)
}
for _, c := range iseq.Consts {
if s, ok := c.(*object.String); ok {
return s.EncName()
}
}
t.Fatalf("no String constant in %q", src)
return ""
}

func TestCompileWithEncodingTagsLiterals(t *testing.T) {
// A plain ASCII literal in a binary source is tagged BINARY.
if got := firstStringConst(t, `x = "abc"`, "ASCII-8BIT"); got != "ASCII-8BIT" {
t.Errorf(`"abc" in binary source: %q, want "ASCII-8BIT"`, got)
}
// A byte-oriented (\x, high bit) literal in a binary source is tagged BINARY.
if got := firstStringConst(t, `x = "\xff"`, "ASCII-8BIT"); got != "ASCII-8BIT" {
t.Errorf(`"\xff" in binary source: %q, want "ASCII-8BIT"`, got)
}
// A valid multi-byte UTF-8 literal (as a \u escape produces) stays UTF-8 even in
// a binary source, matching MRI's rule that \u forces UTF-8.
if got := firstStringConst(t, `x = "あ"`, "ASCII-8BIT"); got != "UTF-8" {
t.Errorf(`"あ" in binary source: %q, want "UTF-8"`, got)
}
// The default (UTF-8) source leaves every literal untagged / UTF-8.
if got := firstStringConst(t, `x = "abc"`, ""); got != "UTF-8" {
t.Errorf(`"abc" in default source: %q, want "UTF-8"`, got)
}
// Compile (the UTF-8 default entry point) behaves the same as an empty srcEnc.
prog, err := parser.Parse(`x = "abc"`)
if err != nil {
t.Fatalf("parse: %v", err)
}
if _, err := Compile(prog); err != nil {
t.Fatalf("Compile: %v", err)
}
}
Loading