From f45c412b66d5de39cf3825ec35897c47bec08384 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 25 Aug 2026 11:30:37 +0200 Subject: [PATCH] Add the 'w' (BER-compressed integer) pack/unpack directive Array#pack("w") and String#unpack("w") raised "unknown directive 'w'". They now implement BER compression: - pack("w") writes an integer as base-128 groups, most significant first, with the high bit set on every byte but the last (zero is a single NUL byte). The value may be any size (a Bignum encodes to as many bytes as needed); a Float is truncated toward zero and an object is coerced with #to_int. A negative value raises ArgumentError, and the result is BINARY. - unpack("w") reads base-128 groups until a byte with the high bit clear, one integer per count (all of them for '*'), returning Integer or Bignum values. The arbitrary-precision arithmetic runs on math/big, so a round trip is exact for values far beyond a machine word. Verified against ruby 4.0.6. Full package coverage stays at 100.0%. Zero- regression rubyspec sweep; core/array/pack/w_spec 6 -> 18 and core/string/unpack/w_spec 4 -> 9. Co-Authored-By: Claude Opus 4.8 --- internal/vm/pack.go | 22 +++++++- internal/vm/pack_ber.go | 73 +++++++++++++++++++++++++++ internal/vm/pack_ber_internal_test.go | 62 +++++++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 internal/vm/pack_ber.go create mode 100644 internal/vm/pack_ber_internal_test.go diff --git a/internal/vm/pack.go b/internal/vm/pack.go index a3bbf409..d102532d 100644 --- a/internal/vm/pack.go +++ b/internal/vm/pack.go @@ -176,7 +176,7 @@ func isPackCode(c byte) bool { case 'C', 'c', 'S', 's', 'L', 'l', 'Q', 'q', 'I', 'i', 'J', 'j', 'n', 'N', 'v', 'V', 'a', 'A', 'Z', 'H', 'h', 'U', 'f', 'F', 'd', 'D', 'e', 'E', 'g', 'G', - 'x', 'X', '@', 'b', 'B', 'm', 'M', 'u': + 'x', 'X', '@', 'b', 'B', 'm', 'M', 'u', 'w': return true } return false @@ -463,6 +463,18 @@ func (vm *VM) packBytes(elems []object.Value, fmtStr string) ([]byte, string) { out = qpencode(out, []byte(vm.displayStr(next())), packQPLen(d)) case d.code == 'u': out = packUuencode(out, d, vm.packStrArg(next())) + case d.code == 'w': + count := d.count + if d.star { + count = len(elems) - idx + } + for k := 0; k < count; k++ { + z := vm.packBERArg(next()) + if z.Sign() < 0 { + raise("ArgumentError", "can't compress negative numbers") + } + out = packBER(out, z) + } } } return out, packEncoding(dirs) @@ -1067,6 +1079,14 @@ func unpackElems(data []byte, fmtStr string) []object.Value { dec := uudecode(data[pos:]) pos = len(data) out = append(out, object.NewStringBytesEnc(dec, "ASCII-8BIT")) + case d.code == 'w': + // A BER integer is variable-length, so '*' means "until the input is + // exhausted"; a numeric count decodes at most that many. + for k := 0; (d.star || k < d.count) && pos < len(data); k++ { + var z *big.Int + z, pos = berDecodeAt(data, pos) + out = append(out, object.NormInt(z)) + } } } return out diff --git a/internal/vm/pack_ber.go b/internal/vm/pack_ber.go new file mode 100644 index 00000000..ecc0059b --- /dev/null +++ b/internal/vm/pack_ber.go @@ -0,0 +1,73 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import ( + "math/big" + + "github.com/go-embedded-ruby/ruby/internal/object" +) + +// packBERArg coerces a pack 'w' element to an arbitrary-precision integer: an +// Integer or Bignum directly, a Float truncated toward zero, anything else via +// #to_int (TypeError when it does not respond). The value may be a Bignum, since +// BER compression encodes integers of any size. +func (vm *VM) packBERArg(v object.Value) *big.Int { + if z, ok := object.BigOf(v); ok { + return z + } + if f, ok := v.(object.Float); ok { + z, _ := big.NewFloat(float64(f)).Int(nil) + return z + } + if vm.respondsToDynamic(v, "to_int") { + return vm.packBERArg(vm.send(v, "to_int", nil, nil)) + } + raise("TypeError", "no implicit conversion of %s into Integer", classNameOf(v)) + return nil +} + +// packBER appends the BER-compressed encoding of a non-negative integer: its +// base-128 digits, most significant first, with the high bit set on every byte +// but the last. Zero encodes as a single 0x00 byte. +func packBER(out []byte, z *big.Int) []byte { + if z.Sign() == 0 { + return append(out, 0) + } + n := new(big.Int).Set(z) + mask := big.NewInt(0x7f) + tmp := new(big.Int) + var groups []byte + for n.Sign() > 0 { + groups = append(groups, byte(tmp.And(n, mask).Int64())) + n.Rsh(n, 7) + } + for i := len(groups) - 1; i >= 0; i-- { + b := groups[i] + if i != 0 { + b |= 0x80 + } + out = append(out, b) + } + return out +} + +// berDecodeAt decodes one BER-compressed integer from data starting at pos, +// reading base-128 groups (7 bits each, most significant first) until a byte +// with the high bit clear, and returns the value and the index just past it. +func berDecodeAt(data []byte, pos int) (*big.Int, int) { + z := new(big.Int) + low := new(big.Int) + for pos < len(data) { + b := data[pos] + pos++ + z.Lsh(z, 7) + z.Or(z, low.SetInt64(int64(b&0x7f))) + if b&0x80 == 0 { + break + } + } + return z, pos +} diff --git a/internal/vm/pack_ber_internal_test.go b/internal/vm/pack_ber_internal_test.go new file mode 100644 index 00000000..ef8f84c4 --- /dev/null +++ b/internal/vm/pack_ber_internal_test.go @@ -0,0 +1,62 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import "testing" + +// TestPackBER covers Array#pack's 'w' directive (BER-compressed integers): the +// base-128 big-endian encoding with the high bit set on all but the last byte, +// zero as a single NUL, arbitrary-size (Bignum) values, the count/'*' modifier, +// a Float truncated toward zero, #to_int coercion, the ArgumentError for a +// negative value, and the BINARY result encoding. Verified against ruby 4.0.6. +func TestPackBER(t *testing.T) { + cases := []struct{ src, want string }{ + {`p [0].pack("w").bytes`, `[0]`}, + {`p [1].pack("w").bytes`, `[1]`}, + {`p [127].pack("w").bytes`, `[127]`}, + {`p [128].pack("w").bytes`, `[129, 0]`}, + {`p [9999].pack("w").bytes`, `[206, 15]`}, + {`p [16384].pack("w").bytes`, `[129, 128, 0]`}, + {`p [2**65].pack("w").bytes`, `[132, 128, 128, 128, 128, 128, 128, 128, 128, 0]`}, + // count / '*' modifiers. + {`p [1, 2, 300].pack("w*").bytes`, `[1, 2, 130, 44]`}, + {`p [1, 2, 3].pack("w2").bytes`, `[1, 2]`}, + // A Float is truncated toward zero; #to_int coerces an object. + {`p [1.9].pack("w").bytes`, `[1]`}, + {`class I; def to_int; 130; end; end; p [I.new].pack("w").bytes`, `[129, 2]`}, + // The result is BINARY. + {`p [1].pack("w").encoding.name`, `"ASCII-8BIT"`}, + // A negative value raises ArgumentError; a non-integer raises TypeError. + {`begin; [-1].pack("w"); rescue ArgumentError => e; p e.message; end`, `"can't compress negative numbers"`}, + {`begin; [Object.new].pack("w"); rescue TypeError => e; p e.class; end`, `TypeError`}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want+"\n" { + t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n") + } + } +} + +// TestUnpackBER covers String#unpack's 'w' directive: decoding one BER integer, +// a numeric count, the '*' modifier decoding every integer, two directives each +// taking one, Bignum results, and a pack/unpack round trip. Verified against ruby +// 4.0.6. +func TestUnpackBER(t *testing.T) { + cases := []struct{ src, want string }{ + {`p "\x00".unpack("w")`, `[0]`}, + {`p "\xce\x0f".unpack("w")`, `[9999]`}, + {`p "\x01\x02\x03\x04".unpack("w*")`, `[1, 2, 3, 4]`}, + {`p "\x01\x02\x03".unpack("w w")`, `[1, 2]`}, + {`p "\x01\x02\x03".unpack("w2")`, `[1, 2]`}, + {`p "\x84\x80\x80\x80\x80\x80\x80\x80\x80\x00".unpack("w")`, `[36893488147419103232]`}, + // A round trip over a mix of small and Bignum values is the identity. + {`a = [0, 127, 128, 9999, 2**65, 1]; p a.pack("w*").unpack("w*") == a`, `true`}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want+"\n" { + t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n") + } + } +}