diff --git a/roaring64/BSI_BENCHMARKS.md b/roaring64/BSI_BENCHMARKS.md index 506a37f2..091e1242 100644 --- a/roaring64/BSI_BENCHMARKS.md +++ b/roaring64/BSI_BENCHMARKS.md @@ -16,6 +16,9 @@ Commands: go test ./roaring64 -count=1 go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64BatchEqual' -benchmem -count 3 go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64Compare(Big)?Value|BenchmarkBSI64BatchEqual(Big)?LargeAgeFixture' -benchmem -count 1 +go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64CompareBSISameRow' -benchmem -count=5 +go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64GetBigValue' -benchmem -count=3 +go test ./roaring64 -run '^$' -bench 'BenchmarkBSI64BatchEqual.*LargeFixture' -benchmem -benchtime=2s -count=5 ``` Representative results: @@ -27,6 +30,9 @@ Representative results: | `BenchmarkBSI64CompareValueEQLargeAgeFixture` | ~4.44s/op, ~461MB/op | ~100-118ms/op, ~19.7MB/op | `EQ` delegates to optimized `BatchEqual`. | | `BenchmarkBSI64CompareValueRangeLargeAgeFixture` | ~7.49s/op, ~501MB/op | ~204-224ms/op, ~122.6MB/op | Uses bitmap-native signed int64 comparison. | | `BenchmarkBSI64CompareValueGELargeAgeFixture` | ~3.45s/op, ~500MB/op | ~168-184ms/op, ~82.3MB/op | Uses bitmap-native signed int64 comparison. | +| `BenchmarkBSI64CompareBSISameRowBitwise` | ~127-168ms/op, ~69.7MB/op | ~568-795us/op, ~619KB/op | Compares two BSI values per column ID through bitplane algebra instead of row-by-row `GetBigValue`. | +| `BenchmarkBSI64GetBigValuesLargeFixture` | ~69-92ms/op, ~35.6MB/op, ~1.3M allocs/op for a row-by-row `GetBigValue` loop | ~23-34ms/op, ~8.2MB/op, ~200k allocs/op | Extracts aligned BSI values for a column batch by walking bit-slices once. | +| `BenchmarkBSI64BatchEqualValuesLargeFixture` | ~5.4-7.1ms/op for `BatchEqual` plus `GetBigValues`; ~10.9-13.0ms/op for `BatchEqual` plus row-by-row `GetValue` | ~1.6-2.3ms/op, ~2.0MB/op, ~432 allocs/op | Emits matched column IDs and int64 values directly from trie leaves, avoiding a second value lookup pass. | Compatibility: @@ -36,6 +42,11 @@ Compatibility: - True wider-than-64-bit values continue to use the existing generic paths. - `BatchEqualBig` now keys values by sign and magnitude so positive and negative values with the same magnitude do not collide. +- `GetBigValues` returns values aligned to the requested column IDs, with nil + entries for missing values, while preserving `GetBigValue` semantics. +- `BatchEqualValues` returns matched column IDs and int64 values for `BatchEqual` + shapes, optionally restricted by a found set. Result order is intentionally + unspecified. Follow-up: diff --git a/roaring64/bsi64.go b/roaring64/bsi64.go index a5428aef..4e918ddb 100644 --- a/roaring64/bsi64.go +++ b/roaring64/bsi64.go @@ -25,6 +25,12 @@ type BSI struct { runOptimized bool } +// BSIValuePair is a column ID and its BSI value. +type BSIValuePair struct { + ColumnID uint64 + Value int64 +} + // NewBSI constructs a new BSI. Note that it is your responsibility to ensure that // the min/max values are set correctly. Queries CompareValue, MinMax, etc. will not // work correctly if the min/max values are not set correctly. @@ -208,6 +214,131 @@ func (b *BSI) GetBigValue(columnID uint64) (value *big.Int, exists bool) { return val, exists } +// GetBigValues gets values for the column IDs. Returned values are aligned with +// columnIDs, and a nil entry means the corresponding column ID has no value. +func (b *BSI) GetBigValues(columnIDs []uint64) []*big.Int { + values := make([]*big.Int, len(columnIDs)) + if len(columnIDs) == 0 { + return values + } + if len(columnIDs) == 1 { + if value, ok := b.GetBigValue(columnIDs[0]); ok { + values[0] = value + } + return values + } + request := newBSIGetBigValuesRequest(columnIDs) + if !b.isBig() { + return b.getBigValuesInt64(request, values) + } + return b.getBigValuesGeneric(request, values) +} + +type bsiGetBigValuesRequest struct { + foundSet *Bitmap + positions map[uint64]int + duplicatePositions map[uint64][]int +} + +func newBSIGetBigValuesRequest(columnIDs []uint64) bsiGetBigValuesRequest { + foundSet := NewBitmap() + positions := make(map[uint64]int, len(columnIDs)) + var duplicatePositions map[uint64][]int + for position, columnID := range columnIDs { + if _, ok := positions[columnID]; ok { + if duplicatePositions == nil { + duplicatePositions = make(map[uint64][]int) + } + duplicatePositions[columnID] = append(duplicatePositions[columnID], position) + continue + } + positions[columnID] = position + foundSet.Add(columnID) + } + return bsiGetBigValuesRequest{ + foundSet: foundSet, + positions: positions, + duplicatePositions: duplicatePositions, + } +} + +func (b *BSI) getBigValuesInt64(request bsiGetBigValuesRequest, values []*big.Int) []*big.Int { + existing := And(&b.eBM, request.foundSet) + if existing.IsEmpty() { + return values + } + + rawValues := make([]uint64, len(values)) + signBit := b.BitCount() + for bit := 0; bit <= signBit; bit++ { + bitSet := And(&b.bA[bit], existing) + iter := bitSet.Iterator() + for iter.HasNext() { + columnID := iter.Next() + rawValues[request.positions[columnID]] |= uint64(1) << uint(bit) + } + } + + width := uint(signBit + 1) + signMask := uint64(1) << uint(signBit) + iter := existing.Iterator() + for iter.HasNext() { + columnID := iter.Next() + position := request.positions[columnID] + rawValue := rawValues[position] + if rawValue&signMask != 0 && width < 64 { + rawValue |= ^uint64(0) << width + } + values[position] = big.NewInt(int64(rawValue)) + } + fillDuplicateBigValues(values, request) + return values +} + +func (b *BSI) getBigValuesGeneric(request bsiGetBigValuesRequest, values []*big.Int) []*big.Int { + existing := And(&b.eBM, request.foundSet) + if existing.IsEmpty() { + return values + } + + iter := existing.Iterator() + for iter.HasNext() { + values[request.positions[iter.Next()]] = big.NewInt(0) + } + for bit := b.BitCount(); bit >= 0; bit-- { + bitSet := And(&b.bA[bit], existing) + iter := bitSet.Iterator() + for iter.HasNext() { + columnID := iter.Next() + position := request.positions[columnID] + values[position].SetBit(values[position], bit, 1) + } + } + + signBit := b.BitCount() + negativeSet := And(&b.bA[signBit], existing) + iter = negativeSet.Iterator() + for iter.HasNext() { + position := request.positions[iter.Next()] + values[position] = negativeTwosComplementToInt(values[position]) + } + + fillDuplicateBigValues(values, request) + return values +} + +func fillDuplicateBigValues(values []*big.Int, request bsiGetBigValuesRequest) { + for columnID, extraPositions := range request.duplicatePositions { + value := values[request.positions[columnID]] + if value == nil { + continue + } + for _, position := range extraPositions { + values[position] = new(big.Int).Set(value) + } + } +} + func negativeTwosComplementToInt(val *big.Int) *big.Int { inverted := new(big.Int).Not(val) mask := new(big.Int).Lsh(big.NewInt(1), uint(val.BitLen())) @@ -356,6 +487,83 @@ func (b *BSI) CompareValue(parallelism int, op Operation, valueOrStart, end int6 return b.CompareBigValue(parallelism, op, big.NewInt(valueOrStart), big.NewInt(end), foundSet) } +// CompareBSI compares values from two BSIs by column ID and returns the column +// IDs where b[columnID] op other[columnID] is true. Only column IDs present in +// both existence bitmaps are considered. When foundSet is not nil, it further +// restricts the comparison universe. +func (b *BSI) CompareBSI(op Operation, other *BSI, foundSet *Bitmap) *Bitmap { + if b == nil || other == nil || b.eBM.IsEmpty() || other.eBM.IsEmpty() { + return NewBitmap() + } + universe := b.eBM.Clone() + universe.And(&other.eBM) + if foundSet != nil { + universe.And(foundSet) + } + if universe.IsEmpty() { + return universe + } + + commonSign := b.BitCount() + if other.BitCount() > commonSign { + commonSign = other.BitCount() + } + less, equal := b.compareBSILessAndEqual(other, commonSign, universe) + + switch op { + case LT: + return less + case LE: + less.Or(equal) + return less + case EQ: + return equal + case GE: + universe.AndNot(less) + return universe + case GT: + less.Or(equal) + universe.AndNot(less) + return universe + default: + panic(fmt.Sprintf("Operation [%v] not supported for BSI comparison", op)) + } +} + +func (b *BSI) compareBSILessAndEqual(other *BSI, commonSign int, universe *Bitmap) (*Bitmap, *Bitmap) { + less := NewBitmap() + equalPrefix := universe.Clone() + for i := commonSign; i >= 0; i-- { + leftOnes := b.compareBSIPlaneChild(equalPrefix, i, commonSign, true, false) + rightOnes := other.compareBSIPlaneChild(equalPrefix, i, commonSign, true, false) + + rightOnly := rightOnes.Clone() + rightOnly.AndNot(leftOnes) + less.Or(rightOnly) + + leftOnly := leftOnes + leftOnly.AndNot(rightOnes) + rightOnly.Or(leftOnly) + equalPrefix.AndNot(rightOnly) + if equalPrefix.IsEmpty() { + break + } + } + return less, equalPrefix +} + +func (b *BSI) compareBSIPlaneChild(prefix *Bitmap, planeIndex, commonSign int, set, owned bool) *Bitmap { + sourcePlane := planeIndex + if sourcePlane > b.BitCount() { + sourcePlane = b.BitCount() + } + rawSet := set + if planeIndex == commonSign { + rawSet = !rawSet + } + return bsi64PlaneChild(prefix, &b.bA[sourcePlane], rawSet, owned) +} + func (b *BSI) compareInt64Value(parallelism int, op Operation, valueOrStart, end int64, foundSet *Bitmap) (*Bitmap, bool) { bitCount := b.BitCount() if bitCount > 63 || !bsi64ValueFitsBitCount(valueOrStart, bitCount) { @@ -1092,24 +1300,11 @@ func (b *BSI) BatchEqual(parallelism int, values []int64) *Bitmap { return b.BatchEqualBig(parallelism, bigValues) } - seen := make(map[uint64]struct{}, len(values)) - vals := make([]uint64, 0, len(values)) - for _, v := range values { - if !bsi64ValueFitsBitCount(v, bitCount) { - continue - } - encoded := encodeBSI64Value(v, bitCount) - if _, ok := seen[encoded]; ok { - continue - } - seen[encoded] = struct{}{} - vals = append(vals, encoded) - } + vals := b.batchEqualInt64Values(values, bitCount) if len(vals) == 0 { return NewBitmap() } - sort.Slice(vals, func(i, j int) bool { return vals[i] < vals[j] }) if result, ok := b.matchInt64Cube(vals, bitCount); ok { if b.runOptimized { result.RunOptimize() @@ -1123,6 +1318,63 @@ func (b *BSI) BatchEqual(parallelism int, values []int64) *Bitmap { return result } +// BatchEqualValues returns column IDs and values where the BSI value is +// contained in values. When foundSet is not nil, only column IDs in foundSet are +// considered. Result order is not guaranteed. +func (b *BSI) BatchEqualValues(parallelism int, values []int64, foundSet *Bitmap) []BSIValuePair { + if b.eBM.IsEmpty() || len(values) == 0 { + return nil + } + + bitCount := b.BitCount() + if bitCount >= 64 { + matched := b.BatchEqual(parallelism, values) + if foundSet != nil { + matched.And(foundSet) + } + return b.bsiValuePairsFromBitmap(matched) + } + + vals := b.batchEqualInt64Values(values, bitCount) + if len(vals) == 0 { + return nil + } + + var universe *Bitmap + owned := false + if foundSet == nil { + universe = &b.eBM + } else { + universe = And(&b.eBM, foundSet) + owned = true + } + if universe.IsEmpty() { + return nil + } + + pairs := make([]BSIValuePair, 0) + b.matchInt64TrieValues(vals, bitCount, universe, owned, 0, &pairs) + return pairs +} + +func (b *BSI) batchEqualInt64Values(values []int64, bitCount int) []uint64 { + seen := make(map[uint64]struct{}, len(values)) + vals := make([]uint64, 0, len(values)) + for _, v := range values { + if !bsi64ValueFitsBitCount(v, bitCount) { + continue + } + encoded := encodeBSI64Value(v, bitCount) + if _, ok := seen[encoded]; ok { + continue + } + seen[encoded] = struct{}{} + vals = append(vals, encoded) + } + sort.Slice(vals, func(i, j int) bool { return vals[i] < vals[j] }) + return vals +} + func bsi64ValueFitsBitCount(value int64, bitCount int) bool { if bitCount >= 63 { return true @@ -1140,6 +1392,18 @@ func encodeBSI64Value(value int64, bitCount int) uint64 { return uint64(value) & mask } +func decodeBSI64Value(encoded uint64, bitCount int) int64 { + if bitCount >= 63 { + return int64(encoded) + } + width := uint(bitCount + 1) + signMask := uint64(1) << uint(bitCount) + if encoded&signMask != 0 && width < 64 { + encoded |= ^uint64(0) << width + } + return int64(encoded) +} + func (b *BSI) matchInt64Cube(vals []uint64, bitCount int) (*Bitmap, bool) { if bitCount >= 63 { return nil, false @@ -1220,6 +1484,56 @@ func (b *BSI) matchInt64Trie(vals []uint64, p int, prefix *Bitmap, owned bool) * } } +func (b *BSI) matchInt64TrieValues(vals []uint64, p int, prefix *Bitmap, owned bool, encoded uint64, pairs *[]BSIValuePair) { + if prefix.IsEmpty() { + return + } + if p < 0 { + value := decodeBSI64Value(encoded, b.BitCount()) + iter := prefix.Iterator() + for iter.HasNext() { + *pairs = append(*pairs, BSIValuePair{ + ColumnID: iter.Next(), + Value: value, + }) + } + return + } + + mask := uint64(1) << uint(p) + cut := sort.Search(len(vals), func(i int) bool { return vals[i]&mask != 0 }) + lo, hi := vals[:cut], vals[cut:] + switch { + case len(hi) == 0: + b.matchInt64TrieValues(lo, p-1, bsi64PlaneChild(prefix, &b.bA[p], false, owned), true, encoded, pairs) + case len(lo) == 0: + b.matchInt64TrieValues(hi, p-1, bsi64PlaneChild(prefix, &b.bA[p], true, owned), true, encoded|mask, pairs) + default: + hiBM := And(prefix, &b.bA[p]) + b.matchInt64TrieValues(lo, p-1, bsi64PlaneChild(prefix, &b.bA[p], false, owned), true, encoded, pairs) + b.matchInt64TrieValues(hi, p-1, hiBM, true, encoded|mask, pairs) + } +} + +func (b *BSI) bsiValuePairsFromBitmap(matched *Bitmap) []BSIValuePair { + if matched == nil || matched.IsEmpty() { + return nil + } + columnIDs := matched.ToArray() + values := b.GetBigValues(columnIDs) + pairs := make([]BSIValuePair, 0, len(columnIDs)) + for i, columnID := range columnIDs { + if i >= len(values) || values[i] == nil { + continue + } + pairs = append(pairs, BSIValuePair{ + ColumnID: columnID, + Value: values[i].Int64(), + }) + } + return pairs +} + func bsi64PlaneChild(prefix, plane *Bitmap, set, owned bool) *Bitmap { if owned { if set { diff --git a/roaring64/bsi64_batch_equal_values_test.go b/roaring64/bsi64_batch_equal_values_test.go new file mode 100644 index 00000000..d5730937 --- /dev/null +++ b/roaring64/bsi64_batch_equal_values_test.go @@ -0,0 +1,149 @@ +package roaring64 + +import ( + "math/big" + "math/rand" + "sort" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBSI64BatchEqualValuesConsistentWithBatchEqual(t *testing.T) { + rg := rand.New(rand.NewSource(909)) + for run := 0; run < 25; run++ { + bsi := NewDefaultBSI() + numCols := rg.Intn(1000) + 50 + for col := 0; col < numCols; col++ { + if rg.Float64() < 0.90 { + bsi.SetValue(uint64(col), rg.Int63n(400)-200) + } + } + + values := []int64{-200, -99, -5, 0, 7, 42, 42, 199} + foundSet := NewBitmap() + for col := 0; col < numCols; col++ { + if col%3 != 0 { + foundSet.Add(uint64(col)) + } + } + + for _, fs := range []*Bitmap{nil, foundSet} { + expected := expectedBSI64BatchEqualValues(bsi, values, fs) + actual := bsi.BatchEqualValues(0, values, fs) + assert.Equal(t, expected, sortedBSI64ValuePairs(actual), "run=%d foundSet=%v", run, fs != nil) + } + } +} + +func TestBSI64BatchEqualValuesHandlesBigWidthFallback(t *testing.T) { + bsi := NewDefaultBSI() + huge := new(big.Int).Lsh(big.NewInt(1), 90) + bsi.SetBigValue(1, huge) + bsi.SetValue(2, -7) + bsi.SetValue(3, 11) + bsi.SetValue(4, -7) + + foundSet := BitmapOf(1, 2, 3) + actual := sortedBSI64ValuePairs(bsi.BatchEqualValues(0, []int64{-7, 11}, foundSet)) + assert.Equal(t, []BSIValuePair{ + {ColumnID: 2, Value: -7}, + {ColumnID: 3, Value: 11}, + }, actual) +} + +func BenchmarkBSI64BatchEqualValuesLargeFixture(b *testing.B) { + bsi, values, foundSet := setupBSI64BatchEqualValuesFixture(b, 100000, 100, 27) + b.ResetTimer() + for i := 0; i < b.N; i++ { + pairs := bsi.BatchEqualValues(0, values, foundSet) + _ = pairs + } +} + +func BenchmarkBSI64BatchEqualGetBigValuesLargeFixture(b *testing.B) { + bsi, values, foundSet := setupBSI64BatchEqualValuesFixture(b, 100000, 100, 27) + b.ResetTimer() + for i := 0; i < b.N; i++ { + matched := bsi.BatchEqual(0, values) + matched.And(foundSet) + columnIDs := matched.ToArray() + bigValues := bsi.GetBigValues(columnIDs) + pairs := make([]BSIValuePair, 0, len(columnIDs)) + for j, columnID := range columnIDs { + if bigValues[j] != nil { + pairs = append(pairs, BSIValuePair{ColumnID: columnID, Value: bigValues[j].Int64()}) + } + } + _ = pairs + } +} + +func BenchmarkBSI64BatchEqualGetValueLoopLargeFixture(b *testing.B) { + bsi, values, foundSet := setupBSI64BatchEqualValuesFixture(b, 100000, 100, 27) + b.ResetTimer() + for i := 0; i < b.N; i++ { + matched := bsi.BatchEqual(0, values) + matched.And(foundSet) + pairs := make([]BSIValuePair, 0, int(matched.GetCardinality())) + iter := matched.Iterator() + for iter.HasNext() { + columnID := iter.Next() + value, ok := bsi.GetValue(columnID) + if ok { + pairs = append(pairs, BSIValuePair{ColumnID: columnID, Value: value}) + } + } + _ = pairs + } +} + +func expectedBSI64BatchEqualValues(bsi *BSI, values []int64, foundSet *Bitmap) []BSIValuePair { + matched := bsi.BatchEqual(0, values) + if foundSet != nil { + matched.And(foundSet) + } + pairs := make([]BSIValuePair, 0, int(matched.GetCardinality())) + iter := matched.Iterator() + for iter.HasNext() { + columnID := iter.Next() + value, ok := bsi.GetValue(columnID) + if ok { + pairs = append(pairs, BSIValuePair{ColumnID: columnID, Value: value}) + } + } + return sortedBSI64ValuePairs(pairs) +} + +func sortedBSI64ValuePairs(pairs []BSIValuePair) []BSIValuePair { + sorted := append([]BSIValuePair(nil), pairs...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].ColumnID != sorted[j].ColumnID { + return sorted[i].ColumnID < sorted[j].ColumnID + } + return sorted[i].Value < sorted[j].Value + }) + return sorted +} + +func setupBSI64BatchEqualValuesFixture(tb testing.TB, rows, valueDomain, valueCount int) (*BSI, []int64, *Bitmap) { + tb.Helper() + bsi := NewDefaultBSI() + for row := 0; row < rows; row++ { + value := int64(row%valueDomain) - int64(valueDomain/2) + bsi.SetValue(uint64(row), value) + } + + values := make([]int64, 0, valueCount) + for i := 0; i < valueCount; i++ { + values = append(values, int64((i*7)%valueDomain)-int64(valueDomain/2)) + } + + foundSet := NewBitmap() + for row := 0; row < rows; row++ { + if row%5 != 0 { + foundSet.Add(uint64(row)) + } + } + return bsi, values, foundSet +} diff --git a/roaring64/bsi64_compare_bsi_test.go b/roaring64/bsi64_compare_bsi_test.go new file mode 100644 index 00000000..9d3b7b67 --- /dev/null +++ b/roaring64/bsi64_compare_bsi_test.go @@ -0,0 +1,166 @@ +package roaring64 + +import ( + "math/big" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func expectedBSI64CompareBSI(left *BSI, op Operation, right *BSI, foundSet *Bitmap) *Bitmap { + expected := NewBitmap() + source := And(left.GetExistenceBitmap(), right.GetExistenceBitmap()) + if foundSet != nil { + source.And(foundSet) + } + iter := source.Iterator() + for iter.HasNext() { + col := iter.Next() + leftValue, leftOK := left.GetBigValue(col) + rightValue, rightOK := right.GetBigValue(col) + if !leftOK || !rightOK { + continue + } + compare := leftValue.Cmp(rightValue) + switch op { + case LT: + if compare < 0 { + expected.Add(col) + } + case LE: + if compare <= 0 { + expected.Add(col) + } + case EQ: + if compare == 0 { + expected.Add(col) + } + case GE: + if compare >= 0 { + expected.Add(col) + } + case GT: + if compare > 0 { + expected.Add(col) + } + default: + panic("unsupported test operation") + } + } + return expected +} + +func TestBSI64CompareBSIConsistentWithGetBigValue(t *testing.T) { + rg := rand.New(rand.NewSource(122)) + for run := 0; run < 25; run++ { + left := NewDefaultBSI() + right := NewDefaultBSI() + numCols := rg.Intn(1000) + 50 + for col := 0; col < numCols; col++ { + if rg.Float64() < 0.90 { + left.SetValue(uint64(col), rg.Int63n(2000)-1000) + } + if rg.Float64() < 0.85 { + right.SetValue(uint64(col), rg.Int63n(2000)-1000) + } + } + // Force different bit widths and signs across the two BSIs. + left.SetValue(uint64(numCols+1), 1<<40) + right.SetValue(uint64(numCols+1), -1) + left.SetValue(uint64(numCols+2), -1) + right.SetValue(uint64(numCols+2), 1<<35) + + foundSet := NewBitmap() + source := And(left.GetExistenceBitmap(), right.GetExistenceBitmap()) + iter := source.Iterator() + for iter.HasNext() { + col := iter.Next() + if col%3 != 0 { + foundSet.Add(col) + } + } + + for _, op := range []Operation{LT, LE, EQ, GE, GT} { + for _, fs := range []*Bitmap{nil, foundSet} { + expected := expectedBSI64CompareBSI(left, op, right, fs) + actual := left.CompareBSI(op, right, fs) + assert.True(t, actual.Equals(expected), "run=%d op=%d foundSet=%v expected=%v actual=%v", + run, op, fs != nil, expected.ToArray(), actual.ToArray()) + } + } + } +} + +func TestBSI64CompareBSIExistenceAndResultIsolation(t *testing.T) { + left := NewDefaultBSI() + right := NewDefaultBSI() + left.SetValue(1, 10) + left.SetValue(2, 20) + right.SetValue(2, 15) + right.SetValue(3, 5) + + actual := left.CompareBSI(GT, right, nil) + assert.True(t, actual.Equals(BitmapOf(2))) + + actual.Add(99) + actual.Remove(2) + assert.True(t, left.GetExistenceBitmap().Contains(2)) + assert.True(t, right.GetExistenceBitmap().Contains(2)) + assert.False(t, left.GetExistenceBitmap().Contains(99)) +} + +func TestBSI64CompareBSIBigWidthConsistentWithGetBigValue(t *testing.T) { + left := NewDefaultBSI() + right := NewDefaultBSI() + huge := new(big.Int).Lsh(big.NewInt(1), 90) + hugePlusOne := new(big.Int).Add(huge, big.NewInt(1)) + negativeHuge := new(big.Int).Neg(huge) + negativeHugeMinusOne := new(big.Int).Sub(negativeHuge, big.NewInt(1)) + + left.SetBigValue(1, huge) + right.SetBigValue(1, hugePlusOne) + left.SetBigValue(2, hugePlusOne) + right.SetBigValue(2, huge) + left.SetBigValue(3, negativeHuge) + right.SetBigValue(3, huge) + left.SetBigValue(4, negativeHugeMinusOne) + right.SetBigValue(4, negativeHuge) + left.SetBigValue(5, negativeHuge) + right.SetBigValue(5, negativeHuge) + + assert.True(t, left.CompareBSI(LT, right, nil).Equals(BitmapOf(1, 3, 4))) + assert.True(t, left.CompareBSI(GT, right, nil).Equals(BitmapOf(2))) + assert.True(t, left.CompareBSI(EQ, right, nil).Equals(BitmapOf(5))) + assert.True(t, left.CompareBSI(LE, right, nil).Equals(BitmapOf(1, 3, 4, 5))) + assert.True(t, left.CompareBSI(GE, right, nil).Equals(BitmapOf(2, 5))) +} + +func BenchmarkBSI64CompareBSISameRowBitwise(b *testing.B) { + left, right := setupBSI64CompareBSIFixture(b, 100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := left.CompareBSI(GT, right, nil) + _ = res + } +} + +func BenchmarkBSI64CompareBSISameRowGetBigValue(b *testing.B) { + left, right := setupBSI64CompareBSIFixture(b, 100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + res := expectedBSI64CompareBSI(left, GT, right, nil) + _ = res + } +} + +func setupBSI64CompareBSIFixture(tb testing.TB, rows int) (*BSI, *BSI) { + tb.Helper() + left := NewDefaultBSI() + right := NewDefaultBSI() + for row := 0; row < rows; row++ { + left.SetValue(uint64(row), int64(row%1000)-500) + right.SetValue(uint64(row), int64((row*7)%1000)-500) + } + return left, right +} diff --git a/roaring64/bsi64_get_big_values_test.go b/roaring64/bsi64_get_big_values_test.go new file mode 100644 index 00000000..38049b2d --- /dev/null +++ b/roaring64/bsi64_get_big_values_test.go @@ -0,0 +1,105 @@ +package roaring64 + +import ( + "math/big" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBSI64GetBigValuesConsistentWithGetBigValue(t *testing.T) { + rg := rand.New(rand.NewSource(864)) + for run := 0; run < 25; run++ { + bsi := NewDefaultBSI() + numCols := rg.Intn(1000) + 50 + for col := 0; col < numCols; col++ { + if rg.Float64() < 0.85 { + bsi.SetValue(uint64(col), rg.Int63n(4000)-2000) + } + } + + columnIDs := make([]uint64, 0, numCols+8) + for col := numCols - 1; col >= 0; col-- { + if col%3 != 0 { + columnIDs = append(columnIDs, uint64(col)) + } + } + columnIDs = append(columnIDs, uint64(numCols+10), 7, 7, 11) + + actual := bsi.GetBigValues(columnIDs) + if len(actual) != len(columnIDs) { + t.Fatalf("run=%d values length = %d, want %d", run, len(actual), len(columnIDs)) + } + for i, columnID := range columnIDs { + expectedValue, expectedOK := bsi.GetBigValue(columnID) + actualValue := actual[i] + if !expectedOK { + assert.Nil(t, actualValue, "run=%d column=%d", run, columnID) + continue + } + if assert.NotNil(t, actualValue, "run=%d column=%d", run, columnID) { + assert.Equal(t, 0, actualValue.Cmp(expectedValue), "run=%d column=%d", run, columnID) + } + } + } +} + +func TestBSI64GetBigValuesHandlesBigWidthAndDuplicates(t *testing.T) { + bsi := NewDefaultBSI() + huge := new(big.Int).Lsh(big.NewInt(1), 90) + hugePlusSeven := new(big.Int).Add(huge, big.NewInt(7)) + negativeHuge := new(big.Int).Neg(hugePlusSeven) + + bsi.SetBigValue(1, hugePlusSeven) + bsi.SetBigValue(2, negativeHuge) + bsi.SetValue(4, 0) + + values := bsi.GetBigValues([]uint64{2, 3, 1, 2, 4}) + assert.Equal(t, 5, len(values)) + assert.Equal(t, 0, values[0].Cmp(negativeHuge)) + assert.Nil(t, values[1]) + assert.Equal(t, 0, values[2].Cmp(hugePlusSeven)) + assert.Equal(t, 0, values[3].Cmp(negativeHuge)) + assert.Equal(t, 0, values[4].Cmp(big.NewInt(0))) + + values[0].SetInt64(12) + assert.Equal(t, 0, values[3].Cmp(negativeHuge), "duplicate result values should be independent") + stored, ok := bsi.GetBigValue(2) + assert.True(t, ok) + assert.Equal(t, 0, stored.Cmp(negativeHuge), "mutating returned values must not alter the BSI") +} + +func BenchmarkBSI64GetBigValuesLargeFixture(b *testing.B) { + bsi, _ := setupBSI64CompareBSIFixture(b, 100000) + columnIDs := bsi64SequentialColumns(100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + values := bsi.GetBigValues(columnIDs) + _ = values + } +} + +func BenchmarkBSI64GetBigValueLoopLargeFixture(b *testing.B) { + bsi, _ := setupBSI64CompareBSIFixture(b, 100000) + columnIDs := bsi64SequentialColumns(100000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + values := make([]*big.Int, len(columnIDs)) + for j, columnID := range columnIDs { + value, ok := bsi.GetBigValue(columnID) + if ok { + values[j] = value + } + } + _ = values + } +} + +func bsi64SequentialColumns(n int) []uint64 { + columnIDs := make([]uint64, n) + for i := range columnIDs { + columnIDs[i] = uint64(i) + } + return columnIDs +}