From d8b6c3a57c9f8f550c5654d43249fa7f60849b38 Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Wed, 4 Nov 2026 14:00:18 -0800 Subject: [PATCH 1/3] change pipe-level none handling, empty subquery, and add has() This commit changes none handling at the output of relevant pipe operators to drop singleton nones. If a none is created, the query writer knows what they are doing, and if they didn't want to drop it, they should use ??. We also changed the behavior of pipe expression subqueries to return none instead of null for empty result. SQL subqueries still return null. In agg output, none values are dropped. For single column output with a none column, the output is a singleton none, which must be handled by the query or it fails type check. To implement this, we needed an expression that could check the existence of a field so we brought back the has() function with new semantics. --- book/src/SUMMARY.md | 1 + book/src/super-sql/expressions/match.md | 3 +- book/src/super-sql/expressions/subqueries.md | 20 +++ .../super-sql/expressions/type-assertions.md | 3 +- book/src/super-sql/functions/generics/ok.md | 5 +- book/src/super-sql/functions/records/has.md | 112 ++++++++++++++ book/src/super-sql/operators/aggregate.md | 4 + book/src/super-sql/operators/debug.md | 2 + book/src/super-sql/operators/unnest.md | 3 +- book/src/super-sql/operators/values.md | 22 +++ compiler/dag/expr.go | 6 +- compiler/dag/op.go | 2 +- compiler/rungen/vexpr.go | 4 +- compiler/rungen/vop.go | 2 +- compiler/semantic/analyzer.go | 3 - compiler/semantic/checker.go | 128 +++++++++++++--- compiler/semantic/compare.go | 2 +- compiler/semantic/dagen.go | 1 + compiler/semantic/expr.go | 20 +-- compiler/semantic/nonempty.go | 71 +++++++++ compiler/semantic/op.go | 43 +++--- compiler/semantic/sem/expr.go | 6 +- compiler/semantic/ztests/agg-missing.yaml | 5 +- compiler/semantic/ztests/imdb.yaml | 4 +- compiler/semantic/ztests/none-absent.yaml | 36 +++++ compiler/semantic/ztests/none-output.yaml | 50 +++--- compiler/semantic/ztests/none.yaml | 3 +- runtime/expr/function/function.go | 6 +- runtime/expr/function/has.go | 93 ++++++++++++ runtime/op/aggregate/aggregate.go | 6 + runtime/op/cachedsubquery.go | 10 +- runtime/op/debug.go | 3 +- runtime/op/nones.go | 142 ++++++++++++++++++ runtime/op/output.go | 103 ++----------- runtime/op/subquery.go | 18 ++- runtime/op/unnest.go | 41 +++-- runtime/op/values.go | 18 ++- runtime/ztests/expr/function/has.yaml | 89 +++++++++++ runtime/ztests/expr/function/under.yaml | 1 - runtime/ztests/expr/index.yaml | 5 - runtime/ztests/expr/subquery-empty.yaml | 21 +++ runtime/ztests/op/aggregate/none-key.yaml | 64 ++++++++ runtime/ztests/op/debug-none.yaml | 11 ++ runtime/ztests/op/unnest.yaml | 55 ++++++- runtime/ztests/op/values/drop-none.yaml | 55 +++++++ sio/bsupio/ztests/issue-5736.yaml | 2 +- spq_test.go | 10 +- sup/ztests/decorate-none.yaml | 1 - vector/recordbuilder.go | 41 +++-- 49 files changed, 1124 insertions(+), 232 deletions(-) create mode 100644 book/src/super-sql/functions/records/has.md create mode 100644 compiler/semantic/nonempty.go create mode 100644 compiler/semantic/ztests/none-absent.yaml create mode 100644 runtime/expr/function/has.go create mode 100644 runtime/op/nones.go create mode 100644 runtime/ztests/expr/function/has.yaml create mode 100644 runtime/ztests/expr/subquery-empty.yaml create mode 100644 runtime/ztests/op/aggregate/none-key.yaml create mode 100644 runtime/ztests/op/debug-none.yaml create mode 100644 runtime/ztests/op/values/drop-none.yaml diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index afc4872425..f512c350d9 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -137,6 +137,7 @@ - [regexp_replace](super-sql/functions/parsing/regexp_replace.md) - [Records](super-sql/functions/records/intro.md) - [flatten](super-sql/functions/records/flatten.md) + - [has](super-sql/functions/records/has.md) - [keys](super-sql/functions/records/keys.md) - [nest_dotted](super-sql/functions/records/nest_dotted.md) - [paths](super-sql/functions/records/paths.md) diff --git a/book/src/super-sql/expressions/match.md b/book/src/super-sql/expressions/match.md index 5688f660d0..664dfbabe0 100644 --- a/book/src/super-sql/expressions/match.md +++ b/book/src/super-sql/expressions/match.md @@ -19,7 +19,8 @@ match expression. A value matches a type by the same rules as a its underlying value and [named types](../types/named.md) are distinct. If no case matches, the result is `` or, without an -`else` clause, `none`, which disappears from a record, array, or map +`else` clause, `none`, which disappears from a record, array, or map, +is dropped from the output of the [values](../operators/values.md) operator, and is typically handled with the `??` operator. An [error](../types/error.md) or `missing` value is passed through unless a case has its type. diff --git a/book/src/super-sql/expressions/subqueries.md b/book/src/super-sql/expressions/subqueries.md index f30e1550a5..268e8ca6b2 100644 --- a/book/src/super-sql/expressions/subqueries.md +++ b/book/src/super-sql/expressions/subqueries.md @@ -19,6 +19,9 @@ results in the value `{s:"HELLO, WORLD"}`. Except for subqueries appearing as the right-hand side of an [in](containment.md) operator, the result of a subquery must be a single value. When multiple values are generated, an [error](../types/error.md) is produced. +When no values are generated, the result is `none`, which disappears from a +record, array, or map and is dropped from the output of the +[values](../operators/values.md) operator, or, in a SQL expression, `null`. For the [in](containment.md) operator, any subquery on the right-hand side is always treated as an [array subquery](#array-subqueries), thus @@ -278,3 +281,20 @@ values walk(this, &addOne) [2,3,4] [{x:[2,"foo"]},{y:3}] ``` + +--- + +_A subquery that produces no values is none_ + +```mdtest-spq +# spq +values {s:(unnest a | where this > 1)}, (unnest a | where this > 1) ?? 0 +# input +{a:[1,2]} +{a:[1]} +# expected output +{s:2} +2 +{} +0 +``` diff --git a/book/src/super-sql/expressions/type-assertions.md b/book/src/super-sql/expressions/type-assertions.md index a0456b4140..f4b1b2fc01 100644 --- a/book/src/super-sql/expressions/type-assertions.md +++ b/book/src/super-sql/expressions/type-assertions.md @@ -7,7 +7,8 @@ A type assertion has the form where `` is any [expression](intro.md) and `` is any [type](../types/intro.md). Its value is the value of `` when that value has type `` and is otherwise `none`, -which disappears from a record, array, or map and is typically handled with +which disappears from a record, array, or map, is dropped from the output of the +[values](../operators/values.md) operator, and is typically handled with the `??` operator, e.g., ``` v.(string) ?? "default" diff --git a/book/src/super-sql/functions/generics/ok.md b/book/src/super-sql/functions/generics/ok.md index 6dff8bf0cf..6ca4093c8f 100644 --- a/book/src/super-sql/functions/generics/ok.md +++ b/book/src/super-sql/functions/generics/ok.md @@ -16,8 +16,9 @@ unmodified. This function is useful when errors are expected and are to be explicitly managed with `none` operators. The `none` disappears from a record expression, an -array, or a map and otherwise must be handled with `??` since a query may not -output a `none`. +array, or a map and is dropped from the output of the +[values](../../operators/values.md) operator, so `??` is needed only to replace +it with another value. ## Examples diff --git a/book/src/super-sql/functions/records/has.md b/book/src/super-sql/functions/records/has.md new file mode 100644 index 0000000000..d6a92d76e3 --- /dev/null +++ b/book/src/super-sql/functions/records/has.md @@ -0,0 +1,112 @@ +# has + +test whether a record or record type has a field or a map has a key + +## Synopsis + +``` +has(val: any, key: any) -> bool +``` + +## Description + +The `has` function returns true if +* `val` is a record whose type has a field named `key`, +* `val` is a type value of a record type with a field named `key`, or +* `val` is a map with the key `key` + +and false otherwise, e.g., for a value that isn't a record, type value, or +map. + +For a record, the result depends only on its type, not on the value of the +field, so `has` is true for a field whose value is null, an error, or an +option's none. A value of a named type or a union type is tested by its +underlying type. + +For a map, `key` may be of any type and is compared with the map's keys after +removing named and union types, so a key of a different type (e.g., an +`int32` for a map with `int64` keys) is never found. + +An error is returned if `val` is a record and `key` isn't a string. + +## Examples + +--- + +_Test records for fields_ + +```mdtest-spq +# spq +values {a:has(this,"a"),b:has(this,"b")} +# input +{a:1} +{b:null} +{a:error("x"),b:2} +1 +# expected output +{a:true,b:false} +{a:false,b:true} +{a:true,b:true} +{a:false,b:false} +``` + +--- + +_Filter on the presence of a field_ + +```mdtest-spq +# spq +where has(this,"b") +# input +{a:1} +{b:2} +# expected output +{b:2} +``` + +--- + +_Test a record type_ + +```mdtest-spq +# spq +values has(this,"a") +# input +<{a:int64}> +<{b:int64}> +# expected output +true +false +``` + +--- + +_Test a map for a key_ + +```mdtest-spq +# spq +values has(this,"a"), has(this,1) +# input +map{"a":1} +map{1:"x"} +# expected output +true +false +false +true +``` + +--- + +_The field name may be computed_ + +```mdtest-spq +# spq +values has(this, f) +# input +{a:1,f:"a"} +{a:1,f:"b"} +# expected output +true +false +``` diff --git a/book/src/super-sql/operators/aggregate.md b/book/src/super-sql/operators/aggregate.md index 6ef616fc09..aa80bfe586 100644 --- a/book/src/super-sql/operators/aggregate.md +++ b/book/src/super-sql/operators/aggregate.md @@ -55,6 +55,10 @@ The output values are records formed from the [field assignments](intro.md#field-assignment) first from the grouping expressions then from the aggregate functions in left-to-right order. +As in a record expression, a grouping key whose value +is a plain `none` (e.g., `ok(x)` when `x` is missing) disappears from its +output record, so the values whose key is `none` form a group whose record +has no field for that key. When the result of `aggregate` is a single value (e.g., a single aggregate function without grouping expressions or a single grouping expression without aggregates) diff --git a/book/src/super-sql/operators/debug.md b/book/src/super-sql/operators/debug.md index 33a2b3a0b5..f8bde42ad9 100644 --- a/book/src/super-sql/operators/debug.md +++ b/book/src/super-sql/operators/debug.md @@ -16,6 +16,8 @@ each value in the input, evaluates the optional and transmits each result to a _debugging output_. If `` is omitted, then each input value is passed unmodified to the debugging output. +As for the [values](values.md) operator, a result that is a plain `none` +(e.g., from [ok](../functions/generics/ok.md) or `?.`) is dropped. An optional filter may be applied to the values sent to the debugging output, which is specified with Boolean-valued expression ``. diff --git a/book/src/super-sql/operators/unnest.md b/book/src/super-sql/operators/unnest.md index aa06bf872b..acdbf43de5 100644 --- a/book/src/super-sql/operators/unnest.md +++ b/book/src/super-sql/operators/unnest.md @@ -41,7 +41,8 @@ produces ``` The value of `` must be an array (or a set), whose elements form the -derived sequence. +derived sequence. A `null` or a plain `none` value (e.g., from +[ok](../functions/generics/ok.md) or `?.`) produces no values. To carry other values along with each unnested element, give the unnested value a name with `:=` or list the values in a `with` clause. Then each diff --git a/book/src/super-sql/operators/values.md b/book/src/super-sql/operators/values.md index 87d0b073ad..316e99ad6e 100644 --- a/book/src/super-sql/operators/values.md +++ b/book/src/super-sql/operators/values.md @@ -19,6 +19,13 @@ in left-to-right order. Each `` may be any valid The input order convolved with left-to-right evaluation order is preserved at the output. +A result that is a plain `none`, e.g., from [ok](../functions/generics/ok.md), +`?.`, a [type assertion](../expressions/type-assertions.md) that fails, or a +[match](../expressions/match.md) with no matching case and no `else`, +is dropped, so a `none` is never passed to a downstream operator. +To output another value in its place, use the `??` operator. +An option's `none` (e.g., `none::int64`) is a value and is output as is. + The `values` operator name is optional since it can be used as a [shortcut](intro.md#shortcuts). When used as a shortcut, only one expression may be present. @@ -127,3 +134,18 @@ values [a,b],[b,a] | collect(this) # expected output [[1,2],[2,1]] ``` + +--- + +_A plain none is dropped unless it's replaced with ??_ +```mdtest-spq +# spq +values ok(a), ok(a) ?? 0 +# input +{a:1} +{b:2} +# expected output +1 +1 +0 +``` diff --git a/compiler/dag/expr.go b/compiler/dag/expr.go index 07e36f9613..ef701942a3 100644 --- a/compiler/dag/expr.go +++ b/compiler/dag/expr.go @@ -146,7 +146,11 @@ type ( SubqueryExpr struct { Kind string `json:"kind" unpack:""` Correlated bool `json:"correlated"` - Body Seq `json:"body"` + // SQL is true for a subquery in a SQL expression, whose value is + // null when the subquery produces no values. Otherwise, the value + // is none. + SQL bool `json:"sql,omitempty"` + Body Seq `json:"body"` } ThisExpr struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 758f4edb25..7e810d9a1e 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -373,7 +373,7 @@ func NewFilterOp(e Expr) *FilterOp { } func NewValuesOp(exprs ...Expr) *ValuesOp { - return &ValuesOp{"ValuesOp", exprs} + return &ValuesOp{Kind: "ValuesOp", Exprs: exprs} } func CopySeq(seq Seq) Seq { diff --git a/compiler/rungen/vexpr.go b/compiler/rungen/vexpr.go index 6cd542c73f..f3776fc9ed 100644 --- a/compiler/rungen/vexpr.go +++ b/compiler/rungen/vexpr.go @@ -313,11 +313,11 @@ func (b *Builder) compileVamSubquery(query *dag.SubqueryExpr) (expr.Evaluator, e return nil, err } body := b.combineVam(exits) - return op.NewCachedSubquery(b.sctx(), body), nil + return op.NewCachedSubquery(b.sctx(), body, query.SQL), nil } var create func() *op.Subquery create = func() *op.Subquery { - subquery := op.NewSubquery(b.rctx.Context, b.sctx(), func() *op.Subquery { + subquery := op.NewSubquery(b.rctx.Context, b.sctx(), query.SQL, func() *op.Subquery { b.mu.Lock() defer b.mu.Unlock() return create() diff --git a/compiler/rungen/vop.go b/compiler/rungen/vop.go index 2a29189f5d..b17c9ceaf4 100644 --- a/compiler/rungen/vop.go +++ b/compiler/rungen/vop.go @@ -317,7 +317,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vio.Puller) (vio.Puller, error return vio.NewPuller(vector.NewNull(1)), nil case *dag.OutputOp: if parent != nil { - parent = op.NewOutput(b.sctx(), parent) + parent = op.NewOutput(parent) } b.channels[o.Name] = append(b.channels[o.Name], parent) return parent, nil diff --git a/compiler/semantic/analyzer.go b/compiler/semantic/analyzer.go index 5ebd59a897..43a51250bb 100644 --- a/compiler/semantic/analyzer.go +++ b/compiler/semantic/analyzer.go @@ -79,9 +79,6 @@ type translator struct { sctx *super.Context types *sup.Analyzer defs *super.Context - // subqueries is the depth of expression subqueries being translated. - // Their results may be none (see checker.output). - subqueries int } func newTranslator(ctx context.Context, r reporter, env *exec.Environment) *translator { diff --git a/compiler/semantic/checker.go b/compiler/semantic/checker.go index 4755574c66..1d786d1bbe 100644 --- a/compiler/semantic/checker.go +++ b/compiler/semantic/checker.go @@ -156,7 +156,7 @@ func (c *checker) op(typ super.Type, op sem.Op) super.Type { case *sem.UnnestOp: return c.seq(c.unnest(op.Expr, c.expr(typ, op.Expr), op.Field), op.Body) case *sem.ValuesOp: - return c.fuse(c.exprs(typ, op.Exprs)) + return c.values(c.exprs(typ, op.Exprs)) default: panic(op) } @@ -207,6 +207,10 @@ func (c *checker) unnest(loc ast.Node, typ super.Type, field string) super.Type if !ok { c.keepErrs(errs) } + if typ == nil { + // The value is always a plain none, so there are no values. + return c.unknown + } return typ } @@ -214,6 +218,10 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { if isUnknown(typ) { return c.unknown, true } + if typ == super.TypeNone { + // A plain none produces no values. + return nil, true + } switch typ := under(typ).(type) { case *super.TypeError: c.error(loc, errors.New("unnested record cannot be an error")) @@ -224,7 +232,9 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { for _, t := range typ.Types { typ, tok := c.unnestCheck(loc, t) if tok { - types = append(types, typ) + if typ != nil { + types = append(types, typ) + } ok = true } } @@ -252,6 +262,9 @@ func (c *checker) unnestField(loc ast.Node, typ super.Type, field string) (super return c.unknown, true } elemType, ok := c.unnestCheck(loc, rec.Fields[k].Type) + if elemType == nil { + return nil, ok + } fields := slices.Clone(rec.Fields) fields[k].Type = elemType out, err := c.t.sctx.LookupTypeRecord(fields) @@ -441,11 +454,7 @@ func (c *checker) expr(typ super.Type, e sem.Expr) super.Type { c.sliceable(e.Expr, container) return container case *sem.SubqueryExpr: - typ = c.seq(typ, e.Body) - if e.Array { - typ = c.t.sctx.LookupTypeArray(typ) - } - return typ + return c.subquery(e, c.seq(typ, e.Body)) case *sem.ThisExpr: return c.this(e.Node, e, typ) case *sem.TypeExpr: @@ -467,6 +476,22 @@ func (c *checker) expr(typ super.Type, e sem.Expr) super.Type { } } +// subquery returns the type of subquery e whose body's output has type typ. +// A pipe subquery whose body may produce no values is none (see mayBeEmpty). +// (A SQL subquery is null.) +func (c *checker) subquery(e *sem.SubqueryExpr, typ super.Type) super.Type { + if e.Array { + return c.t.sctx.LookupTypeArray(typ) + } + if e.SQL || !mayBeEmpty(e) { + return typ + } + if typ == nil { + return super.TypeNone + } + return c.orNone(typ) +} + func (c *checker) ok(typ super.Type, call *sem.CallExpr) super.Type { if len(call.Args) != 1 { panic(call) @@ -813,6 +838,8 @@ func (c *checker) formRecordElems(elems []sem.RecordElem, types []super.Type) su } continue } + // A field whose value may be none may be absent. + typ = c.noneAsMissing(typ) if k, ok := order[elem.Name]; ok { // As in the runtime, a field of the same name as an earlier // field (e.g., from a spread) replaces it in place. @@ -828,6 +855,38 @@ func (c *checker) formRecordElems(elems []sem.RecordElem, types []super.Type) su return c.t.sctx.MustLookupTypeRecord(fields) } +// noneAsMissing returns typ with plain none replaced by missing if typ is a +// union (or the fusion of a union) with plain none as a member, i.e., the type +// of a field whose value may be none, which disappears from a record, so the +// field may be missing. A named type is a value and is returned as is. +func (c *checker) noneAsMissing(typ super.Type) super.Type { + if typ == nil || hasUnknown(typ) { + return typ + } + u, ok := defuse(typ).(*super.TypeUnion) + if !ok || !slices.Contains(u.Types, super.Type(super.TypeNone)) { + return typ + } + var types []super.Type + for _, t := range u.Types { + if t == super.TypeNone { + t = super.TypeMissing + } + types = append(types, t) + } + out := c.unionOf(super.Flatten(types)) + if _, isFusion := typ.(*super.TypeFusion); isFusion { + if _, isUnion := out.(*super.TypeUnion); isUnion { + return c.t.sctx.MustLookupTypeFusion(out) + } + } + return out +} + +func isPlainNone(typ super.Type) bool { + return defuse(typ) == super.TypeNone +} + // unionOf returns the single type in types or else the union of types. func (c *checker) unionOf(types []super.Type) super.Type { if len(types) == 1 { @@ -1015,7 +1074,13 @@ func (c *checker) putPaths(inType super.Type, puts []pathType) super.Type { return c.unknown } for _, put := range puts { - typ := c.putInto(recType, put.elems, put.typ) + // A field whose value is a plain none disappears from the record + // (e.g., an aggregate's group key), so it may be missing. + leaf := c.noneAsMissing(put.typ) + if isPlainNone(leaf) { + leaf = super.TypeMissing + } + typ := c.putInto(recType, put.elems, leaf) var ok bool recType, ok = typ.(*super.TypeRecord) if !ok { @@ -1739,23 +1804,44 @@ func (c *checker) result(loc ast.Node, typ super.Type) { } } -// output reports an error at loc if typ, the type of an output value, may -// be a plain none. (An option none or a none of a named type is a value -// and is allowed.) -func (c *checker) output(loc ast.Node, typ super.Type) { - typ = defuse(typ) - if typ != nil && !hasUnknown(typ) && hasPlainNone(typ) { - c.error(loc, errNone) +// values returns the type of the output of a values operator whose +// expressions have types. The operator drops plain nones, so none is removed +// from each type and an expression that's always a plain none produces no +// values. (An option none or a none of a named type is a value and is kept.) +func (c *checker) values(types []super.Type) super.Type { + var out []super.Type + for _, typ := range types { + if typ, ok := c.withoutPlainNone(typ); ok { + out = append(out, typ) + } } + return c.fuse(out) } -// hasPlainNone reports whether typ is none or a union with none as one of -// its types. Unlike hasNone, a named type whose type is none isn't none. -func hasPlainNone(typ super.Type) bool { - if u, ok := typ.(*super.TypeUnion); ok { - return slices.ContainsFunc(u.Types, hasPlainNone) +// withoutPlainNone returns typ with plain none removed from it (from its +// members if it's a union). It returns false if nothing is left. +func (c *checker) withoutPlainNone(typ super.Type) (super.Type, bool) { + if typ == nil || hasUnknown(typ) { + return typ, true + } + if _, ok := defuse(typ).(*super.TypeNamed); ok { + // A none of a named type (including a named union) is a value. + return typ, true + } + fusion, isFusion := typ.(*super.TypeFusion) + out, ok := c.withoutTypes(defuse(typ), func(typ super.Type) bool { + return typ == super.TypeNone + }) + if !ok { + return nil, false + } + if isFusion && out != fusion.Type { + if _, isUnion := out.(*super.TypeUnion); isUnion { + // The values are still those of a fusion. + return c.t.sctx.MustLookupTypeFusion(out), true + } } - return typ == super.TypeNone + return out, true } // notMissing reports an error at loc if typ may be missing. diff --git a/compiler/semantic/compare.go b/compiler/semantic/compare.go index 16c51081cd..add8cdde91 100644 --- a/compiler/semantic/compare.go +++ b/compiler/semantic/compare.go @@ -281,7 +281,7 @@ func eqExpr(aexpr, bexpr sem.Expr) bool { return ok && eqExpr(a.Expr, b.Expr) && eqExpr(a.From, b.From) && eqExpr(a.To, b.To) case *sem.SubqueryExpr: b, ok := bexpr.(*sem.SubqueryExpr) - return ok && a.Correlated == b.Correlated && a.Array == b.Array && eqSeq(a.Body, b.Body) + return ok && a.Correlated == b.Correlated && a.Array == b.Array && a.SQL == b.SQL && eqSeq(a.Body, b.Body) case *sem.ThisExpr: b, ok := bexpr.(*sem.ThisExpr) if !ok || len(a.Chain) != len(b.Chain) { diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index 2d946e57c4..ff561d8eeb 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -590,6 +590,7 @@ func (d *dagen) subquery(e *sem.SubqueryExpr) *dag.SubqueryExpr { subquery := &dag.SubqueryExpr{ Kind: "SubqueryExpr", Correlated: e.Correlated, + SQL: e.SQL, Body: d.seq(e.Body), } if e.Array { diff --git a/compiler/semantic/expr.go b/compiler/semantic/expr.go index c33736e766..2822311fb9 100644 --- a/compiler/semantic/expr.go +++ b/compiler/semantic/expr.go @@ -735,11 +735,13 @@ func (t *translator) semCall(call *ast.CallExpr, inType super.Type) (sem.Expr, s func (t *translator) maybeSubquery(n ast.Node, name string, inType super.Type) (*sem.SubqueryExpr, super.Type) { if seq, typ := t.scope.lookupQuery(t, name); seq != nil { - return &sem.SubqueryExpr{ + e := &sem.SubqueryExpr{ Node: n, Correlated: isCorrelated(seq), + SQL: t.scope.sql != nil, Body: seq, - }, typ + } + return e, t.checker.subquery(e, typ) } return nil, nil } @@ -888,12 +890,14 @@ func (t *translator) maybeSubqueryCall(call *ast.CallExpr, name string, inType s // (i.e., starts with a from), in which case we put the uncorrelated body // in the subquery without the correlating values logic. This is controlled // via the Correlated flag. - return &sem.SubqueryExpr{ + e := &sem.SubqueryExpr{ Node: call, Array: false, Correlated: isCorrelated(userOp), + SQL: t.scope.sql != nil, Body: userOp, - }, typ + } + return e, t.checker.subquery(e, typ) } func (t *translator) semMapCall(call *ast.CallExpr, args []sem.Expr, argTypes []super.Type) (sem.Expr, super.Type) { @@ -1341,14 +1345,13 @@ func (t *translator) arraySubquery(elems []sem.ArrayElem) *sem.SubqueryExpr { func (t *translator) subqueryExpr(astExpr ast.Expr, array bool, body ast.Seq, inType super.Type) (*sem.SubqueryExpr, super.Type) { // We pass inType in whether or not it's correlated since an uncorrelated // subquery will just ignore it. - t.subqueries++ seq, outType := t.seq(body, inType) - t.subqueries-- correlated := isCorrelated(seq) e := &sem.SubqueryExpr{ Node: astExpr, Array: array, Correlated: correlated, + SQL: t.scope.sql != nil, Body: seq, } // Add a nullscan only for uncorrelated queries that don't have a source. @@ -1360,10 +1363,7 @@ func (t *translator) subqueryExpr(astExpr ast.Expr, array bool, body ast.Seq, in // condition with this complex cleanup... e.Body, outType = t.scalarSubqueryCheck(astExpr, e.Body, outType) } - if array { - outType = t.sctx.LookupTypeArray(outType) - } - return e, outType + return e, t.checker.subquery(e, outType) } func (t *translator) scalarSubqueryCheck(n ast.Node, seq sem.Seq, thisType super.Type) (sem.Seq, super.Type) { diff --git a/compiler/semantic/nonempty.go b/compiler/semantic/nonempty.go new file mode 100644 index 0000000000..3fcc79872b --- /dev/null +++ b/compiler/semantic/nonempty.go @@ -0,0 +1,71 @@ +package semantic + +import ( + "slices" + + "github.com/superdb/super/compiler/semantic/sem" +) + +// mayBeEmpty returns false if the body of subquery e always produces a value, +// e.g., (values a) or (sum(x)), and true if it might not, in which case the +// value of a pipe subquery is none. It's conservative: a body that's not +// known to produce a value may be empty. +func mayBeEmpty(e *sem.SubqueryExpr) bool { + // A correlated subquery runs on the value of its outer expression. + nonEmpty := e.Correlated + for _, op := range e.Body { + nonEmpty = producesValue(op, nonEmpty) + } + return !nonEmpty +} + +// producesValue returns true if op is known to produce a value when its input +// is nonEmpty. +func producesValue(op sem.Op, nonEmpty bool) bool { + switch op := op.(type) { + case *sem.NullScan: + return true + case *sem.AggregateOp: + // An aggregate without keys produces one value even with no + // input, and with keys, a value per group. + return len(op.Keys) == 0 || nonEmpty + case *sem.ValuesOp: + // A value that can't be a plain none isn't dropped. + return nonEmpty && slices.ContainsFunc(op.Exprs, notNone) + case *sem.HeadOp: + return nonEmpty && op.Count > 0 + case *sem.TailOp: + return nonEmpty && op.Count > 0 + case *sem.CountOp, *sem.CutOp, *sem.DebugOp, *sem.DistinctOp, *sem.DropOp, + *sem.FuseOp, *sem.PassOp, *sem.PutOp, *sem.RenameOp, *sem.SortOp, + *sem.UniqOp: + // These produce a value (possibly an error) for some input. + return nonEmpty + } + return false +} + +// notNone returns true if e is known not to be a plain none. +func notNone(e sem.Expr) bool { + switch e := e.(type) { + case *sem.PrimitiveExpr: + return e.Value != "none" + case *sem.ArrayExpr, *sem.MapExpr, *sem.RecordExpr, *sem.SetExpr: + return true + case *sem.DotExpr: + return !e.Noneish && notNone(e.LHS) + case *sem.IndexExpr: + // An index out of range is an error. + return notNone(e.Expr) + case *sem.ThisExpr: + // A field of a record and a value input to an operator are + // never plain nones. + for _, elem := range e.Chain { + if elem.Noneish { + return false + } + } + return true + } + return false +} diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index 85c390c196..0d153a0b62 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -1171,9 +1171,8 @@ func (t *translator) semOp(o ast.Op, seq sem.Seq, inType super.Type) (sem.Seq, s exprs, types := t.exprs(o.Exprs, inType) for k, typ := range types { t.checker.result(o.Exprs[k], typ) - t.output(o.Exprs[k], typ) } - return append(seq, sem.NewValues(o, exprs...)), t.checker.fuse(types) + return append(seq, sem.NewValues(o, exprs...)), t.checker.values(types) case *ast.Script: t.error(o, errors.New("a script of multiple statements cannot be used here")) return append(seq, badOp), badType @@ -1380,13 +1379,24 @@ func (t *translator) singletonKey(agg ast.Assignment, seq sem.Seq, inType super. if !ok || len(this.Chain) != 1 { return nil, nil } + // A plain none key disappears from the aggregate's record (see the + // aggregate operator), so the key's value is none when the record + // doesn't have its field, and values drops it. (ok() would turn an error + // key into none too.) + name := &sem.PrimitiveExpr{Node: out.Node, Value: sup.QuotedString(this.Chain[0].ID)} + key := &sem.CondExpr{ + Node: out.Node, + Cond: sem.NewCall(out.Node, "has", []sem.Expr{sem.NewThis(out.Node, nil), name}), + Then: this, + Else: &sem.PrimitiveExpr{Node: out.Node, Value: "none"}, + } return append(seq, &sem.AggregateOp{ Node: out.Node, Keys: []sem.Assignment{out}, }, - sem.NewValues(out.Node, this), - ), path.typ + sem.NewValues(out.Node, key), + ), t.checker.values([]super.Type{path.typ}) } // semDecls enters a block of declarations into the current scope. We do late binding @@ -1553,6 +1563,10 @@ func (t *translator) typeQueryDecl(d *ast.TypeQueryDecl) { t.error(d, fmt.Errorf("type query for %q: %s", d.Name.Name, sup.FormatValue(val))) return } + if val.Type() == super.TypeNone { + t.error(d, fmt.Errorf("type query for %q must yield a single type value but yielded no values", d.Name.Name)) + return + } if val.Type() != super.TypeType || val.IsNull() { t.error(d, fmt.Errorf("type query for %q must yield a single type value: %s", d.Name.Name, sup.FormatValue(val))) return @@ -1709,10 +1723,10 @@ func (t *translator) exprOp(e ast.Expr, seq sem.Seq, inType super.Type) (sem.Seq if t.isBool(out) { return append(seq, &sem.FilterOp{Node: e, Expr: out}), t.checker.narrow(inType, out) } - // As with the values operator, the value must not be missing or none. + // As with the values operator, the value must not be missing and a + // plain none is dropped. t.checker.result(e, typ) - t.output(e, typ) - return append(seq, sem.NewValues(e, out)), typ + return append(seq, sem.NewValues(e, out)), t.checker.values([]super.Type{typ}) } func (t *translator) isBool(e sem.Expr) bool { @@ -2022,21 +2036,6 @@ func (t *translator) maybeEval(e sem.Expr) (super.Value, bool) { return newEvaluator(t, t.resolver.funcs).maybeEval(t.sctx, e) } -// output reports an error at loc if typ, the type of a value output by the -// values operator, may be a plain none (e.g., from ok() or ?.). A plain none -// disappears from a record, array, or map but otherwise must be handled with -// ?? before it leaves the operator that produced it, i.e., a plain none is -// never passed to a downstream operator, even one that would handle it. -// (A plain none that reaches the query output at runtime is an error.) The -// result of an expression subquery may be none since it's an expression -// value. -func (t *translator) output(loc ast.Node, typ super.Type) { - if t.subqueries > 0 { - return - } - t.checker.output(loc, typ) -} - // unnestExpr translates the expression of an unnest operator and returns it, // its type, and, if the unnest is named ("unnest name:=expr") or has a with // clause ("with assignments"), the name of the field to unnest. In that diff --git a/compiler/semantic/sem/expr.go b/compiler/semantic/sem/expr.go index 5632ef5379..ce66492f05 100644 --- a/compiler/semantic/sem/expr.go +++ b/compiler/semantic/sem/expr.go @@ -133,7 +133,10 @@ type ( ast.Node Correlated bool Array bool - Body Seq + // SQL is true for a subquery in a SQL expression (see + // dag.SubqueryExpr). + SQL bool + Body Seq } ThisExpr struct { ast.Node @@ -462,6 +465,7 @@ func CopyExpr(e Expr) Expr { Node: e.Node, Correlated: e.Correlated, Array: e.Array, + SQL: e.SQL, Body: CopySeq(e.Body), } case *ThisExpr: diff --git a/compiler/semantic/ztests/agg-missing.yaml b/compiler/semantic/ztests/agg-missing.yaml index ac8e642f35..9dae20952b 100644 --- a/compiler/semantic/ztests/agg-missing.yaml +++ b/compiler/semantic/ztests/agg-missing.yaml @@ -86,7 +86,8 @@ error: | --- -# With ok(), the absent values form the none group. +# With ok(), the absent values form the none group, whose record has no y +# since a plain none disappears from a record. spq: aggregate sum(x) by y.ok() | sort y input: | @@ -96,7 +97,7 @@ input: | output: | {y:"a",sum:4} - {y:none,sum:2} + {sum:2} --- diff --git a/compiler/semantic/ztests/imdb.yaml b/compiler/semantic/ztests/imdb.yaml index 42ae636fd7..770ba72e19 100644 --- a/compiler/semantic/ztests/imdb.yaml +++ b/compiler/semantic/ztests/imdb.yaml @@ -14,10 +14,10 @@ inputs: | values {...p,name:name.primaryName} ) values { - title: title.primaryTitle, + title: title?.primaryTitle, director: (people 'director' | collect(name)), writer: (people 'writer' | collect(name)), - genres: title.genres, + genres: title?.genres, actors: (people 'actor' | order by ordering | limit 10 diff --git a/compiler/semantic/ztests/none-absent.yaml b/compiler/semantic/ztests/none-absent.yaml new file mode 100644 index 0000000000..d9f23176f1 --- /dev/null +++ b/compiler/semantic/ztests/none-absent.yaml @@ -0,0 +1,36 @@ +# Since plain nones are dropped, the type checker must still catch a none that +# can reach an expression that doesn't handle it: +# - a pipe subquery that may produce no values is none, +# - a record field or aggregate key whose value may be none may be absent, +# - and a none of a named type is a value that's kept. +script: | + echo === subquery + ! super -s -c 'values {a:1},{b:1} | values {c:(values a.ok())} | values c+1' 2> /dev/null + ! super -s -c 'values 1 | values (values 2 | where false) + 1' 2> /dev/null + super -s -c 'values 1 | values (values 2) + 1, (values this | sum(this)) + 1, (values [this][0]) + 1' + super -s -c 'values 1 | values ((values 2 | where false) ?? 0) + 1' + echo === absent + ! super -s -c 'values {a:1},{b:2} | count() by k:=a.ok() | values k' 2> /dev/null + super -s -c 'values {a:1},{b:2} | count() by k:=a.ok() | values ok(k) ?? 0 | sort this' + ! super -s -c 'values {a:1},{b:2} | values {k:a.ok()} | values k' 2> /dev/null + super -s -c 'values {a:1},{b:2} | values {k:a.ok()} | values ok(k)' + echo === named + ! super -s -c 'type u=int64|none values {a:1},{b:2} | values ok(a)::u | values this+1' 2> /dev/null + super -s -c 'type u=int64|none values {a:1},{b:2} | values ok(a)::u' + +outputs: + - name: stdout + data: | + === subquery + 3 + 2 + 2 + 1 + === absent + 0 + 1 + 1 + === named + type u=int64|none + 1::u + none::u diff --git a/compiler/semantic/ztests/none-output.yaml b/compiler/semantic/ztests/none-output.yaml index 4c3b0ac9ae..47d7d6c018 100644 --- a/compiler/semantic/ztests/none-output.yaml +++ b/compiler/semantic/ztests/none-output.yaml @@ -22,12 +22,20 @@ output: | --- -# Statically, a value that may be a plain none (e.g., from ?. or ok()) can't -# be output and must be handled with ??. +# A plain none (e.g., from ?. or ok()) output by the values operator is +# dropped, so it isn't passed downstream, unless it's handled +# with ??. script: | super -o in.bsup in.sup - ! super -s -c 'from in.bsup | values org?.id' - ! super -s -c 'from in.bsup | values y.ok()' + echo === + super -s -c 'from in.bsup | values org?.id' + echo === + super -s -c 'from in.bsup | values y.ok()' + echo === + super -s -c 'from in.bsup | values y.ok() | values typeof(this)' + echo === + super -s -c 'from in.bsup | y.ok()' + echo === super -s -c 'from in.bsup | values org?.id ?? 0, {y:y.ok()}' inputs: @@ -39,22 +47,23 @@ inputs: outputs: - name: stdout data: | + === + 1 + === + 2 + === + + === + 2 + === 0 {} 1 {y:2} - - name: stderr - data: | - none not handled (consider ?? or ?.) at line 1, column 23: - from in.bsup | values org?.id - ~~~~~~~ - none not handled (consider ?? or ?.) at line 1, column 23: - from in.bsup | values y.ok() - ~~~~~~ --- -# At runtime, a plain none output is an error. +# At runtime, a plain none output is dropped. spq: values org?.id flags: -dynamic @@ -64,18 +73,15 @@ input: | {org?:{id:1}} output: | - error("none not handled (consider ?? or ?.)") 1 --- -# A none literal can't be output. -spq: values none +# A none literal is a plain none, so it's dropped. +spq: values none, 1 -error: | - none not handled (consider ?? or ?.) at line 1, column 8: - values none - ~~~~ +output: | + 1 --- @@ -111,8 +117,8 @@ output: | --- -# At runtime, a union whose type has a none member keeps its type for values -# that aren't plain nones. (Statically, the type may be none so it's an error.) +# A union whose type has a none member keeps its type for values that +# aren't plain nones. spq: values this flags: -dynamic diff --git a/compiler/semantic/ztests/none.yaml b/compiler/semantic/ztests/none.yaml index d6d20e081a..ea9c184b3b 100644 --- a/compiler/semantic/ztests/none.yaml +++ b/compiler/semantic/ztests/none.yaml @@ -144,7 +144,7 @@ output: | --- -# The runtime ?. passes the none from ok() through, which is an error when +# The runtime ?. passes the none from ok() through, which is dropped when # it's output. spq: values r.ok()?.f @@ -156,7 +156,6 @@ input: | output: | 1 - error("none not handled (consider ?? or ?.)") --- diff --git a/runtime/expr/function/function.go b/runtime/expr/function/function.go index 0f0575e94a..552d9ac8e5 100644 --- a/runtime/expr/function/function.go +++ b/runtime/expr/function/function.go @@ -80,6 +80,10 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { case "grok": argmin, argmax = 2, 3 f = newGrok(sctx) + case "has": + argmin = 2 + argmax = 2 + f = &Has{sctx: sctx} case "has_error": f = HasError{sctx} case "hex": @@ -262,7 +266,7 @@ func (f *samFunc) Call(args ...vector.Any) vector.Any { // signatures so the return type can be introspected. func HasBoolResult(name string) bool { switch name { - case "grep", "has_error", "is", "is_error", "is_ok", "missing", "cidr_match": + case "grep", "has", "has_error", "is", "is_error", "is_ok", "missing", "cidr_match": return true } return false diff --git a/runtime/expr/function/has.go b/runtime/expr/function/has.go new file mode 100644 index 0000000000..b62d61aba9 --- /dev/null +++ b/runtime/expr/function/has.go @@ -0,0 +1,93 @@ +package function + +import ( + "bytes" + + "github.com/superdb/super" + "github.com/superdb/super/scode" + "github.com/superdb/super/vector" + "github.com/superdb/super/vector/bitvec" +) + +// Has implements has(val, key), which is true if +// - val is a record whose type has a field named key, +// - val is a type value of a record type with a field named key, or +// - val is a map with the key key. +// +// For a record, the answer depends only on its type, so it's true for a +// field whose value is null, an error, or an option's none. (A plain none +// never appears in a record.) It's false for any other value. +type Has struct { + sctx *super.Context + mb scode.Builder + kb scode.Builder +} + +func (h *Has) Call(args ...vector.Any) vector.Any { + val, key := args[0], vector.Under(args[1]) + if key.Kind() == vector.KindError { + return key + } + n := val.Len() + switch typ := super.TypeUnder(val.Type()).(type) { + case *super.TypeRecord: + if err := h.checkFieldName(key); err != nil { + return err + } + if c, ok := key.(*vector.Const); ok { + return vector.NewConstBool(typ.HasField(vector.StringValue(c, 0)), n) + } + return h.eachSlot(n, func(slot uint32) bool { + return typ.HasField(vector.StringValue(key, slot)) + }) + case *super.TypeMap: + return h.eachSlot(n, func(slot uint32) bool { + return h.hasMapKey(typ, val, key, slot) + }) + } + if val.Kind() == vector.KindType && key.Kind() == vector.KindString { + // A type value is tested by its record type. Any other type + // value, or a key that isn't a string, is false. + return h.eachSlot(n, func(slot uint32) bool { + rec, ok := super.TypeUnder(vector.TypeValueValue(val, slot)).(*super.TypeRecord) + return ok && rec.HasField(vector.StringValue(key, slot)) + }) + } + return vector.NewConstBool(false, n) +} + +func (h *Has) checkFieldName(key vector.Any) vector.Any { + if key.Kind() != vector.KindString { + return vector.NewWrappedError(h.sctx, "has: field name must be a string", key) + } + return nil +} + +func (h *Has) eachSlot(n uint32, fn func(uint32) bool) vector.Any { + bits := bitvec.NewFalse(n) + for slot := range n { + if fn(slot) { + bits.Set(slot) + } + } + return vector.NewBool(bits) +} + +// hasMapKey returns true if the map at slot of vec has the key at slot of +// key. Keys are compared by value after removing named and union types. +func (h *Has) hasMapKey(typ *super.TypeMap, vec, key vector.Any, slot uint32) bool { + m := vector.ValueAt(&h.mb, vec, slot) + if m.IsNull() { + return false + } + body := m.Bytes() + k := vector.ValueAt(&h.kb, key, slot).Under() + for it := body.Iter(); !it.Done(); { + mk := super.NewValue(typ.KeyType, it.Next()).Under() + it.Next() + if mk.Type() == k.Type() && bytes.Equal(mk.Bytes(), k.Bytes()) { + return true + } + } + return false +} diff --git a/runtime/op/aggregate/aggregate.go b/runtime/op/aggregate/aggregate.go index b85009f8c7..3f00d1b725 100644 --- a/runtime/op/aggregate/aggregate.go +++ b/runtime/op/aggregate/aggregate.go @@ -32,6 +32,12 @@ func New(parent vio.Puller, sctx *super.Context, aggNames []field.Path, aggExprs if err != nil { return nil, err } + // A plain none group key (e.g., from ok() or ?.) disappears from an + // output record. It's kept in a partial so the downstream aggregate can + // group by it. + if !partialsOut { + builder.NoneDrops = len(keyNames) + } return &Aggregate{ parent: parent, sctx: sctx, diff --git a/runtime/op/cachedsubquery.go b/runtime/op/cachedsubquery.go index cfbcd1b67f..f47ce59ece 100644 --- a/runtime/op/cachedsubquery.go +++ b/runtime/op/cachedsubquery.go @@ -11,11 +11,15 @@ import ( type cachedSubquery struct { sctx *super.Context body vio.Puller + sql bool cached vector.Any } -func NewCachedSubquery(sctx *super.Context, body vio.Puller) expr.Evaluator { - return &cachedSubquery{sctx: sctx, body: body} +// NewCachedSubquery returns an uncorrelated subquery, which is evaluated once. +// If it produces no values, its value is null if sql is true (as in SQL) and +// otherwise none. +func NewCachedSubquery(sctx *super.Context, body vio.Puller, sql bool) expr.Evaluator { + return &cachedSubquery{sctx: sctx, body: body, sql: sql} } func (c *cachedSubquery) Eval(this vector.Any) vector.Any { @@ -40,7 +44,7 @@ func (c *cachedSubquery) exec() vector.Any { vecs = append(vecs, vec) } if len(vecs) == 0 { - return vector.NewNull(1) + return emptyResult(c.sql) } if len(vecs) == 1 && vecs[0].Len() == 1 { return vecs[0] diff --git a/runtime/op/debug.go b/runtime/op/debug.go index b5741089f5..c159b7e4a9 100644 --- a/runtime/op/debug.go +++ b/runtime/op/debug.go @@ -56,7 +56,8 @@ func (d *Debug) Pull(done bool) (vector.Any, error) { if d.filter != nil { filtered, _ = applyMask(val, d.filter.Eval(filtered)) } - if e := d.expr.Eval(filtered); e.Len() != 0 { + // A plain none is dropped, as for the values operator. + if e := dropPlainNones(d.expr.Eval(filtered)); e != nil && e.Len() != 0 { select { case local <- e: case <-d.ctx.Done(): diff --git a/runtime/op/nones.go b/runtime/op/nones.go new file mode 100644 index 0000000000..bea5510b6f --- /dev/null +++ b/runtime/op/nones.go @@ -0,0 +1,142 @@ +package op + +import ( + "slices" + + "github.com/superdb/super" + "github.com/superdb/super/vector" +) + +// dropPlainNones returns vec without its plain none values or nil if all of +// its values are plain nones. A plain none comes from an expression like +// ok() or ?. that handles a none. It disappears from a record, array, or +// map and, as a value output by the values operator or the query, is +// dropped. A none of a named type and an option's none are values and +// are kept. +func dropPlainNones(vec vector.Any) vector.Any { + nones := plainNoneSlots(vec) + if len(nones) == 0 { + return vec + } + n := vec.Len() + if len(nones) == int(n) { + return nil + } + rest := make([]uint32, 0, int(n)-len(nones)) + var k int + for slot := range n { + if k < len(nones) && nones[k] == slot { + k++ + continue + } + rest = append(rest, slot) + } + return vector.Pick(vec, rest) +} + +// plainNoneSlots returns the slots of vec, in order, whose values are plain +// nones. +func plainNoneSlots(vec vector.Any) []uint32 { + switch v := vec.(type) { + case *vector.Dynamic: + // The values of a dynamic may themselves be dynamics. + marks := make([][]bool, len(v.Values)) + var found bool + for k, val := range v.Values { + if val == nil { + continue + } + if slots := plainNoneSlots(val); len(slots) != 0 { + marks[k] = markSlots(val.Len(), slots) + found = true + } + } + if !found { + return nil + } + var nones []uint32 + counts := make([]uint32, len(v.Values)) + for slot, tag := range v.Tags { + if m := marks[tag]; m != nil && m[counts[tag]] { + nones = append(nones, uint32(slot)) + } + counts[tag]++ + } + return nones + case *vector.Union: + // Check the type first so the tags of a union without a none + // member aren't loaded. + if !hasPlainNoneMember(v.Typ) { + return nil + } + tags := v.Tags() + values := v.Values() + var nones []uint32 + for slot, tag := range tags { + if isNoneVec(values[tag]) { + nones = append(nones, uint32(slot)) + } + } + return nones + case *vector.Fusion: + // A fusion value is a plain none if its subtype is none, which is + // possible only if the fusion's type has none. Check the type first + // so the subtypes needn't be loaded. + if typ, ok := v.Typ.Type.(*super.TypeUnion); !ok || !hasPlainNoneMember(typ) { + if v.Typ.Type != super.TypeNone { + return nil + } + } + var nones []uint32 + for slot, typ := range v.Subtypes.Types() { + if typ == super.TypeNone { + nones = append(nones, uint32(slot)) + } + } + return nones + case *vector.View: + slots := plainNoneSlots(v.Any) + if len(slots) == 0 { + return nil + } + marks := markSlots(v.Any.Len(), slots) + var nones []uint32 + for slot, index := range v.Index { + if marks[index] { + nones = append(nones, uint32(slot)) + } + } + return nones + } + if isNoneVec(vec) { + return allSlots(vec.Len()) + } + return nil +} + +// hasPlainNoneMember reports whether typ has plain none as a member. +func hasPlainNoneMember(typ *super.TypeUnion) bool { + return slices.ContainsFunc(typ.Types, func(t super.Type) bool { return t == super.TypeNone }) +} + +func allSlots(n uint32) []uint32 { + slots := make([]uint32, n) + for k := range slots { + slots[k] = uint32(k) + } + return slots +} + +func markSlots(n uint32, slots []uint32) []bool { + marks := make([]bool, n) + for _, slot := range slots { + marks[slot] = true + } + return marks +} + +// isNoneVec reports whether vec is a plain none. (A none of a named type +// is a value.) +func isNoneVec(vec vector.Any) bool { + return vec != nil && vec.Type() == super.TypeNone +} diff --git a/runtime/op/output.go b/runtime/op/output.go index 45094835c6..647c95269c 100644 --- a/runtime/op/output.go +++ b/runtime/op/output.go @@ -1,109 +1,30 @@ package op import ( - "slices" - - "github.com/superdb/super" "github.com/superdb/super/vector" "github.com/superdb/super/vector/vio" ) -// Output is the puller for an output of a query. It turns any plain none -// values into errors. A plain none comes from an operator like ok() or ?. -// that handles a none so it can disappear from a record, array, or map, but -// a query must not output one (it should be handled with ??). A none of a -// named type is a value and is output as is. +// Output is the puller for an output of a query. It drops any plain none +// values, which come from an expression like ok() or ?. that handles a none +// (see dropPlainNones). A none of a named type is a value and is output +// as is. type Output struct { - sctx *super.Context parent vio.Puller } -func NewOutput(sctx *super.Context, parent vio.Puller) *Output { - return &Output{sctx: sctx, parent: parent} +func NewOutput(parent vio.Puller) *Output { + return &Output{parent: parent} } func (o *Output) Pull(done bool) (vector.Any, error) { - vec, err := o.parent.Pull(done) - if vec == nil || err != nil { - return vec, err - } - return o.noneToError(vec), nil -} - -const noneErr = "none not handled (consider ?? or ?.)" - -func (o *Output) noneToError(vec vector.Any) vector.Any { - switch v := vec.(type) { - case *vector.None: - return vector.NewStringError(o.sctx, noneErr, v.Len()) - case *vector.Dynamic: - // The values of a dynamic may themselves be dynamics. - var vals []vector.Any - for k, val := range v.Values { - if val == nil { - continue - } - if out := o.noneToError(val); out != val { - if vals == nil { - vals = slices.Clone(v.Values) - } - vals[k] = out - } - } - if vals == nil { - return vec - } - return vector.NewDynamic(v.Tags, vals) - case *vector.Union: - // Only the slots that are plain nones become errors. The rest - // keep the union type. - if !slices.ContainsFunc(v.Values(), isNoneVec) { - return vec - } - values := v.Values() - var index, rest []uint32 - for i, tag := range v.Tags() { - if isNoneVec(values[tag]) { - index = append(index, uint32(i)) - } else { - rest = append(rest, uint32(i)) - } + for { + vec, err := o.parent.Pull(done) + if vec == nil || err != nil { + return vec, err } - return o.slotsToError(vec, index, rest) - case *vector.Fusion: - // A fusion value is a plain none if its subtype is none. - var index, rest []uint32 - for i, typ := range v.Subtypes.Types() { - if typ == super.TypeNone { - index = append(index, uint32(i)) - } else { - rest = append(rest, uint32(i)) - } + if vec = dropPlainNones(vec); vec != nil { + return vec, nil } - return o.slotsToError(vec, index, rest) - case *vector.View: - if isNoneVec(v.Any) { - return vector.NewStringError(o.sctx, noneErr, v.Len()) - } - } - return vec -} - -// slotsToError returns vec with the slots in index replaced by errors and the -// slots in rest left as is. -func (o *Output) slotsToError(vec vector.Any, index, rest []uint32) vector.Any { - if len(index) == 0 { - return vec } - errs := vector.NewStringError(o.sctx, noneErr, uint32(len(index))) - if len(rest) == 0 { - return errs - } - return vector.Combine(vector.Pick(vec, rest), index, errs) -} - -// isNoneVec reports whether vec is a plain none. (A none of a named type -// is a value.) -func isNoneVec(vec vector.Any) bool { - return vec != nil && vec.Type() == super.TypeNone } diff --git a/runtime/op/subquery.go b/runtime/op/subquery.go index a04fe279c3..2027bb1863 100644 --- a/runtime/op/subquery.go +++ b/runtime/op/subquery.go @@ -14,10 +14,12 @@ import ( // the expression. When Eval is called, it repeatedly constructs a single-slot // vector from the current slot of the this vector, sends it to the embedded // query, pulls from the query until EOS, appends the value to the result -// vector, and then advances to the next slot. +// vector, and then advances to the next slot. If the query produces no +// values, the result is null if sql is true (as in SQL) and otherwise none. type Subquery struct { ctx context.Context sctx *super.Context + sql bool create func() *Subquery body vio.Puller @@ -28,10 +30,11 @@ type Subquery struct { tos int } -func NewSubquery(ctx context.Context, sctx *super.Context, create func() *Subquery) *Subquery { +func NewSubquery(ctx context.Context, sctx *super.Context, sql bool, create func() *Subquery) *Subquery { return &Subquery{ ctx: ctx, sctx: sctx, + sql: sql, create: create, ch: make(chan vector.Any, 1), tos: -2, @@ -83,7 +86,7 @@ func (s *Subquery) bodyPull() vector.Any { return vector.NewStringError(s.sctx, err.Error(), 1) } if vec == nil { - return vector.NewNull(1) + return emptyResult(s.sql) } vec2, err := s.body.Pull(false) if err != nil { @@ -113,3 +116,12 @@ func (s *Subquery) Pull(done bool) (vector.Any, error) { return nil, s.ctx.Err() } } + +// emptyResult returns the value of a subquery that produces no values: null +// in a SQL expression and otherwise none. +func emptyResult(sql bool) vector.Any { + if sql { + return vector.NewNull(1) + } + return vector.NewNone(1) +} diff --git a/runtime/op/unnest.go b/runtime/op/unnest.go index 568f230f87..861da24e88 100644 --- a/runtime/op/unnest.go +++ b/runtime/op/unnest.go @@ -1,8 +1,6 @@ package op import ( - "slices" - "github.com/superdb/super" "github.com/superdb/super/runtime/expr" "github.com/superdb/super/vector" @@ -41,7 +39,11 @@ func (u *Unnest) Pull(done bool) (vector.Any, error) { vec = vector.Apply(vector.ApplyRipUnions, func(vecs ...vector.Any) vector.Any { return vecs[0] }, u.defuse.Eval(u.expr.Eval(vec))) - if vec, _ = u.flatten(vec, u.field != ""); vec != nil { + // An unnested value may be a plain none (e.g., an element of an + // array of type [int64|none]), which is dropped as for the values + // operator. + vec, _ = u.flatten(vec, u.field != "") + if vec = dropPlainNones(vec); vec != nil { return vec, nil } } @@ -50,12 +52,17 @@ func (u *Unnest) Pull(done bool) (vector.Any, error) { // flatten unnests vec, which is a record whose field named u.field is // unnested if record is true and otherwise an array or set, and returns the // unnested values and the offsets of the values of each slot of vec. It -// returns nil values for nulls and nil offsets for errors, which map -// one-to-one to the slots of vec. +// returns nil values for nulls and plain nones, which produce no values, and +// nil offsets for errors, which map one-to-one to the slots of vec. func (u *Unnest) flatten(vec vector.Any, record bool) (vector.Any, []uint32) { if dynamic, ok := vec.(*vector.Dynamic); ok { return u.flattenDynamic(dynamic, record) } + if vec.Len() == 0 || isNoneVec(vec) { + // An empty vector (e.g., a member of a union with no values) + // has nothing to unnest. + return nil, nil + } vec = vector.Under(vec) switch vec.Kind() { case vector.KindNull: @@ -90,6 +97,8 @@ func (u *Unnest) flattenDynamic(vec *vector.Dynamic, record bool) (vector.Any, [ var tags []uint32 for _, tag := range vec.Tags { if vecs[tag] == nil { + // A null or plain none produces no values. + offsets = append(offsets, length) continue } if offsets := vecOffsets[tag]; offsets != nil { @@ -105,6 +114,10 @@ func (u *Unnest) flattenDynamic(vec *vector.Dynamic, record bool) (vector.Any, [ } offsets = append(offsets, length) } + if length == 0 { + // Every slot is a null or plain none, which produce no values. + return nil, nil + } return vector.NewDynamic(tags, vecs), offsets } @@ -114,8 +127,9 @@ func (u *Unnest) flattenRecord(vec *vector.Record) (vector.Any, []uint32) { k, ok := vec.Typ.IndexOfField(u.field) if !ok { // The field is absent since its value is a plain none, which - // disappears from a record, so report it as for an unnamed unnest. - return vector.NewWrappedError(u.sctx, "unnest: encountered non-array value", vector.NewNone(vec.Len())), nil + // disappears from a record, so as for an unnamed unnest, there's + // nothing to unnest. + return nil, nil } if union, ok := vec.Fields[k].(*vector.Union); ok { dynamic := union.Dynamic() @@ -161,10 +175,17 @@ func (u *Unnest) flattenRecord(vec *vector.Record) (vector.Any, []uint32) { // newRecord returns a record vector of the fields of typ whose values are // vecs, where the type of each field is the type of its value. All vecs // have the same length. +// A field whose value is a plain none disappears, as in a record expression. func (u *Unnest) newRecord(typ *super.TypeRecord, vecs []vector.Any) vector.Any { - fields := slices.Clone(typ.Fields) + n := vecs[0].Len() + fields := make([]super.Field, 0, len(vecs)) + out := make([]vector.Any, 0, len(vecs)) for j, vec := range vecs { - fields[j].Type = vec.Type() + if isNoneVec(vec) { + continue + } + fields = append(fields, super.NewField(typ.Fields[j].Name, vec.Type())) + out = append(out, vec) } - return vector.NewRecord(u.sctx.MustLookupTypeRecord(fields), vecs, vecs[0].Len()) + return vector.NewRecord(u.sctx.MustLookupTypeRecord(fields), out, n) } diff --git a/runtime/op/values.go b/runtime/op/values.go index c86c7509d7..468b5fcc3c 100644 --- a/runtime/op/values.go +++ b/runtime/op/values.go @@ -15,6 +15,8 @@ type Values struct { var _ vio.Puller = (*Values)(nil) +// NewValues returns a Values that outputs the values of exprs. Plain none +// values are dropped (see dropPlainNones). func NewValues(sctx *super.Context, parent vio.Puller, exprs []expr.Evaluator) *Values { return &Values{ sctx: sctx, @@ -33,12 +35,18 @@ func (v *Values) Pull(done bool) (vector.Any, error) { for _, e := range v.exprs { vals = append(vals, e.Eval(val)) } - if len(vals) == 1 { - return vals[0], nil - } else if len(vals) != 0 { - return interleave(vals), nil + var out vector.Any + switch len(vals) { + case 0: + continue + case 1: + out = vals[0] + default: + out = interleave(vals) + } + if out = dropPlainNones(out); out != nil { + return out, nil } - // If no vals, continue the loop. } } diff --git a/runtime/ztests/expr/function/has.yaml b/runtime/ztests/expr/function/has.yaml new file mode 100644 index 0000000000..09cc39c142 --- /dev/null +++ b/runtime/ztests/expr/function/has.yaml @@ -0,0 +1,89 @@ +spq: values has(this,"a") + +input: | + {a:1} + {b:1} + {a:null} + {a:error("x")} + {a?:none::int64} + 1 + null + error("x") + +output: | + true + false + true + true + true + false + false + false + +--- + +# A named type or union is tested by its underlying record type. +spq: | + type r={a:int64} + values {a:1}::r, cast({a:1},<{a:int64}|int64>), cast(1,<{a:int64}|int64>) + | values has(this,"a") + +output: | + true + true + false + +--- + +spq: values has(this,f) + +input: | + {a:1,f:"a"} + {a:1,f:"b"} + {a:1,f:1} + {a:1,f:error("x")} + +output: | + true + false + error({message:"has: field name must be a string",on:1}) + error("x") + +--- + +# A type value is tested by its record type. +spq: | + type r={a:int64} + values <{a:int64}>, <{b:int64}>, , + | values has(this,"a"), has(this,1) + +output: | + true + false + false + false + true + false + false + false + +--- + +# A map is tested for a key of any type. +spq: values has(this,"a"), has(this,1) + +input: | + map{"a":1,"b":2} + map{"b":2} + map{1:"x"} + map{1::(int64|string):"x","a"::(int64|string):"y"} + +output: | + true + false + false + false + false + true + true + true diff --git a/runtime/ztests/expr/function/under.yaml b/runtime/ztests/expr/function/under.yaml index b410f77a26..eb4e0333a5 100644 --- a/runtime/ztests/expr/function/under.yaml +++ b/runtime/ztests/expr/function/under.yaml @@ -31,7 +31,6 @@ output: | 1 <{s:string}> 1 - error("none not handled (consider ?? or ?.)") --- diff --git a/runtime/ztests/expr/index.yaml b/runtime/ztests/expr/index.yaml index cd9951a201..c8fdff44b3 100644 --- a/runtime/ztests/expr/index.yaml +++ b/runtime/ztests/expr/index.yaml @@ -59,7 +59,6 @@ output: | 1 2 1 - error("none not handled (consider ?? or ?.)") 127.0.0.1 --- @@ -107,7 +106,6 @@ output: | 5 error("cannot access field 'x' of set") "foo" - error("none not handled (consider ?? or ?.)") error({message:"invalid record index",on:1}) 123 @@ -143,8 +141,5 @@ input: | output: | "foo" "bar" - error("none not handled (consider ?? or ?.)") - error("none not handled (consider ?? or ?.)") - error("none not handled (consider ?? or ?.)") "foo" "foo" diff --git a/runtime/ztests/expr/subquery-empty.yaml b/runtime/ztests/expr/subquery-empty.yaml new file mode 100644 index 0000000000..d69820307a --- /dev/null +++ b/runtime/ztests/expr/subquery-empty.yaml @@ -0,0 +1,21 @@ +# A subquery that produces no values is none in a pipe expression and null +# in a SQL expression, for both correlated and uncorrelated subqueries. +spq: | + values {c:(values this | where false)}, + {u:(values 1 | where false)}, + (values this | where false) ?? "none" + +input: | + 1 + +output: | + {} + {} + "none" + +--- + +spq: select (select 1 where false) as x + +output: | + {x:null} diff --git a/runtime/ztests/op/aggregate/none-key.yaml b/runtime/ztests/op/aggregate/none-key.yaml new file mode 100644 index 0000000000..447586a551 --- /dev/null +++ b/runtime/ztests/op/aggregate/none-key.yaml @@ -0,0 +1,64 @@ +# A plain none group key (e.g., from ok()) disappears from its output record, +# including when the aggregate is split into partials (as for a file scan) +# and for a nested key. With a single unnamed key and no aggregate +# functions, the none key is dropped, but an error key isn't. An aggregate +# function's none result isn't a key, so it isn't dropped (as for shapes, +# whose sample is from any). Keys and aggregate values interleaved in +# nested records are assigned to the right fields. +script: | + super -s -c 'values {a:1},{b:2},{b:3} | count() by k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | count() by k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | count() by x.k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | aggregate by ok(a)' + echo === + super -s -c 'values {a:1},{b:2},{b:3} | aggregate by ok(a)' + echo === + super -s -c 'values {a:1},{b:2} | shapes ok(a)' + echo === + super -s -c 'values {a:1},{a:error("x")} | aggregate by a | sort this' + echo === + super -s -c 'from errors.sup | aggregate by a | sort this' + echo === + super -s -c 'values {a:1,b:3},{b:4} | aggregate r.c:=count(), y:=sum(b) by r.k:=b, x:=a.ok() | sort r.k' + +inputs: + - name: test.sup + data: | + {a:1} + {b:2} + {b:3} + - name: errors.sup + data: | + {a:1} + {a:error("x")} + {a:1} + +outputs: + - name: stdout + data: | + {k:1,count:1} + {count:2} + === + {k:1,count:1} + {count:2} + === + {x:{k:1},count:1} + {x:{},count:2} + === + 1 + === + 1 + === + 1 + === + 1 + error("x") + === + 1 + error("x") + === + {r:{k:3,c:1},x:1,y:3} + {r:{k:4,c:1},y:4} diff --git a/runtime/ztests/op/debug-none.yaml b/runtime/ztests/op/debug-none.yaml new file mode 100644 index 0000000000..e9168d596c --- /dev/null +++ b/runtime/ztests/op/debug-none.yaml @@ -0,0 +1,11 @@ +# A plain none debug value is dropped, as for the values operator. +script: | + super -s -c 'values {a:1},{b:2} | debug ok(a) | count()' + +outputs: + - name: stdout + data: | + 2 + - name: stderr + data: | + 1 diff --git a/runtime/ztests/op/unnest.yaml b/runtime/ztests/op/unnest.yaml index cb0ce48780..12b7dc332e 100644 --- a/runtime/ztests/op/unnest.yaml +++ b/runtime/ztests/op/unnest.yaml @@ -92,7 +92,7 @@ output: | --- -# A plain none to unnest is an error as for an unnamed unnest. +# A plain none to unnest produces no values as for an unnamed unnest. spq: unnest x:=o?.a with s flags: -dynamic @@ -102,5 +102,56 @@ input: | {o:some({a:[5]}),s:2} output: | - error({message:"unnest: encountered non-array value",on:none}) {x:5,s:2} + +--- + +# A plain none to unnest (e.g., from ok()) produces no values. +spq: unnest ok(a) + +input: | + {a:[1,2]} + {b:3} + {a:[4]} + +output: | + 1 + 2 + 4 + +--- + +# Statically, a plain none to unnest isn't an error. +spq: values {a:[1,2]},{b:3} | unnest ok(a) | values typeof(this) + +output: | + + + +--- + +# A plain none element or a null or plain none array produces no values, +# including for an unnest with a with clause. +spq: | + values [1::(int64|none), none::(int64|none)], none::([int64]|none) + | unnest this + | values typeof(this) + +output: | + + +--- + +spq: | + values + {x:null::([int64]|null),y:1}, + {x:[1,2]::([int64]|null),y:2}, + {x:none::([int64]|none),y:3}, + {x:[3]::([int64]|none),y:4} + | unnest x with y + | values this + +output: | + {x:1,y:2} + {x:2,y:2} + {x:3,y:4} diff --git a/runtime/ztests/op/values/drop-none.yaml b/runtime/ztests/op/values/drop-none.yaml new file mode 100644 index 0000000000..e331512620 --- /dev/null +++ b/runtime/ztests/op/values/drop-none.yaml @@ -0,0 +1,55 @@ +# The values operator drops plain nones, including those interleaved with +# the values of other expressions and those in a union, but keeps an +# option's none. +spq: values ok(a), 0 + +input: | + {a:1} + {b:2} + {a:"s"} + +output: | + 1 + 0 + 0 + "s" + 0 + +--- + +spq: values this + +input: | + 1::(int64|none) + none::(int64|none) + 2::(int64|none) + +output: | + 1::(int64|none) + 2::(int64|none) + +--- + +spq: values ok(a) ?? none::int64 + +input: | + {a:1} + {b:2} + +output: | + 1 + none::int64 + +--- + +# A dropped none isn't passed downstream. +spq: "values this==2 ? none : this | values typeof(this)" + +input: | + 1 + 2 + 3 + +output: | + + diff --git a/sio/bsupio/ztests/issue-5736.yaml b/sio/bsupio/ztests/issue-5736.yaml index 1e03f3e846..d1bbd588d2 100644 --- a/sio/bsupio/ztests/issue-5736.yaml +++ b/sio/bsupio/ztests/issue-5736.yaml @@ -9,7 +9,7 @@ script: | did | count := sum(count), users := count() - by event + by event:=ok(event) | sort -r count" inputs: diff --git a/spq_test.go b/spq_test.go index a81731eab5..8ce50f7ec1 100644 --- a/spq_test.go +++ b/spq_test.go @@ -184,6 +184,11 @@ func runOneBoomerang(t *testing.T, format, data string) { } t.Fatalf("unexpected error writing %s baseline: %s", format, err) } + if baseline == "" { + // The data has no values (e.g., it's only plain nones, which are + // dropped). + t.Skip("skipping due to no values") + } readerFormat := format if readerFormat == "bsup -rows" { readerFormat = "bsup" @@ -308,8 +313,9 @@ func (f *fusionRemovingReader) Read() (*super.Value, error) { if f.h.hasFusion(val.Type()) { continue } - if val.Type() == super.TypeNone { - // A plain none can't be output by a query. + if val.Deunion().Type() == super.TypeNone { + // A plain none (even in a union) is dropped from the + // output of a query. continue } } diff --git a/sup/ztests/decorate-none.yaml b/sup/ztests/decorate-none.yaml index 8fcaf9f9b3..dac6223dd1 100644 --- a/sup/ztests/decorate-none.yaml +++ b/sup/ztests/decorate-none.yaml @@ -24,7 +24,6 @@ input: | missing output: | - error("none not handled (consider ?? or ?.)") [1,none] [some(1),none::int64] set[none::int64,some(1)] diff --git a/vector/recordbuilder.go b/vector/recordbuilder.go index d9164f6369..cfb02fb152 100644 --- a/vector/recordbuilder.go +++ b/vector/recordbuilder.go @@ -10,29 +10,40 @@ import ( type RecordBuilder struct { sctx *super.Context base *rec + // NoneDrops is the number of leading fields (e.g., an aggregate's group + // keys) that disappear from a record when their values are plain nones + // (e.g., from ok() or ?.), as in a record expression. + NoneDrops int } +// NewRecordBuilder returns a builder of records with the leaf fields in +// fields. Paths with a common prefix are nested in the same record, in the +// order in which the prefix first appears. func NewRecordBuilder(sctx *super.Context, fields field.List) (*RecordBuilder, error) { base := &rec{} - for _, path := range fields { - if err := addPath(base, path); err != nil { + for k, path := range fields { + if err := addPath(base, path, k); err != nil { return nil, err } } return &RecordBuilder{sctx: sctx, base: base}, nil } +// New returns a record whose leaf fields are vecs in the order of the fields +// passed to NewRecordBuilder. func (r *RecordBuilder) New(vecs []Any) *Record { - rec, _ := r.base.build(r.sctx, vecs) - return rec + return r.base.build(r.sctx, vecs, r.NoneDrops) } type rec struct { paths []string - recs []*rec + // recs[k] is the record for paths[k] or nil if it's a leaf, in which + // case leaves[k] is the index of its value in the vectors passed to New. + recs []*rec + leaves []int } -func addPath(r *rec, path field.Path) error { +func addPath(r *rec, path field.Path, leaf int) error { for k, name := range path { idx := slices.Index(r.paths, name) if k == len(path)-1 { @@ -41,12 +52,14 @@ func addPath(r *rec, path field.Path) error { } r.paths = append(r.paths, path[k]) r.recs = append(r.recs, nil) + r.leaves = append(r.leaves, leaf) return nil } if idx == -1 { idx = len(r.paths) r.paths = append(r.paths, name) r.recs = append(r.recs, &rec{}) + r.leaves = append(r.leaves, -1) } if r.recs[idx] == nil { return &super.DuplicateFieldError{Name: path[:k+1].String()} @@ -56,19 +69,27 @@ func addPath(r *rec, path field.Path) error { return nil } -func (r *rec) build(sctx *super.Context, leafs []Any) (*Record, []Any) { +// build returns a record whose leaf fields are taken from leafs. A leaf field +// whose index is less than noneDrops and whose value is a plain none +// disappears from the record. +func (r *rec) build(sctx *super.Context, leafs []Any, noneDrops int) *Record { + n := leafs[0].Len() var fields []super.Field var out []Any for i, name := range r.paths { var vec Any if r.recs[i] != nil { - vec, leafs = r.recs[i].build(sctx, leafs) + vec = r.recs[i].build(sctx, leafs, noneDrops) } else { - vec, leafs = leafs[0], leafs[1:] + leaf := r.leaves[i] + vec = leafs[leaf] + if leaf < noneDrops && vec.Type() == super.TypeNone { + continue + } } fields = append(fields, super.NewField(name, vec.Type())) out = append(out, vec) } typ := sctx.MustLookupTypeRecord(fields) - return NewRecord(typ, out, out[0].Len()), leafs + return NewRecord(typ, out, n) } From 8a262d6c849c86f112ab31ef93df2f1a386fb3ac Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Mon, 9 Nov 2026 15:42:55 -0800 Subject: [PATCH 2/3] address PR feedback --- compiler/semantic/checker.go | 26 ++++++---- compiler/semantic/ztests/none-absent.yaml | 3 ++ runtime/op/nones.go | 58 +++++++++++------------ runtime/ztests/op/unnest.yaml | 20 +++++++- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/compiler/semantic/checker.go b/compiler/semantic/checker.go index 1d786d1bbe..c8c089b3b3 100644 --- a/compiler/semantic/checker.go +++ b/compiler/semantic/checker.go @@ -214,6 +214,16 @@ func (c *checker) unnest(loc ast.Node, typ super.Type, field string) super.Type return typ } +// unnestElem returns the type of the values unnested from an array or set +// with element type typ (or nil if there are none), which are those of typ +// without plain none since unnest drops plain nones as values does. +func (c *checker) unnestElem(typ super.Type) super.Type { + if out, ok := c.withoutPlainNone(typ); ok { + return out + } + return nil +} + func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { if isUnknown(typ) { return c.unknown, true @@ -242,9 +252,11 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { case *super.TypeOption: return c.unnestCheck(loc, typ.Type) case *super.TypeArray: - return typ.Type, true + return c.unnestElem(typ.Type), true + case *super.TypeSet: + return c.unnestElem(typ.Type), true default: - c.error(loc, errors.New("unnest value must be an array")) + c.error(loc, errors.New("unnest value must be an array or set")) return c.unknown, false } } @@ -565,8 +577,10 @@ func isMissingType(typ super.Type) bool { return under(typ) == super.TypeMissing } +// isNoneType reports whether typ is the type of a plain none, which +// disappears from a record or array. A none of a named type is a value. func isNoneType(typ super.Type) bool { - return under(typ) == super.TypeNone + return defuse(typ) == super.TypeNone } // isNoneishType reports whether typ is a type whose values ?? replaces with @@ -883,10 +897,6 @@ func (c *checker) noneAsMissing(typ super.Type) super.Type { return out } -func isPlainNone(typ super.Type) bool { - return defuse(typ) == super.TypeNone -} - // unionOf returns the single type in types or else the union of types. func (c *checker) unionOf(types []super.Type) super.Type { if len(types) == 1 { @@ -1077,7 +1087,7 @@ func (c *checker) putPaths(inType super.Type, puts []pathType) super.Type { // A field whose value is a plain none disappears from the record // (e.g., an aggregate's group key), so it may be missing. leaf := c.noneAsMissing(put.typ) - if isPlainNone(leaf) { + if isNoneType(leaf) { leaf = super.TypeMissing } typ := c.putInto(recType, put.elems, leaf) diff --git a/compiler/semantic/ztests/none-absent.yaml b/compiler/semantic/ztests/none-absent.yaml index d9f23176f1..28850b4b35 100644 --- a/compiler/semantic/ztests/none-absent.yaml +++ b/compiler/semantic/ztests/none-absent.yaml @@ -17,6 +17,7 @@ script: | echo === named ! super -s -c 'type u=int64|none values {a:1},{b:2} | values ok(a)::u | values this+1' 2> /dev/null super -s -c 'type u=int64|none values {a:1},{b:2} | values ok(a)::u' + super -s -c 'type n=none values {a:none::n,b:1} | values a' outputs: - name: stdout @@ -34,3 +35,5 @@ outputs: type u=int64|none 1::u none::u + type n=none + none::n diff --git a/runtime/op/nones.go b/runtime/op/nones.go index bea5510b6f..1336b755d7 100644 --- a/runtime/op/nones.go +++ b/runtime/op/nones.go @@ -14,14 +14,14 @@ import ( // dropped. A none of a named type and an option's none are values and // are kept. func dropPlainNones(vec vector.Any) vector.Any { - nones := plainNoneSlots(vec) - if len(nones) == 0 { - return vec - } + nones, all := plainNoneSlots(vec) n := vec.Len() - if len(nones) == int(n) { + if all || len(nones) == int(n) { return nil } + if len(nones) == 0 { + return vec + } rest := make([]uint32, 0, int(n)-len(nones)) var k int for slot := range n { @@ -35,39 +35,45 @@ func dropPlainNones(vec vector.Any) vector.Any { } // plainNoneSlots returns the slots of vec, in order, whose values are plain -// nones. -func plainNoneSlots(vec vector.Any) []uint32 { +// nones, or true if all of its values are plain nones, in which case the +// slots aren't returned. +func plainNoneSlots(vec vector.Any) ([]uint32, bool) { switch v := vec.(type) { case *vector.Dynamic: // The values of a dynamic may themselves be dynamics. marks := make([][]bool, len(v.Values)) + allNone := make([]bool, len(v.Values)) var found bool for k, val := range v.Values { if val == nil { continue } - if slots := plainNoneSlots(val); len(slots) != 0 { + slots, all := plainNoneSlots(val) + if all { + allNone[k] = true + found = true + } else if len(slots) != 0 { marks[k] = markSlots(val.Len(), slots) found = true } } if !found { - return nil + return nil, false } var nones []uint32 counts := make([]uint32, len(v.Values)) for slot, tag := range v.Tags { - if m := marks[tag]; m != nil && m[counts[tag]] { + if allNone[tag] || marks[tag] != nil && marks[tag][counts[tag]] { nones = append(nones, uint32(slot)) } counts[tag]++ } - return nones + return nones, false case *vector.Union: // Check the type first so the tags of a union without a none // member aren't loaded. if !hasPlainNoneMember(v.Typ) { - return nil + return nil, false } tags := v.Tags() values := v.Values() @@ -77,14 +83,14 @@ func plainNoneSlots(vec vector.Any) []uint32 { nones = append(nones, uint32(slot)) } } - return nones + return nones, false case *vector.Fusion: // A fusion value is a plain none if its subtype is none, which is // possible only if the fusion's type has none. Check the type first // so the subtypes needn't be loaded. if typ, ok := v.Typ.Type.(*super.TypeUnion); !ok || !hasPlainNoneMember(typ) { if v.Typ.Type != super.TypeNone { - return nil + return nil, false } } var nones []uint32 @@ -93,11 +99,14 @@ func plainNoneSlots(vec vector.Any) []uint32 { nones = append(nones, uint32(slot)) } } - return nones + return nones, false case *vector.View: - slots := plainNoneSlots(v.Any) + slots, all := plainNoneSlots(v.Any) + if all { + return nil, true + } if len(slots) == 0 { - return nil + return nil, false } marks := markSlots(v.Any.Len(), slots) var nones []uint32 @@ -106,12 +115,9 @@ func plainNoneSlots(vec vector.Any) []uint32 { nones = append(nones, uint32(slot)) } } - return nones - } - if isNoneVec(vec) { - return allSlots(vec.Len()) + return nones, false } - return nil + return nil, isNoneVec(vec) } // hasPlainNoneMember reports whether typ has plain none as a member. @@ -119,14 +125,6 @@ func hasPlainNoneMember(typ *super.TypeUnion) bool { return slices.ContainsFunc(typ.Types, func(t super.Type) bool { return t == super.TypeNone }) } -func allSlots(n uint32) []uint32 { - slots := make([]uint32, n) - for k := range slots { - slots[k] = uint32(k) - } - return slots -} - func markSlots(n uint32, slots []uint32) []bool { marks := make([]bool, n) for _, slot := range slots { diff --git a/runtime/ztests/op/unnest.yaml b/runtime/ztests/op/unnest.yaml index 12b7dc332e..049a9391df 100644 --- a/runtime/ztests/op/unnest.yaml +++ b/runtime/ztests/op/unnest.yaml @@ -73,7 +73,7 @@ input: | {x:"foo",y:[1,2]} error: | - unnest value must be an array at line 1, column 8: + unnest value must be an array or set at line 1, column 8: unnest this ~~~~ @@ -155,3 +155,21 @@ output: | {x:1,y:2} {x:2,y:2} {x:3,y:4} + +--- + +# The type of an unnested value excludes plain none, which is dropped. +spq: | + values [1::(int64|none), none::(int64|none)] | unnest this | values this+1 + +output: | + 2 + +--- + +spq: | + values {s:set[1,2],y:3} | unnest s with y | values this + +output: | + {s:1,y:3} + {s:2,y:3} From 5a663679cc6c143051092f75d91f623b66af054b Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Mon, 9 Nov 2026 15:52:18 -0800 Subject: [PATCH 3/3] oops --- runtime/op/nones.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/op/nones.go b/runtime/op/nones.go index 1336b755d7..2116c7cf01 100644 --- a/runtime/op/nones.go +++ b/runtime/op/nones.go @@ -14,6 +14,9 @@ import ( // dropped. A none of a named type and an option's none are values and // are kept. func dropPlainNones(vec vector.Any) vector.Any { + if vec == nil { + return nil + } nones, all := plainNoneSlots(vec) n := vec.Len() if all || len(nones) == int(n) {