diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index afc487242..f512c350d 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -137,6 +137,7 @@ - [regexp_replace](super-sql/functions/parsing/regexp_replace.md) - [Records](super-sql/functions/records/intro.md) - [flatten](super-sql/functions/records/flatten.md) + - [has](super-sql/functions/records/has.md) - [keys](super-sql/functions/records/keys.md) - [nest_dotted](super-sql/functions/records/nest_dotted.md) - [paths](super-sql/functions/records/paths.md) diff --git a/book/src/super-sql/expressions/match.md b/book/src/super-sql/expressions/match.md index 5688f660d..664dfbabe 100644 --- a/book/src/super-sql/expressions/match.md +++ b/book/src/super-sql/expressions/match.md @@ -19,7 +19,8 @@ match expression. A value matches a type by the same rules as a its underlying value and [named types](../types/named.md) are distinct. If no case matches, the result is `` or, without an -`else` clause, `none`, which disappears from a record, array, or map +`else` clause, `none`, which disappears from a record, array, or map, +is dropped from the output of the [values](../operators/values.md) operator, and is typically handled with the `??` operator. An [error](../types/error.md) or `missing` value is passed through unless a case has its type. diff --git a/book/src/super-sql/expressions/subqueries.md b/book/src/super-sql/expressions/subqueries.md index f30e1550a..268e8ca6b 100644 --- a/book/src/super-sql/expressions/subqueries.md +++ b/book/src/super-sql/expressions/subqueries.md @@ -19,6 +19,9 @@ results in the value `{s:"HELLO, WORLD"}`. Except for subqueries appearing as the right-hand side of an [in](containment.md) operator, the result of a subquery must be a single value. When multiple values are generated, an [error](../types/error.md) is produced. +When no values are generated, the result is `none`, which disappears from a +record, array, or map and is dropped from the output of the +[values](../operators/values.md) operator, or, in a SQL expression, `null`. For the [in](containment.md) operator, any subquery on the right-hand side is always treated as an [array subquery](#array-subqueries), thus @@ -278,3 +281,20 @@ values walk(this, &addOne) [2,3,4] [{x:[2,"foo"]},{y:3}] ``` + +--- + +_A subquery that produces no values is none_ + +```mdtest-spq +# spq +values {s:(unnest a | where this > 1)}, (unnest a | where this > 1) ?? 0 +# input +{a:[1,2]} +{a:[1]} +# expected output +{s:2} +2 +{} +0 +``` diff --git a/book/src/super-sql/expressions/type-assertions.md b/book/src/super-sql/expressions/type-assertions.md index a0456b414..f4b1b2fc0 100644 --- a/book/src/super-sql/expressions/type-assertions.md +++ b/book/src/super-sql/expressions/type-assertions.md @@ -7,7 +7,8 @@ A type assertion has the form where `` is any [expression](intro.md) and `` is any [type](../types/intro.md). Its value is the value of `` when that value has type `` and is otherwise `none`, -which disappears from a record, array, or map and is typically handled with +which disappears from a record, array, or map, is dropped from the output of the +[values](../operators/values.md) operator, and is typically handled with the `??` operator, e.g., ``` v.(string) ?? "default" diff --git a/book/src/super-sql/functions/generics/ok.md b/book/src/super-sql/functions/generics/ok.md index 6dff8bf0c..6ca4093c8 100644 --- a/book/src/super-sql/functions/generics/ok.md +++ b/book/src/super-sql/functions/generics/ok.md @@ -16,8 +16,9 @@ unmodified. This function is useful when errors are expected and are to be explicitly managed with `none` operators. The `none` disappears from a record expression, an -array, or a map and otherwise must be handled with `??` since a query may not -output a `none`. +array, or a map and is dropped from the output of the +[values](../../operators/values.md) operator, so `??` is needed only to replace +it with another value. ## Examples diff --git a/book/src/super-sql/functions/records/has.md b/book/src/super-sql/functions/records/has.md new file mode 100644 index 000000000..d6a92d76e --- /dev/null +++ b/book/src/super-sql/functions/records/has.md @@ -0,0 +1,112 @@ +# has + +test whether a record or record type has a field or a map has a key + +## Synopsis + +``` +has(val: any, key: any) -> bool +``` + +## Description + +The `has` function returns true if +* `val` is a record whose type has a field named `key`, +* `val` is a type value of a record type with a field named `key`, or +* `val` is a map with the key `key` + +and false otherwise, e.g., for a value that isn't a record, type value, or +map. + +For a record, the result depends only on its type, not on the value of the +field, so `has` is true for a field whose value is null, an error, or an +option's none. A value of a named type or a union type is tested by its +underlying type. + +For a map, `key` may be of any type and is compared with the map's keys after +removing named and union types, so a key of a different type (e.g., an +`int32` for a map with `int64` keys) is never found. + +An error is returned if `val` is a record and `key` isn't a string. + +## Examples + +--- + +_Test records for fields_ + +```mdtest-spq +# spq +values {a:has(this,"a"),b:has(this,"b")} +# input +{a:1} +{b:null} +{a:error("x"),b:2} +1 +# expected output +{a:true,b:false} +{a:false,b:true} +{a:true,b:true} +{a:false,b:false} +``` + +--- + +_Filter on the presence of a field_ + +```mdtest-spq +# spq +where has(this,"b") +# input +{a:1} +{b:2} +# expected output +{b:2} +``` + +--- + +_Test a record type_ + +```mdtest-spq +# spq +values has(this,"a") +# input +<{a:int64}> +<{b:int64}> +# expected output +true +false +``` + +--- + +_Test a map for a key_ + +```mdtest-spq +# spq +values has(this,"a"), has(this,1) +# input +map{"a":1} +map{1:"x"} +# expected output +true +false +false +true +``` + +--- + +_The field name may be computed_ + +```mdtest-spq +# spq +values has(this, f) +# input +{a:1,f:"a"} +{a:1,f:"b"} +# expected output +true +false +``` diff --git a/book/src/super-sql/operators/aggregate.md b/book/src/super-sql/operators/aggregate.md index 6ef616fc0..aa80bfe58 100644 --- a/book/src/super-sql/operators/aggregate.md +++ b/book/src/super-sql/operators/aggregate.md @@ -55,6 +55,10 @@ The output values are records formed from the [field assignments](intro.md#field-assignment) first from the grouping expressions then from the aggregate functions in left-to-right order. +As in a record expression, a grouping key whose value +is a plain `none` (e.g., `ok(x)` when `x` is missing) disappears from its +output record, so the values whose key is `none` form a group whose record +has no field for that key. When the result of `aggregate` is a single value (e.g., a single aggregate function without grouping expressions or a single grouping expression without aggregates) diff --git a/book/src/super-sql/operators/debug.md b/book/src/super-sql/operators/debug.md index 33a2b3a0b..f8bde42ad 100644 --- a/book/src/super-sql/operators/debug.md +++ b/book/src/super-sql/operators/debug.md @@ -16,6 +16,8 @@ each value in the input, evaluates the optional and transmits each result to a _debugging output_. If `` is omitted, then each input value is passed unmodified to the debugging output. +As for the [values](values.md) operator, a result that is a plain `none` +(e.g., from [ok](../functions/generics/ok.md) or `?.`) is dropped. An optional filter may be applied to the values sent to the debugging output, which is specified with Boolean-valued expression ``. diff --git a/book/src/super-sql/operators/unnest.md b/book/src/super-sql/operators/unnest.md index aa06bf872..acdbf43de 100644 --- a/book/src/super-sql/operators/unnest.md +++ b/book/src/super-sql/operators/unnest.md @@ -41,7 +41,8 @@ produces ``` The value of `` must be an array (or a set), whose elements form the -derived sequence. +derived sequence. A `null` or a plain `none` value (e.g., from +[ok](../functions/generics/ok.md) or `?.`) produces no values. To carry other values along with each unnested element, give the unnested value a name with `:=` or list the values in a `with` clause. Then each diff --git a/book/src/super-sql/operators/values.md b/book/src/super-sql/operators/values.md index 87d0b073a..316e99ad6 100644 --- a/book/src/super-sql/operators/values.md +++ b/book/src/super-sql/operators/values.md @@ -19,6 +19,13 @@ in left-to-right order. Each `` may be any valid The input order convolved with left-to-right evaluation order is preserved at the output. +A result that is a plain `none`, e.g., from [ok](../functions/generics/ok.md), +`?.`, a [type assertion](../expressions/type-assertions.md) that fails, or a +[match](../expressions/match.md) with no matching case and no `else`, +is dropped, so a `none` is never passed to a downstream operator. +To output another value in its place, use the `??` operator. +An option's `none` (e.g., `none::int64`) is a value and is output as is. + The `values` operator name is optional since it can be used as a [shortcut](intro.md#shortcuts). When used as a shortcut, only one expression may be present. @@ -127,3 +134,18 @@ values [a,b],[b,a] | collect(this) # expected output [[1,2],[2,1]] ``` + +--- + +_A plain none is dropped unless it's replaced with ??_ +```mdtest-spq +# spq +values ok(a), ok(a) ?? 0 +# input +{a:1} +{b:2} +# expected output +1 +1 +0 +``` diff --git a/compiler/dag/expr.go b/compiler/dag/expr.go index 07e36f961..ef701942a 100644 --- a/compiler/dag/expr.go +++ b/compiler/dag/expr.go @@ -146,7 +146,11 @@ type ( SubqueryExpr struct { Kind string `json:"kind" unpack:""` Correlated bool `json:"correlated"` - Body Seq `json:"body"` + // SQL is true for a subquery in a SQL expression, whose value is + // null when the subquery produces no values. Otherwise, the value + // is none. + SQL bool `json:"sql,omitempty"` + Body Seq `json:"body"` } ThisExpr struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 758f4edb2..7e810d9a1 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -373,7 +373,7 @@ func NewFilterOp(e Expr) *FilterOp { } func NewValuesOp(exprs ...Expr) *ValuesOp { - return &ValuesOp{"ValuesOp", exprs} + return &ValuesOp{Kind: "ValuesOp", Exprs: exprs} } func CopySeq(seq Seq) Seq { diff --git a/compiler/rungen/vexpr.go b/compiler/rungen/vexpr.go index 6cd542c73..f3776fc9e 100644 --- a/compiler/rungen/vexpr.go +++ b/compiler/rungen/vexpr.go @@ -313,11 +313,11 @@ func (b *Builder) compileVamSubquery(query *dag.SubqueryExpr) (expr.Evaluator, e return nil, err } body := b.combineVam(exits) - return op.NewCachedSubquery(b.sctx(), body), nil + return op.NewCachedSubquery(b.sctx(), body, query.SQL), nil } var create func() *op.Subquery create = func() *op.Subquery { - subquery := op.NewSubquery(b.rctx.Context, b.sctx(), func() *op.Subquery { + subquery := op.NewSubquery(b.rctx.Context, b.sctx(), query.SQL, func() *op.Subquery { b.mu.Lock() defer b.mu.Unlock() return create() diff --git a/compiler/rungen/vop.go b/compiler/rungen/vop.go index 2a29189f5..b17c9ceaf 100644 --- a/compiler/rungen/vop.go +++ b/compiler/rungen/vop.go @@ -317,7 +317,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vio.Puller) (vio.Puller, error return vio.NewPuller(vector.NewNull(1)), nil case *dag.OutputOp: if parent != nil { - parent = op.NewOutput(b.sctx(), parent) + parent = op.NewOutput(parent) } b.channels[o.Name] = append(b.channels[o.Name], parent) return parent, nil diff --git a/compiler/semantic/analyzer.go b/compiler/semantic/analyzer.go index 5ebd59a89..43a51250b 100644 --- a/compiler/semantic/analyzer.go +++ b/compiler/semantic/analyzer.go @@ -79,9 +79,6 @@ type translator struct { sctx *super.Context types *sup.Analyzer defs *super.Context - // subqueries is the depth of expression subqueries being translated. - // Their results may be none (see checker.output). - subqueries int } func newTranslator(ctx context.Context, r reporter, env *exec.Environment) *translator { diff --git a/compiler/semantic/checker.go b/compiler/semantic/checker.go index 4755574c6..c8c089b3b 100644 --- a/compiler/semantic/checker.go +++ b/compiler/semantic/checker.go @@ -156,7 +156,7 @@ func (c *checker) op(typ super.Type, op sem.Op) super.Type { case *sem.UnnestOp: return c.seq(c.unnest(op.Expr, c.expr(typ, op.Expr), op.Field), op.Body) case *sem.ValuesOp: - return c.fuse(c.exprs(typ, op.Exprs)) + return c.values(c.exprs(typ, op.Exprs)) default: panic(op) } @@ -207,13 +207,31 @@ func (c *checker) unnest(loc ast.Node, typ super.Type, field string) super.Type if !ok { c.keepErrs(errs) } + if typ == nil { + // The value is always a plain none, so there are no values. + return c.unknown + } return typ } +// unnestElem returns the type of the values unnested from an array or set +// with element type typ (or nil if there are none), which are those of typ +// without plain none since unnest drops plain nones as values does. +func (c *checker) unnestElem(typ super.Type) super.Type { + if out, ok := c.withoutPlainNone(typ); ok { + return out + } + return nil +} + func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { if isUnknown(typ) { return c.unknown, true } + if typ == super.TypeNone { + // A plain none produces no values. + return nil, true + } switch typ := under(typ).(type) { case *super.TypeError: c.error(loc, errors.New("unnested record cannot be an error")) @@ -224,7 +242,9 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { for _, t := range typ.Types { typ, tok := c.unnestCheck(loc, t) if tok { - types = append(types, typ) + if typ != nil { + types = append(types, typ) + } ok = true } } @@ -232,9 +252,11 @@ func (c *checker) unnestCheck(loc ast.Node, typ super.Type) (super.Type, bool) { case *super.TypeOption: return c.unnestCheck(loc, typ.Type) case *super.TypeArray: - return typ.Type, true + return c.unnestElem(typ.Type), true + case *super.TypeSet: + return c.unnestElem(typ.Type), true default: - c.error(loc, errors.New("unnest value must be an array")) + c.error(loc, errors.New("unnest value must be an array or set")) return c.unknown, false } } @@ -252,6 +274,9 @@ func (c *checker) unnestField(loc ast.Node, typ super.Type, field string) (super return c.unknown, true } elemType, ok := c.unnestCheck(loc, rec.Fields[k].Type) + if elemType == nil { + return nil, ok + } fields := slices.Clone(rec.Fields) fields[k].Type = elemType out, err := c.t.sctx.LookupTypeRecord(fields) @@ -441,11 +466,7 @@ func (c *checker) expr(typ super.Type, e sem.Expr) super.Type { c.sliceable(e.Expr, container) return container case *sem.SubqueryExpr: - typ = c.seq(typ, e.Body) - if e.Array { - typ = c.t.sctx.LookupTypeArray(typ) - } - return typ + return c.subquery(e, c.seq(typ, e.Body)) case *sem.ThisExpr: return c.this(e.Node, e, typ) case *sem.TypeExpr: @@ -467,6 +488,22 @@ func (c *checker) expr(typ super.Type, e sem.Expr) super.Type { } } +// subquery returns the type of subquery e whose body's output has type typ. +// A pipe subquery whose body may produce no values is none (see mayBeEmpty). +// (A SQL subquery is null.) +func (c *checker) subquery(e *sem.SubqueryExpr, typ super.Type) super.Type { + if e.Array { + return c.t.sctx.LookupTypeArray(typ) + } + if e.SQL || !mayBeEmpty(e) { + return typ + } + if typ == nil { + return super.TypeNone + } + return c.orNone(typ) +} + func (c *checker) ok(typ super.Type, call *sem.CallExpr) super.Type { if len(call.Args) != 1 { panic(call) @@ -540,8 +577,10 @@ func isMissingType(typ super.Type) bool { return under(typ) == super.TypeMissing } +// isNoneType reports whether typ is the type of a plain none, which +// disappears from a record or array. A none of a named type is a value. func isNoneType(typ super.Type) bool { - return under(typ) == super.TypeNone + return defuse(typ) == super.TypeNone } // isNoneishType reports whether typ is a type whose values ?? replaces with @@ -813,6 +852,8 @@ func (c *checker) formRecordElems(elems []sem.RecordElem, types []super.Type) su } continue } + // A field whose value may be none may be absent. + typ = c.noneAsMissing(typ) if k, ok := order[elem.Name]; ok { // As in the runtime, a field of the same name as an earlier // field (e.g., from a spread) replaces it in place. @@ -828,6 +869,34 @@ func (c *checker) formRecordElems(elems []sem.RecordElem, types []super.Type) su return c.t.sctx.MustLookupTypeRecord(fields) } +// noneAsMissing returns typ with plain none replaced by missing if typ is a +// union (or the fusion of a union) with plain none as a member, i.e., the type +// of a field whose value may be none, which disappears from a record, so the +// field may be missing. A named type is a value and is returned as is. +func (c *checker) noneAsMissing(typ super.Type) super.Type { + if typ == nil || hasUnknown(typ) { + return typ + } + u, ok := defuse(typ).(*super.TypeUnion) + if !ok || !slices.Contains(u.Types, super.Type(super.TypeNone)) { + return typ + } + var types []super.Type + for _, t := range u.Types { + if t == super.TypeNone { + t = super.TypeMissing + } + types = append(types, t) + } + out := c.unionOf(super.Flatten(types)) + if _, isFusion := typ.(*super.TypeFusion); isFusion { + if _, isUnion := out.(*super.TypeUnion); isUnion { + return c.t.sctx.MustLookupTypeFusion(out) + } + } + return out +} + // unionOf returns the single type in types or else the union of types. func (c *checker) unionOf(types []super.Type) super.Type { if len(types) == 1 { @@ -1015,7 +1084,13 @@ func (c *checker) putPaths(inType super.Type, puts []pathType) super.Type { return c.unknown } for _, put := range puts { - typ := c.putInto(recType, put.elems, put.typ) + // A field whose value is a plain none disappears from the record + // (e.g., an aggregate's group key), so it may be missing. + leaf := c.noneAsMissing(put.typ) + if isNoneType(leaf) { + leaf = super.TypeMissing + } + typ := c.putInto(recType, put.elems, leaf) var ok bool recType, ok = typ.(*super.TypeRecord) if !ok { @@ -1739,23 +1814,44 @@ func (c *checker) result(loc ast.Node, typ super.Type) { } } -// output reports an error at loc if typ, the type of an output value, may -// be a plain none. (An option none or a none of a named type is a value -// and is allowed.) -func (c *checker) output(loc ast.Node, typ super.Type) { - typ = defuse(typ) - if typ != nil && !hasUnknown(typ) && hasPlainNone(typ) { - c.error(loc, errNone) +// values returns the type of the output of a values operator whose +// expressions have types. The operator drops plain nones, so none is removed +// from each type and an expression that's always a plain none produces no +// values. (An option none or a none of a named type is a value and is kept.) +func (c *checker) values(types []super.Type) super.Type { + var out []super.Type + for _, typ := range types { + if typ, ok := c.withoutPlainNone(typ); ok { + out = append(out, typ) + } } + return c.fuse(out) } -// hasPlainNone reports whether typ is none or a union with none as one of -// its types. Unlike hasNone, a named type whose type is none isn't none. -func hasPlainNone(typ super.Type) bool { - if u, ok := typ.(*super.TypeUnion); ok { - return slices.ContainsFunc(u.Types, hasPlainNone) +// withoutPlainNone returns typ with plain none removed from it (from its +// members if it's a union). It returns false if nothing is left. +func (c *checker) withoutPlainNone(typ super.Type) (super.Type, bool) { + if typ == nil || hasUnknown(typ) { + return typ, true + } + if _, ok := defuse(typ).(*super.TypeNamed); ok { + // A none of a named type (including a named union) is a value. + return typ, true + } + fusion, isFusion := typ.(*super.TypeFusion) + out, ok := c.withoutTypes(defuse(typ), func(typ super.Type) bool { + return typ == super.TypeNone + }) + if !ok { + return nil, false + } + if isFusion && out != fusion.Type { + if _, isUnion := out.(*super.TypeUnion); isUnion { + // The values are still those of a fusion. + return c.t.sctx.MustLookupTypeFusion(out), true + } } - return typ == super.TypeNone + return out, true } // notMissing reports an error at loc if typ may be missing. diff --git a/compiler/semantic/compare.go b/compiler/semantic/compare.go index 16c51081c..add8cdde9 100644 --- a/compiler/semantic/compare.go +++ b/compiler/semantic/compare.go @@ -281,7 +281,7 @@ func eqExpr(aexpr, bexpr sem.Expr) bool { return ok && eqExpr(a.Expr, b.Expr) && eqExpr(a.From, b.From) && eqExpr(a.To, b.To) case *sem.SubqueryExpr: b, ok := bexpr.(*sem.SubqueryExpr) - return ok && a.Correlated == b.Correlated && a.Array == b.Array && eqSeq(a.Body, b.Body) + return ok && a.Correlated == b.Correlated && a.Array == b.Array && a.SQL == b.SQL && eqSeq(a.Body, b.Body) case *sem.ThisExpr: b, ok := bexpr.(*sem.ThisExpr) if !ok || len(a.Chain) != len(b.Chain) { diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index 2d946e57c..ff561d8ee 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -590,6 +590,7 @@ func (d *dagen) subquery(e *sem.SubqueryExpr) *dag.SubqueryExpr { subquery := &dag.SubqueryExpr{ Kind: "SubqueryExpr", Correlated: e.Correlated, + SQL: e.SQL, Body: d.seq(e.Body), } if e.Array { diff --git a/compiler/semantic/expr.go b/compiler/semantic/expr.go index c33736e76..2822311fb 100644 --- a/compiler/semantic/expr.go +++ b/compiler/semantic/expr.go @@ -735,11 +735,13 @@ func (t *translator) semCall(call *ast.CallExpr, inType super.Type) (sem.Expr, s func (t *translator) maybeSubquery(n ast.Node, name string, inType super.Type) (*sem.SubqueryExpr, super.Type) { if seq, typ := t.scope.lookupQuery(t, name); seq != nil { - return &sem.SubqueryExpr{ + e := &sem.SubqueryExpr{ Node: n, Correlated: isCorrelated(seq), + SQL: t.scope.sql != nil, Body: seq, - }, typ + } + return e, t.checker.subquery(e, typ) } return nil, nil } @@ -888,12 +890,14 @@ func (t *translator) maybeSubqueryCall(call *ast.CallExpr, name string, inType s // (i.e., starts with a from), in which case we put the uncorrelated body // in the subquery without the correlating values logic. This is controlled // via the Correlated flag. - return &sem.SubqueryExpr{ + e := &sem.SubqueryExpr{ Node: call, Array: false, Correlated: isCorrelated(userOp), + SQL: t.scope.sql != nil, Body: userOp, - }, typ + } + return e, t.checker.subquery(e, typ) } func (t *translator) semMapCall(call *ast.CallExpr, args []sem.Expr, argTypes []super.Type) (sem.Expr, super.Type) { @@ -1341,14 +1345,13 @@ func (t *translator) arraySubquery(elems []sem.ArrayElem) *sem.SubqueryExpr { func (t *translator) subqueryExpr(astExpr ast.Expr, array bool, body ast.Seq, inType super.Type) (*sem.SubqueryExpr, super.Type) { // We pass inType in whether or not it's correlated since an uncorrelated // subquery will just ignore it. - t.subqueries++ seq, outType := t.seq(body, inType) - t.subqueries-- correlated := isCorrelated(seq) e := &sem.SubqueryExpr{ Node: astExpr, Array: array, Correlated: correlated, + SQL: t.scope.sql != nil, Body: seq, } // Add a nullscan only for uncorrelated queries that don't have a source. @@ -1360,10 +1363,7 @@ func (t *translator) subqueryExpr(astExpr ast.Expr, array bool, body ast.Seq, in // condition with this complex cleanup... e.Body, outType = t.scalarSubqueryCheck(astExpr, e.Body, outType) } - if array { - outType = t.sctx.LookupTypeArray(outType) - } - return e, outType + return e, t.checker.subquery(e, outType) } func (t *translator) scalarSubqueryCheck(n ast.Node, seq sem.Seq, thisType super.Type) (sem.Seq, super.Type) { diff --git a/compiler/semantic/nonempty.go b/compiler/semantic/nonempty.go new file mode 100644 index 000000000..3fcc79872 --- /dev/null +++ b/compiler/semantic/nonempty.go @@ -0,0 +1,71 @@ +package semantic + +import ( + "slices" + + "github.com/superdb/super/compiler/semantic/sem" +) + +// mayBeEmpty returns false if the body of subquery e always produces a value, +// e.g., (values a) or (sum(x)), and true if it might not, in which case the +// value of a pipe subquery is none. It's conservative: a body that's not +// known to produce a value may be empty. +func mayBeEmpty(e *sem.SubqueryExpr) bool { + // A correlated subquery runs on the value of its outer expression. + nonEmpty := e.Correlated + for _, op := range e.Body { + nonEmpty = producesValue(op, nonEmpty) + } + return !nonEmpty +} + +// producesValue returns true if op is known to produce a value when its input +// is nonEmpty. +func producesValue(op sem.Op, nonEmpty bool) bool { + switch op := op.(type) { + case *sem.NullScan: + return true + case *sem.AggregateOp: + // An aggregate without keys produces one value even with no + // input, and with keys, a value per group. + return len(op.Keys) == 0 || nonEmpty + case *sem.ValuesOp: + // A value that can't be a plain none isn't dropped. + return nonEmpty && slices.ContainsFunc(op.Exprs, notNone) + case *sem.HeadOp: + return nonEmpty && op.Count > 0 + case *sem.TailOp: + return nonEmpty && op.Count > 0 + case *sem.CountOp, *sem.CutOp, *sem.DebugOp, *sem.DistinctOp, *sem.DropOp, + *sem.FuseOp, *sem.PassOp, *sem.PutOp, *sem.RenameOp, *sem.SortOp, + *sem.UniqOp: + // These produce a value (possibly an error) for some input. + return nonEmpty + } + return false +} + +// notNone returns true if e is known not to be a plain none. +func notNone(e sem.Expr) bool { + switch e := e.(type) { + case *sem.PrimitiveExpr: + return e.Value != "none" + case *sem.ArrayExpr, *sem.MapExpr, *sem.RecordExpr, *sem.SetExpr: + return true + case *sem.DotExpr: + return !e.Noneish && notNone(e.LHS) + case *sem.IndexExpr: + // An index out of range is an error. + return notNone(e.Expr) + case *sem.ThisExpr: + // A field of a record and a value input to an operator are + // never plain nones. + for _, elem := range e.Chain { + if elem.Noneish { + return false + } + } + return true + } + return false +} diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index 85c390c19..0d153a0b6 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -1171,9 +1171,8 @@ func (t *translator) semOp(o ast.Op, seq sem.Seq, inType super.Type) (sem.Seq, s exprs, types := t.exprs(o.Exprs, inType) for k, typ := range types { t.checker.result(o.Exprs[k], typ) - t.output(o.Exprs[k], typ) } - return append(seq, sem.NewValues(o, exprs...)), t.checker.fuse(types) + return append(seq, sem.NewValues(o, exprs...)), t.checker.values(types) case *ast.Script: t.error(o, errors.New("a script of multiple statements cannot be used here")) return append(seq, badOp), badType @@ -1380,13 +1379,24 @@ func (t *translator) singletonKey(agg ast.Assignment, seq sem.Seq, inType super. if !ok || len(this.Chain) != 1 { return nil, nil } + // A plain none key disappears from the aggregate's record (see the + // aggregate operator), so the key's value is none when the record + // doesn't have its field, and values drops it. (ok() would turn an error + // key into none too.) + name := &sem.PrimitiveExpr{Node: out.Node, Value: sup.QuotedString(this.Chain[0].ID)} + key := &sem.CondExpr{ + Node: out.Node, + Cond: sem.NewCall(out.Node, "has", []sem.Expr{sem.NewThis(out.Node, nil), name}), + Then: this, + Else: &sem.PrimitiveExpr{Node: out.Node, Value: "none"}, + } return append(seq, &sem.AggregateOp{ Node: out.Node, Keys: []sem.Assignment{out}, }, - sem.NewValues(out.Node, this), - ), path.typ + sem.NewValues(out.Node, key), + ), t.checker.values([]super.Type{path.typ}) } // semDecls enters a block of declarations into the current scope. We do late binding @@ -1553,6 +1563,10 @@ func (t *translator) typeQueryDecl(d *ast.TypeQueryDecl) { t.error(d, fmt.Errorf("type query for %q: %s", d.Name.Name, sup.FormatValue(val))) return } + if val.Type() == super.TypeNone { + t.error(d, fmt.Errorf("type query for %q must yield a single type value but yielded no values", d.Name.Name)) + return + } if val.Type() != super.TypeType || val.IsNull() { t.error(d, fmt.Errorf("type query for %q must yield a single type value: %s", d.Name.Name, sup.FormatValue(val))) return @@ -1709,10 +1723,10 @@ func (t *translator) exprOp(e ast.Expr, seq sem.Seq, inType super.Type) (sem.Seq if t.isBool(out) { return append(seq, &sem.FilterOp{Node: e, Expr: out}), t.checker.narrow(inType, out) } - // As with the values operator, the value must not be missing or none. + // As with the values operator, the value must not be missing and a + // plain none is dropped. t.checker.result(e, typ) - t.output(e, typ) - return append(seq, sem.NewValues(e, out)), typ + return append(seq, sem.NewValues(e, out)), t.checker.values([]super.Type{typ}) } func (t *translator) isBool(e sem.Expr) bool { @@ -2022,21 +2036,6 @@ func (t *translator) maybeEval(e sem.Expr) (super.Value, bool) { return newEvaluator(t, t.resolver.funcs).maybeEval(t.sctx, e) } -// output reports an error at loc if typ, the type of a value output by the -// values operator, may be a plain none (e.g., from ok() or ?.). A plain none -// disappears from a record, array, or map but otherwise must be handled with -// ?? before it leaves the operator that produced it, i.e., a plain none is -// never passed to a downstream operator, even one that would handle it. -// (A plain none that reaches the query output at runtime is an error.) The -// result of an expression subquery may be none since it's an expression -// value. -func (t *translator) output(loc ast.Node, typ super.Type) { - if t.subqueries > 0 { - return - } - t.checker.output(loc, typ) -} - // unnestExpr translates the expression of an unnest operator and returns it, // its type, and, if the unnest is named ("unnest name:=expr") or has a with // clause ("with assignments"), the name of the field to unnest. In that diff --git a/compiler/semantic/sem/expr.go b/compiler/semantic/sem/expr.go index 5632ef537..ce66492f0 100644 --- a/compiler/semantic/sem/expr.go +++ b/compiler/semantic/sem/expr.go @@ -133,7 +133,10 @@ type ( ast.Node Correlated bool Array bool - Body Seq + // SQL is true for a subquery in a SQL expression (see + // dag.SubqueryExpr). + SQL bool + Body Seq } ThisExpr struct { ast.Node @@ -462,6 +465,7 @@ func CopyExpr(e Expr) Expr { Node: e.Node, Correlated: e.Correlated, Array: e.Array, + SQL: e.SQL, Body: CopySeq(e.Body), } case *ThisExpr: diff --git a/compiler/semantic/ztests/agg-missing.yaml b/compiler/semantic/ztests/agg-missing.yaml index ac8e642f3..9dae20952 100644 --- a/compiler/semantic/ztests/agg-missing.yaml +++ b/compiler/semantic/ztests/agg-missing.yaml @@ -86,7 +86,8 @@ error: | --- -# With ok(), the absent values form the none group. +# With ok(), the absent values form the none group, whose record has no y +# since a plain none disappears from a record. spq: aggregate sum(x) by y.ok() | sort y input: | @@ -96,7 +97,7 @@ input: | output: | {y:"a",sum:4} - {y:none,sum:2} + {sum:2} --- diff --git a/compiler/semantic/ztests/imdb.yaml b/compiler/semantic/ztests/imdb.yaml index 42ae636fd..770ba72e1 100644 --- a/compiler/semantic/ztests/imdb.yaml +++ b/compiler/semantic/ztests/imdb.yaml @@ -14,10 +14,10 @@ inputs: | values {...p,name:name.primaryName} ) values { - title: title.primaryTitle, + title: title?.primaryTitle, director: (people 'director' | collect(name)), writer: (people 'writer' | collect(name)), - genres: title.genres, + genres: title?.genres, actors: (people 'actor' | order by ordering | limit 10 diff --git a/compiler/semantic/ztests/none-absent.yaml b/compiler/semantic/ztests/none-absent.yaml new file mode 100644 index 000000000..28850b4b3 --- /dev/null +++ b/compiler/semantic/ztests/none-absent.yaml @@ -0,0 +1,39 @@ +# Since plain nones are dropped, the type checker must still catch a none that +# can reach an expression that doesn't handle it: +# - a pipe subquery that may produce no values is none, +# - a record field or aggregate key whose value may be none may be absent, +# - and a none of a named type is a value that's kept. +script: | + echo === subquery + ! super -s -c 'values {a:1},{b:1} | values {c:(values a.ok())} | values c+1' 2> /dev/null + ! super -s -c 'values 1 | values (values 2 | where false) + 1' 2> /dev/null + super -s -c 'values 1 | values (values 2) + 1, (values this | sum(this)) + 1, (values [this][0]) + 1' + super -s -c 'values 1 | values ((values 2 | where false) ?? 0) + 1' + echo === absent + ! super -s -c 'values {a:1},{b:2} | count() by k:=a.ok() | values k' 2> /dev/null + super -s -c 'values {a:1},{b:2} | count() by k:=a.ok() | values ok(k) ?? 0 | sort this' + ! super -s -c 'values {a:1},{b:2} | values {k:a.ok()} | values k' 2> /dev/null + super -s -c 'values {a:1},{b:2} | values {k:a.ok()} | values ok(k)' + echo === named + ! super -s -c 'type u=int64|none values {a:1},{b:2} | values ok(a)::u | values this+1' 2> /dev/null + super -s -c 'type u=int64|none values {a:1},{b:2} | values ok(a)::u' + super -s -c 'type n=none values {a:none::n,b:1} | values a' + +outputs: + - name: stdout + data: | + === subquery + 3 + 2 + 2 + 1 + === absent + 0 + 1 + 1 + === named + type u=int64|none + 1::u + none::u + type n=none + none::n diff --git a/compiler/semantic/ztests/none-output.yaml b/compiler/semantic/ztests/none-output.yaml index 4c3b0ac9a..47d7d6c01 100644 --- a/compiler/semantic/ztests/none-output.yaml +++ b/compiler/semantic/ztests/none-output.yaml @@ -22,12 +22,20 @@ output: | --- -# Statically, a value that may be a plain none (e.g., from ?. or ok()) can't -# be output and must be handled with ??. +# A plain none (e.g., from ?. or ok()) output by the values operator is +# dropped, so it isn't passed downstream, unless it's handled +# with ??. script: | super -o in.bsup in.sup - ! super -s -c 'from in.bsup | values org?.id' - ! super -s -c 'from in.bsup | values y.ok()' + echo === + super -s -c 'from in.bsup | values org?.id' + echo === + super -s -c 'from in.bsup | values y.ok()' + echo === + super -s -c 'from in.bsup | values y.ok() | values typeof(this)' + echo === + super -s -c 'from in.bsup | y.ok()' + echo === super -s -c 'from in.bsup | values org?.id ?? 0, {y:y.ok()}' inputs: @@ -39,22 +47,23 @@ inputs: outputs: - name: stdout data: | + === + 1 + === + 2 + === + + === + 2 + === 0 {} 1 {y:2} - - name: stderr - data: | - none not handled (consider ?? or ?.) at line 1, column 23: - from in.bsup | values org?.id - ~~~~~~~ - none not handled (consider ?? or ?.) at line 1, column 23: - from in.bsup | values y.ok() - ~~~~~~ --- -# At runtime, a plain none output is an error. +# At runtime, a plain none output is dropped. spq: values org?.id flags: -dynamic @@ -64,18 +73,15 @@ input: | {org?:{id:1}} output: | - error("none not handled (consider ?? or ?.)") 1 --- -# A none literal can't be output. -spq: values none +# A none literal is a plain none, so it's dropped. +spq: values none, 1 -error: | - none not handled (consider ?? or ?.) at line 1, column 8: - values none - ~~~~ +output: | + 1 --- @@ -111,8 +117,8 @@ output: | --- -# At runtime, a union whose type has a none member keeps its type for values -# that aren't plain nones. (Statically, the type may be none so it's an error.) +# A union whose type has a none member keeps its type for values that +# aren't plain nones. spq: values this flags: -dynamic diff --git a/compiler/semantic/ztests/none.yaml b/compiler/semantic/ztests/none.yaml index d6d20e081..ea9c184b3 100644 --- a/compiler/semantic/ztests/none.yaml +++ b/compiler/semantic/ztests/none.yaml @@ -144,7 +144,7 @@ output: | --- -# The runtime ?. passes the none from ok() through, which is an error when +# The runtime ?. passes the none from ok() through, which is dropped when # it's output. spq: values r.ok()?.f @@ -156,7 +156,6 @@ input: | output: | 1 - error("none not handled (consider ?? or ?.)") --- diff --git a/runtime/expr/function/function.go b/runtime/expr/function/function.go index 0f0575e94..552d9ac8e 100644 --- a/runtime/expr/function/function.go +++ b/runtime/expr/function/function.go @@ -80,6 +80,10 @@ func New(sctx *super.Context, name string, narg int) (expr.Function, error) { case "grok": argmin, argmax = 2, 3 f = newGrok(sctx) + case "has": + argmin = 2 + argmax = 2 + f = &Has{sctx: sctx} case "has_error": f = HasError{sctx} case "hex": @@ -262,7 +266,7 @@ func (f *samFunc) Call(args ...vector.Any) vector.Any { // signatures so the return type can be introspected. func HasBoolResult(name string) bool { switch name { - case "grep", "has_error", "is", "is_error", "is_ok", "missing", "cidr_match": + case "grep", "has", "has_error", "is", "is_error", "is_ok", "missing", "cidr_match": return true } return false diff --git a/runtime/expr/function/has.go b/runtime/expr/function/has.go new file mode 100644 index 000000000..b62d61aba --- /dev/null +++ b/runtime/expr/function/has.go @@ -0,0 +1,93 @@ +package function + +import ( + "bytes" + + "github.com/superdb/super" + "github.com/superdb/super/scode" + "github.com/superdb/super/vector" + "github.com/superdb/super/vector/bitvec" +) + +// Has implements has(val, key), which is true if +// - val is a record whose type has a field named key, +// - val is a type value of a record type with a field named key, or +// - val is a map with the key key. +// +// For a record, the answer depends only on its type, so it's true for a +// field whose value is null, an error, or an option's none. (A plain none +// never appears in a record.) It's false for any other value. +type Has struct { + sctx *super.Context + mb scode.Builder + kb scode.Builder +} + +func (h *Has) Call(args ...vector.Any) vector.Any { + val, key := args[0], vector.Under(args[1]) + if key.Kind() == vector.KindError { + return key + } + n := val.Len() + switch typ := super.TypeUnder(val.Type()).(type) { + case *super.TypeRecord: + if err := h.checkFieldName(key); err != nil { + return err + } + if c, ok := key.(*vector.Const); ok { + return vector.NewConstBool(typ.HasField(vector.StringValue(c, 0)), n) + } + return h.eachSlot(n, func(slot uint32) bool { + return typ.HasField(vector.StringValue(key, slot)) + }) + case *super.TypeMap: + return h.eachSlot(n, func(slot uint32) bool { + return h.hasMapKey(typ, val, key, slot) + }) + } + if val.Kind() == vector.KindType && key.Kind() == vector.KindString { + // A type value is tested by its record type. Any other type + // value, or a key that isn't a string, is false. + return h.eachSlot(n, func(slot uint32) bool { + rec, ok := super.TypeUnder(vector.TypeValueValue(val, slot)).(*super.TypeRecord) + return ok && rec.HasField(vector.StringValue(key, slot)) + }) + } + return vector.NewConstBool(false, n) +} + +func (h *Has) checkFieldName(key vector.Any) vector.Any { + if key.Kind() != vector.KindString { + return vector.NewWrappedError(h.sctx, "has: field name must be a string", key) + } + return nil +} + +func (h *Has) eachSlot(n uint32, fn func(uint32) bool) vector.Any { + bits := bitvec.NewFalse(n) + for slot := range n { + if fn(slot) { + bits.Set(slot) + } + } + return vector.NewBool(bits) +} + +// hasMapKey returns true if the map at slot of vec has the key at slot of +// key. Keys are compared by value after removing named and union types. +func (h *Has) hasMapKey(typ *super.TypeMap, vec, key vector.Any, slot uint32) bool { + m := vector.ValueAt(&h.mb, vec, slot) + if m.IsNull() { + return false + } + body := m.Bytes() + k := vector.ValueAt(&h.kb, key, slot).Under() + for it := body.Iter(); !it.Done(); { + mk := super.NewValue(typ.KeyType, it.Next()).Under() + it.Next() + if mk.Type() == k.Type() && bytes.Equal(mk.Bytes(), k.Bytes()) { + return true + } + } + return false +} diff --git a/runtime/op/aggregate/aggregate.go b/runtime/op/aggregate/aggregate.go index b85009f8c..3f00d1b72 100644 --- a/runtime/op/aggregate/aggregate.go +++ b/runtime/op/aggregate/aggregate.go @@ -32,6 +32,12 @@ func New(parent vio.Puller, sctx *super.Context, aggNames []field.Path, aggExprs if err != nil { return nil, err } + // A plain none group key (e.g., from ok() or ?.) disappears from an + // output record. It's kept in a partial so the downstream aggregate can + // group by it. + if !partialsOut { + builder.NoneDrops = len(keyNames) + } return &Aggregate{ parent: parent, sctx: sctx, diff --git a/runtime/op/cachedsubquery.go b/runtime/op/cachedsubquery.go index cfbcd1b67..f47ce59ec 100644 --- a/runtime/op/cachedsubquery.go +++ b/runtime/op/cachedsubquery.go @@ -11,11 +11,15 @@ import ( type cachedSubquery struct { sctx *super.Context body vio.Puller + sql bool cached vector.Any } -func NewCachedSubquery(sctx *super.Context, body vio.Puller) expr.Evaluator { - return &cachedSubquery{sctx: sctx, body: body} +// NewCachedSubquery returns an uncorrelated subquery, which is evaluated once. +// If it produces no values, its value is null if sql is true (as in SQL) and +// otherwise none. +func NewCachedSubquery(sctx *super.Context, body vio.Puller, sql bool) expr.Evaluator { + return &cachedSubquery{sctx: sctx, body: body, sql: sql} } func (c *cachedSubquery) Eval(this vector.Any) vector.Any { @@ -40,7 +44,7 @@ func (c *cachedSubquery) exec() vector.Any { vecs = append(vecs, vec) } if len(vecs) == 0 { - return vector.NewNull(1) + return emptyResult(c.sql) } if len(vecs) == 1 && vecs[0].Len() == 1 { return vecs[0] diff --git a/runtime/op/debug.go b/runtime/op/debug.go index b5741089f..c159b7e4a 100644 --- a/runtime/op/debug.go +++ b/runtime/op/debug.go @@ -56,7 +56,8 @@ func (d *Debug) Pull(done bool) (vector.Any, error) { if d.filter != nil { filtered, _ = applyMask(val, d.filter.Eval(filtered)) } - if e := d.expr.Eval(filtered); e.Len() != 0 { + // A plain none is dropped, as for the values operator. + if e := dropPlainNones(d.expr.Eval(filtered)); e != nil && e.Len() != 0 { select { case local <- e: case <-d.ctx.Done(): diff --git a/runtime/op/nones.go b/runtime/op/nones.go new file mode 100644 index 000000000..2116c7cf0 --- /dev/null +++ b/runtime/op/nones.go @@ -0,0 +1,143 @@ +package op + +import ( + "slices" + + "github.com/superdb/super" + "github.com/superdb/super/vector" +) + +// dropPlainNones returns vec without its plain none values or nil if all of +// its values are plain nones. A plain none comes from an expression like +// ok() or ?. that handles a none. It disappears from a record, array, or +// map and, as a value output by the values operator or the query, is +// dropped. A none of a named type and an option's none are values and +// are kept. +func dropPlainNones(vec vector.Any) vector.Any { + if vec == nil { + return nil + } + nones, all := plainNoneSlots(vec) + n := vec.Len() + if all || len(nones) == int(n) { + return nil + } + if len(nones) == 0 { + return vec + } + rest := make([]uint32, 0, int(n)-len(nones)) + var k int + for slot := range n { + if k < len(nones) && nones[k] == slot { + k++ + continue + } + rest = append(rest, slot) + } + return vector.Pick(vec, rest) +} + +// plainNoneSlots returns the slots of vec, in order, whose values are plain +// nones, or true if all of its values are plain nones, in which case the +// slots aren't returned. +func plainNoneSlots(vec vector.Any) ([]uint32, bool) { + switch v := vec.(type) { + case *vector.Dynamic: + // The values of a dynamic may themselves be dynamics. + marks := make([][]bool, len(v.Values)) + allNone := make([]bool, len(v.Values)) + var found bool + for k, val := range v.Values { + if val == nil { + continue + } + slots, all := plainNoneSlots(val) + if all { + allNone[k] = true + found = true + } else if len(slots) != 0 { + marks[k] = markSlots(val.Len(), slots) + found = true + } + } + if !found { + return nil, false + } + var nones []uint32 + counts := make([]uint32, len(v.Values)) + for slot, tag := range v.Tags { + if allNone[tag] || marks[tag] != nil && marks[tag][counts[tag]] { + nones = append(nones, uint32(slot)) + } + counts[tag]++ + } + return nones, false + case *vector.Union: + // Check the type first so the tags of a union without a none + // member aren't loaded. + if !hasPlainNoneMember(v.Typ) { + return nil, false + } + tags := v.Tags() + values := v.Values() + var nones []uint32 + for slot, tag := range tags { + if isNoneVec(values[tag]) { + nones = append(nones, uint32(slot)) + } + } + return nones, false + case *vector.Fusion: + // A fusion value is a plain none if its subtype is none, which is + // possible only if the fusion's type has none. Check the type first + // so the subtypes needn't be loaded. + if typ, ok := v.Typ.Type.(*super.TypeUnion); !ok || !hasPlainNoneMember(typ) { + if v.Typ.Type != super.TypeNone { + return nil, false + } + } + var nones []uint32 + for slot, typ := range v.Subtypes.Types() { + if typ == super.TypeNone { + nones = append(nones, uint32(slot)) + } + } + return nones, false + case *vector.View: + slots, all := plainNoneSlots(v.Any) + if all { + return nil, true + } + if len(slots) == 0 { + return nil, false + } + marks := markSlots(v.Any.Len(), slots) + var nones []uint32 + for slot, index := range v.Index { + if marks[index] { + nones = append(nones, uint32(slot)) + } + } + return nones, false + } + return nil, isNoneVec(vec) +} + +// hasPlainNoneMember reports whether typ has plain none as a member. +func hasPlainNoneMember(typ *super.TypeUnion) bool { + return slices.ContainsFunc(typ.Types, func(t super.Type) bool { return t == super.TypeNone }) +} + +func markSlots(n uint32, slots []uint32) []bool { + marks := make([]bool, n) + for _, slot := range slots { + marks[slot] = true + } + return marks +} + +// isNoneVec reports whether vec is a plain none. (A none of a named type +// is a value.) +func isNoneVec(vec vector.Any) bool { + return vec != nil && vec.Type() == super.TypeNone +} diff --git a/runtime/op/output.go b/runtime/op/output.go index 45094835c..647c95269 100644 --- a/runtime/op/output.go +++ b/runtime/op/output.go @@ -1,109 +1,30 @@ package op import ( - "slices" - - "github.com/superdb/super" "github.com/superdb/super/vector" "github.com/superdb/super/vector/vio" ) -// Output is the puller for an output of a query. It turns any plain none -// values into errors. A plain none comes from an operator like ok() or ?. -// that handles a none so it can disappear from a record, array, or map, but -// a query must not output one (it should be handled with ??). A none of a -// named type is a value and is output as is. +// Output is the puller for an output of a query. It drops any plain none +// values, which come from an expression like ok() or ?. that handles a none +// (see dropPlainNones). A none of a named type is a value and is output +// as is. type Output struct { - sctx *super.Context parent vio.Puller } -func NewOutput(sctx *super.Context, parent vio.Puller) *Output { - return &Output{sctx: sctx, parent: parent} +func NewOutput(parent vio.Puller) *Output { + return &Output{parent: parent} } func (o *Output) Pull(done bool) (vector.Any, error) { - vec, err := o.parent.Pull(done) - if vec == nil || err != nil { - return vec, err - } - return o.noneToError(vec), nil -} - -const noneErr = "none not handled (consider ?? or ?.)" - -func (o *Output) noneToError(vec vector.Any) vector.Any { - switch v := vec.(type) { - case *vector.None: - return vector.NewStringError(o.sctx, noneErr, v.Len()) - case *vector.Dynamic: - // The values of a dynamic may themselves be dynamics. - var vals []vector.Any - for k, val := range v.Values { - if val == nil { - continue - } - if out := o.noneToError(val); out != val { - if vals == nil { - vals = slices.Clone(v.Values) - } - vals[k] = out - } - } - if vals == nil { - return vec - } - return vector.NewDynamic(v.Tags, vals) - case *vector.Union: - // Only the slots that are plain nones become errors. The rest - // keep the union type. - if !slices.ContainsFunc(v.Values(), isNoneVec) { - return vec - } - values := v.Values() - var index, rest []uint32 - for i, tag := range v.Tags() { - if isNoneVec(values[tag]) { - index = append(index, uint32(i)) - } else { - rest = append(rest, uint32(i)) - } + for { + vec, err := o.parent.Pull(done) + if vec == nil || err != nil { + return vec, err } - return o.slotsToError(vec, index, rest) - case *vector.Fusion: - // A fusion value is a plain none if its subtype is none. - var index, rest []uint32 - for i, typ := range v.Subtypes.Types() { - if typ == super.TypeNone { - index = append(index, uint32(i)) - } else { - rest = append(rest, uint32(i)) - } + if vec = dropPlainNones(vec); vec != nil { + return vec, nil } - return o.slotsToError(vec, index, rest) - case *vector.View: - if isNoneVec(v.Any) { - return vector.NewStringError(o.sctx, noneErr, v.Len()) - } - } - return vec -} - -// slotsToError returns vec with the slots in index replaced by errors and the -// slots in rest left as is. -func (o *Output) slotsToError(vec vector.Any, index, rest []uint32) vector.Any { - if len(index) == 0 { - return vec } - errs := vector.NewStringError(o.sctx, noneErr, uint32(len(index))) - if len(rest) == 0 { - return errs - } - return vector.Combine(vector.Pick(vec, rest), index, errs) -} - -// isNoneVec reports whether vec is a plain none. (A none of a named type -// is a value.) -func isNoneVec(vec vector.Any) bool { - return vec != nil && vec.Type() == super.TypeNone } diff --git a/runtime/op/subquery.go b/runtime/op/subquery.go index a04fe279c..2027bb186 100644 --- a/runtime/op/subquery.go +++ b/runtime/op/subquery.go @@ -14,10 +14,12 @@ import ( // the expression. When Eval is called, it repeatedly constructs a single-slot // vector from the current slot of the this vector, sends it to the embedded // query, pulls from the query until EOS, appends the value to the result -// vector, and then advances to the next slot. +// vector, and then advances to the next slot. If the query produces no +// values, the result is null if sql is true (as in SQL) and otherwise none. type Subquery struct { ctx context.Context sctx *super.Context + sql bool create func() *Subquery body vio.Puller @@ -28,10 +30,11 @@ type Subquery struct { tos int } -func NewSubquery(ctx context.Context, sctx *super.Context, create func() *Subquery) *Subquery { +func NewSubquery(ctx context.Context, sctx *super.Context, sql bool, create func() *Subquery) *Subquery { return &Subquery{ ctx: ctx, sctx: sctx, + sql: sql, create: create, ch: make(chan vector.Any, 1), tos: -2, @@ -83,7 +86,7 @@ func (s *Subquery) bodyPull() vector.Any { return vector.NewStringError(s.sctx, err.Error(), 1) } if vec == nil { - return vector.NewNull(1) + return emptyResult(s.sql) } vec2, err := s.body.Pull(false) if err != nil { @@ -113,3 +116,12 @@ func (s *Subquery) Pull(done bool) (vector.Any, error) { return nil, s.ctx.Err() } } + +// emptyResult returns the value of a subquery that produces no values: null +// in a SQL expression and otherwise none. +func emptyResult(sql bool) vector.Any { + if sql { + return vector.NewNull(1) + } + return vector.NewNone(1) +} diff --git a/runtime/op/unnest.go b/runtime/op/unnest.go index 568f230f8..861da24e8 100644 --- a/runtime/op/unnest.go +++ b/runtime/op/unnest.go @@ -1,8 +1,6 @@ package op import ( - "slices" - "github.com/superdb/super" "github.com/superdb/super/runtime/expr" "github.com/superdb/super/vector" @@ -41,7 +39,11 @@ func (u *Unnest) Pull(done bool) (vector.Any, error) { vec = vector.Apply(vector.ApplyRipUnions, func(vecs ...vector.Any) vector.Any { return vecs[0] }, u.defuse.Eval(u.expr.Eval(vec))) - if vec, _ = u.flatten(vec, u.field != ""); vec != nil { + // An unnested value may be a plain none (e.g., an element of an + // array of type [int64|none]), which is dropped as for the values + // operator. + vec, _ = u.flatten(vec, u.field != "") + if vec = dropPlainNones(vec); vec != nil { return vec, nil } } @@ -50,12 +52,17 @@ func (u *Unnest) Pull(done bool) (vector.Any, error) { // flatten unnests vec, which is a record whose field named u.field is // unnested if record is true and otherwise an array or set, and returns the // unnested values and the offsets of the values of each slot of vec. It -// returns nil values for nulls and nil offsets for errors, which map -// one-to-one to the slots of vec. +// returns nil values for nulls and plain nones, which produce no values, and +// nil offsets for errors, which map one-to-one to the slots of vec. func (u *Unnest) flatten(vec vector.Any, record bool) (vector.Any, []uint32) { if dynamic, ok := vec.(*vector.Dynamic); ok { return u.flattenDynamic(dynamic, record) } + if vec.Len() == 0 || isNoneVec(vec) { + // An empty vector (e.g., a member of a union with no values) + // has nothing to unnest. + return nil, nil + } vec = vector.Under(vec) switch vec.Kind() { case vector.KindNull: @@ -90,6 +97,8 @@ func (u *Unnest) flattenDynamic(vec *vector.Dynamic, record bool) (vector.Any, [ var tags []uint32 for _, tag := range vec.Tags { if vecs[tag] == nil { + // A null or plain none produces no values. + offsets = append(offsets, length) continue } if offsets := vecOffsets[tag]; offsets != nil { @@ -105,6 +114,10 @@ func (u *Unnest) flattenDynamic(vec *vector.Dynamic, record bool) (vector.Any, [ } offsets = append(offsets, length) } + if length == 0 { + // Every slot is a null or plain none, which produce no values. + return nil, nil + } return vector.NewDynamic(tags, vecs), offsets } @@ -114,8 +127,9 @@ func (u *Unnest) flattenRecord(vec *vector.Record) (vector.Any, []uint32) { k, ok := vec.Typ.IndexOfField(u.field) if !ok { // The field is absent since its value is a plain none, which - // disappears from a record, so report it as for an unnamed unnest. - return vector.NewWrappedError(u.sctx, "unnest: encountered non-array value", vector.NewNone(vec.Len())), nil + // disappears from a record, so as for an unnamed unnest, there's + // nothing to unnest. + return nil, nil } if union, ok := vec.Fields[k].(*vector.Union); ok { dynamic := union.Dynamic() @@ -161,10 +175,17 @@ func (u *Unnest) flattenRecord(vec *vector.Record) (vector.Any, []uint32) { // newRecord returns a record vector of the fields of typ whose values are // vecs, where the type of each field is the type of its value. All vecs // have the same length. +// A field whose value is a plain none disappears, as in a record expression. func (u *Unnest) newRecord(typ *super.TypeRecord, vecs []vector.Any) vector.Any { - fields := slices.Clone(typ.Fields) + n := vecs[0].Len() + fields := make([]super.Field, 0, len(vecs)) + out := make([]vector.Any, 0, len(vecs)) for j, vec := range vecs { - fields[j].Type = vec.Type() + if isNoneVec(vec) { + continue + } + fields = append(fields, super.NewField(typ.Fields[j].Name, vec.Type())) + out = append(out, vec) } - return vector.NewRecord(u.sctx.MustLookupTypeRecord(fields), vecs, vecs[0].Len()) + return vector.NewRecord(u.sctx.MustLookupTypeRecord(fields), out, n) } diff --git a/runtime/op/values.go b/runtime/op/values.go index c86c7509d..468b5fcc3 100644 --- a/runtime/op/values.go +++ b/runtime/op/values.go @@ -15,6 +15,8 @@ type Values struct { var _ vio.Puller = (*Values)(nil) +// NewValues returns a Values that outputs the values of exprs. Plain none +// values are dropped (see dropPlainNones). func NewValues(sctx *super.Context, parent vio.Puller, exprs []expr.Evaluator) *Values { return &Values{ sctx: sctx, @@ -33,12 +35,18 @@ func (v *Values) Pull(done bool) (vector.Any, error) { for _, e := range v.exprs { vals = append(vals, e.Eval(val)) } - if len(vals) == 1 { - return vals[0], nil - } else if len(vals) != 0 { - return interleave(vals), nil + var out vector.Any + switch len(vals) { + case 0: + continue + case 1: + out = vals[0] + default: + out = interleave(vals) + } + if out = dropPlainNones(out); out != nil { + return out, nil } - // If no vals, continue the loop. } } diff --git a/runtime/ztests/expr/function/has.yaml b/runtime/ztests/expr/function/has.yaml new file mode 100644 index 000000000..09cc39c14 --- /dev/null +++ b/runtime/ztests/expr/function/has.yaml @@ -0,0 +1,89 @@ +spq: values has(this,"a") + +input: | + {a:1} + {b:1} + {a:null} + {a:error("x")} + {a?:none::int64} + 1 + null + error("x") + +output: | + true + false + true + true + true + false + false + false + +--- + +# A named type or union is tested by its underlying record type. +spq: | + type r={a:int64} + values {a:1}::r, cast({a:1},<{a:int64}|int64>), cast(1,<{a:int64}|int64>) + | values has(this,"a") + +output: | + true + true + false + +--- + +spq: values has(this,f) + +input: | + {a:1,f:"a"} + {a:1,f:"b"} + {a:1,f:1} + {a:1,f:error("x")} + +output: | + true + false + error({message:"has: field name must be a string",on:1}) + error("x") + +--- + +# A type value is tested by its record type. +spq: | + type r={a:int64} + values <{a:int64}>, <{b:int64}>, , + | values has(this,"a"), has(this,1) + +output: | + true + false + false + false + true + false + false + false + +--- + +# A map is tested for a key of any type. +spq: values has(this,"a"), has(this,1) + +input: | + map{"a":1,"b":2} + map{"b":2} + map{1:"x"} + map{1::(int64|string):"x","a"::(int64|string):"y"} + +output: | + true + false + false + false + false + true + true + true diff --git a/runtime/ztests/expr/function/under.yaml b/runtime/ztests/expr/function/under.yaml index b410f77a2..eb4e0333a 100644 --- a/runtime/ztests/expr/function/under.yaml +++ b/runtime/ztests/expr/function/under.yaml @@ -31,7 +31,6 @@ output: | 1 <{s:string}> 1 - error("none not handled (consider ?? or ?.)") --- diff --git a/runtime/ztests/expr/index.yaml b/runtime/ztests/expr/index.yaml index cd9951a20..c8fdff44b 100644 --- a/runtime/ztests/expr/index.yaml +++ b/runtime/ztests/expr/index.yaml @@ -59,7 +59,6 @@ output: | 1 2 1 - error("none not handled (consider ?? or ?.)") 127.0.0.1 --- @@ -107,7 +106,6 @@ output: | 5 error("cannot access field 'x' of set") "foo" - error("none not handled (consider ?? or ?.)") error({message:"invalid record index",on:1}) 123 @@ -143,8 +141,5 @@ input: | output: | "foo" "bar" - error("none not handled (consider ?? or ?.)") - error("none not handled (consider ?? or ?.)") - error("none not handled (consider ?? or ?.)") "foo" "foo" diff --git a/runtime/ztests/expr/subquery-empty.yaml b/runtime/ztests/expr/subquery-empty.yaml new file mode 100644 index 000000000..d69820307 --- /dev/null +++ b/runtime/ztests/expr/subquery-empty.yaml @@ -0,0 +1,21 @@ +# A subquery that produces no values is none in a pipe expression and null +# in a SQL expression, for both correlated and uncorrelated subqueries. +spq: | + values {c:(values this | where false)}, + {u:(values 1 | where false)}, + (values this | where false) ?? "none" + +input: | + 1 + +output: | + {} + {} + "none" + +--- + +spq: select (select 1 where false) as x + +output: | + {x:null} diff --git a/runtime/ztests/op/aggregate/none-key.yaml b/runtime/ztests/op/aggregate/none-key.yaml new file mode 100644 index 000000000..447586a55 --- /dev/null +++ b/runtime/ztests/op/aggregate/none-key.yaml @@ -0,0 +1,64 @@ +# A plain none group key (e.g., from ok()) disappears from its output record, +# including when the aggregate is split into partials (as for a file scan) +# and for a nested key. With a single unnamed key and no aggregate +# functions, the none key is dropped, but an error key isn't. An aggregate +# function's none result isn't a key, so it isn't dropped (as for shapes, +# whose sample is from any). Keys and aggregate values interleaved in +# nested records are assigned to the right fields. +script: | + super -s -c 'values {a:1},{b:2},{b:3} | count() by k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | count() by k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | count() by x.k:=ok(a) | sort count' + echo === + super -s -c 'from test.sup | aggregate by ok(a)' + echo === + super -s -c 'values {a:1},{b:2},{b:3} | aggregate by ok(a)' + echo === + super -s -c 'values {a:1},{b:2} | shapes ok(a)' + echo === + super -s -c 'values {a:1},{a:error("x")} | aggregate by a | sort this' + echo === + super -s -c 'from errors.sup | aggregate by a | sort this' + echo === + super -s -c 'values {a:1,b:3},{b:4} | aggregate r.c:=count(), y:=sum(b) by r.k:=b, x:=a.ok() | sort r.k' + +inputs: + - name: test.sup + data: | + {a:1} + {b:2} + {b:3} + - name: errors.sup + data: | + {a:1} + {a:error("x")} + {a:1} + +outputs: + - name: stdout + data: | + {k:1,count:1} + {count:2} + === + {k:1,count:1} + {count:2} + === + {x:{k:1},count:1} + {x:{},count:2} + === + 1 + === + 1 + === + 1 + === + 1 + error("x") + === + 1 + error("x") + === + {r:{k:3,c:1},x:1,y:3} + {r:{k:4,c:1},y:4} diff --git a/runtime/ztests/op/debug-none.yaml b/runtime/ztests/op/debug-none.yaml new file mode 100644 index 000000000..e9168d596 --- /dev/null +++ b/runtime/ztests/op/debug-none.yaml @@ -0,0 +1,11 @@ +# A plain none debug value is dropped, as for the values operator. +script: | + super -s -c 'values {a:1},{b:2} | debug ok(a) | count()' + +outputs: + - name: stdout + data: | + 2 + - name: stderr + data: | + 1 diff --git a/runtime/ztests/op/unnest.yaml b/runtime/ztests/op/unnest.yaml index cb0ce4878..049a9391d 100644 --- a/runtime/ztests/op/unnest.yaml +++ b/runtime/ztests/op/unnest.yaml @@ -73,7 +73,7 @@ input: | {x:"foo",y:[1,2]} error: | - unnest value must be an array at line 1, column 8: + unnest value must be an array or set at line 1, column 8: unnest this ~~~~ @@ -92,7 +92,7 @@ output: | --- -# A plain none to unnest is an error as for an unnamed unnest. +# A plain none to unnest produces no values as for an unnamed unnest. spq: unnest x:=o?.a with s flags: -dynamic @@ -102,5 +102,74 @@ input: | {o:some({a:[5]}),s:2} output: | - error({message:"unnest: encountered non-array value",on:none}) {x:5,s:2} + +--- + +# A plain none to unnest (e.g., from ok()) produces no values. +spq: unnest ok(a) + +input: | + {a:[1,2]} + {b:3} + {a:[4]} + +output: | + 1 + 2 + 4 + +--- + +# Statically, a plain none to unnest isn't an error. +spq: values {a:[1,2]},{b:3} | unnest ok(a) | values typeof(this) + +output: | + + + +--- + +# A plain none element or a null or plain none array produces no values, +# including for an unnest with a with clause. +spq: | + values [1::(int64|none), none::(int64|none)], none::([int64]|none) + | unnest this + | values typeof(this) + +output: | + + +--- + +spq: | + values + {x:null::([int64]|null),y:1}, + {x:[1,2]::([int64]|null),y:2}, + {x:none::([int64]|none),y:3}, + {x:[3]::([int64]|none),y:4} + | unnest x with y + | values this + +output: | + {x:1,y:2} + {x:2,y:2} + {x:3,y:4} + +--- + +# The type of an unnested value excludes plain none, which is dropped. +spq: | + values [1::(int64|none), none::(int64|none)] | unnest this | values this+1 + +output: | + 2 + +--- + +spq: | + values {s:set[1,2],y:3} | unnest s with y | values this + +output: | + {s:1,y:3} + {s:2,y:3} diff --git a/runtime/ztests/op/values/drop-none.yaml b/runtime/ztests/op/values/drop-none.yaml new file mode 100644 index 000000000..e33151262 --- /dev/null +++ b/runtime/ztests/op/values/drop-none.yaml @@ -0,0 +1,55 @@ +# The values operator drops plain nones, including those interleaved with +# the values of other expressions and those in a union, but keeps an +# option's none. +spq: values ok(a), 0 + +input: | + {a:1} + {b:2} + {a:"s"} + +output: | + 1 + 0 + 0 + "s" + 0 + +--- + +spq: values this + +input: | + 1::(int64|none) + none::(int64|none) + 2::(int64|none) + +output: | + 1::(int64|none) + 2::(int64|none) + +--- + +spq: values ok(a) ?? none::int64 + +input: | + {a:1} + {b:2} + +output: | + 1 + none::int64 + +--- + +# A dropped none isn't passed downstream. +spq: "values this==2 ? none : this | values typeof(this)" + +input: | + 1 + 2 + 3 + +output: | + + diff --git a/sio/bsupio/ztests/issue-5736.yaml b/sio/bsupio/ztests/issue-5736.yaml index 1e03f3e84..d1bbd588d 100644 --- a/sio/bsupio/ztests/issue-5736.yaml +++ b/sio/bsupio/ztests/issue-5736.yaml @@ -9,7 +9,7 @@ script: | did | count := sum(count), users := count() - by event + by event:=ok(event) | sort -r count" inputs: diff --git a/spq_test.go b/spq_test.go index a81731eab..8ce50f7ec 100644 --- a/spq_test.go +++ b/spq_test.go @@ -184,6 +184,11 @@ func runOneBoomerang(t *testing.T, format, data string) { } t.Fatalf("unexpected error writing %s baseline: %s", format, err) } + if baseline == "" { + // The data has no values (e.g., it's only plain nones, which are + // dropped). + t.Skip("skipping due to no values") + } readerFormat := format if readerFormat == "bsup -rows" { readerFormat = "bsup" @@ -308,8 +313,9 @@ func (f *fusionRemovingReader) Read() (*super.Value, error) { if f.h.hasFusion(val.Type()) { continue } - if val.Type() == super.TypeNone { - // A plain none can't be output by a query. + if val.Deunion().Type() == super.TypeNone { + // A plain none (even in a union) is dropped from the + // output of a query. continue } } diff --git a/sup/ztests/decorate-none.yaml b/sup/ztests/decorate-none.yaml index 8fcaf9f9b..dac6223dd 100644 --- a/sup/ztests/decorate-none.yaml +++ b/sup/ztests/decorate-none.yaml @@ -24,7 +24,6 @@ input: | missing output: | - error("none not handled (consider ?? or ?.)") [1,none] [some(1),none::int64] set[none::int64,some(1)] diff --git a/vector/recordbuilder.go b/vector/recordbuilder.go index d9164f636..cfb02fb15 100644 --- a/vector/recordbuilder.go +++ b/vector/recordbuilder.go @@ -10,29 +10,40 @@ import ( type RecordBuilder struct { sctx *super.Context base *rec + // NoneDrops is the number of leading fields (e.g., an aggregate's group + // keys) that disappear from a record when their values are plain nones + // (e.g., from ok() or ?.), as in a record expression. + NoneDrops int } +// NewRecordBuilder returns a builder of records with the leaf fields in +// fields. Paths with a common prefix are nested in the same record, in the +// order in which the prefix first appears. func NewRecordBuilder(sctx *super.Context, fields field.List) (*RecordBuilder, error) { base := &rec{} - for _, path := range fields { - if err := addPath(base, path); err != nil { + for k, path := range fields { + if err := addPath(base, path, k); err != nil { return nil, err } } return &RecordBuilder{sctx: sctx, base: base}, nil } +// New returns a record whose leaf fields are vecs in the order of the fields +// passed to NewRecordBuilder. func (r *RecordBuilder) New(vecs []Any) *Record { - rec, _ := r.base.build(r.sctx, vecs) - return rec + return r.base.build(r.sctx, vecs, r.NoneDrops) } type rec struct { paths []string - recs []*rec + // recs[k] is the record for paths[k] or nil if it's a leaf, in which + // case leaves[k] is the index of its value in the vectors passed to New. + recs []*rec + leaves []int } -func addPath(r *rec, path field.Path) error { +func addPath(r *rec, path field.Path, leaf int) error { for k, name := range path { idx := slices.Index(r.paths, name) if k == len(path)-1 { @@ -41,12 +52,14 @@ func addPath(r *rec, path field.Path) error { } r.paths = append(r.paths, path[k]) r.recs = append(r.recs, nil) + r.leaves = append(r.leaves, leaf) return nil } if idx == -1 { idx = len(r.paths) r.paths = append(r.paths, name) r.recs = append(r.recs, &rec{}) + r.leaves = append(r.leaves, -1) } if r.recs[idx] == nil { return &super.DuplicateFieldError{Name: path[:k+1].String()} @@ -56,19 +69,27 @@ func addPath(r *rec, path field.Path) error { return nil } -func (r *rec) build(sctx *super.Context, leafs []Any) (*Record, []Any) { +// build returns a record whose leaf fields are taken from leafs. A leaf field +// whose index is less than noneDrops and whose value is a plain none +// disappears from the record. +func (r *rec) build(sctx *super.Context, leafs []Any, noneDrops int) *Record { + n := leafs[0].Len() var fields []super.Field var out []Any for i, name := range r.paths { var vec Any if r.recs[i] != nil { - vec, leafs = r.recs[i].build(sctx, leafs) + vec = r.recs[i].build(sctx, leafs, noneDrops) } else { - vec, leafs = leafs[0], leafs[1:] + leaf := r.leaves[i] + vec = leafs[leaf] + if leaf < noneDrops && vec.Type() == super.TypeNone { + continue + } } fields = append(fields, super.NewField(name, vec.Type())) out = append(out, vec) } typ := sctx.MustLookupTypeRecord(fields) - return NewRecord(typ, out, out[0].Len()), leafs + return NewRecord(typ, out, n) }