diff --git a/book/src/super-sql/sql/join.md b/book/src/super-sql/sql/join.md index 2526db161..6c987712a 100644 --- a/book/src/super-sql/sql/join.md +++ b/book/src/super-sql/sql/join.md @@ -99,7 +99,7 @@ JOIN U ON x=z --- _Left outer join_ -```mdtest-spq-skip +```mdtest-spq # spq WITH T(x,y) AS ( VALUES (1,2), (3,4), (5,6) @@ -114,15 +114,15 @@ ORDER BY x # input # expected output -{x:1,y:2,z:error("missing")} +{x:1,y:2,z:null} {x:3,y:4,z:3} -{x:5,y:6,z:error("missing")} +{x:5,y:6,z:null} ``` --- _Right outer join_ -```mdtest-spq-skip +```mdtest-spq # spq WITH T(x,y) AS ( VALUES (1,2), (3,4), (5,6) @@ -138,7 +138,7 @@ ORDER BY x # expected output {x:3,y:4,z:3} -{x:error("missing"),y:error("missing"),z:2} +{x:null,y:null,z:2} ``` --- diff --git a/book/src/super-sql/sql/values.md b/book/src/super-sql/sql/values.md index ce9fac080..5a5de1411 100644 --- a/book/src/super-sql/sql/values.md +++ b/book/src/super-sql/sql/values.md @@ -59,15 +59,15 @@ FROM (VALUES ('hello, world'),('to be or not to be')) T(message) --- -_Column variation filled in with missing values_ -```mdtest-spq-skip +_Column variation filled in with null values_ +```mdtest-spq # spq SELECT * FROM (VALUES (1,2),(3)) T(x,y) # input # expected output {x:1,y:2} -{x:3,y:error("missing")} +{x:3,y:null} ``` ---- \ No newline at end of file +--- diff --git a/compiler/dag/expr.go b/compiler/dag/expr.go index 1195d87b8..4b86463d7 100644 --- a/compiler/dag/expr.go +++ b/compiler/dag/expr.go @@ -66,6 +66,7 @@ type ( LHS Expr `json:"lhs"` RHS string `json:"rhs"` Noneish bool `json:"noneish"` + Nullish bool `json:"nullish"` } IndexExpr struct { Kind string `json:"kind" unpack:""` diff --git a/compiler/optimizer/optimizer.go b/compiler/optimizer/optimizer.go index 645a23eec..60c7e2982 100644 --- a/compiler/optimizer/optimizer.go +++ b/compiler/optimizer/optimizer.go @@ -599,11 +599,11 @@ func liftFilterOps(seq dag.Seq) dag.Seq { return newErrorMissing() } // Copy spread so f and y don't share dag.Exprs. - e, liftOK = addPathToExpr(dag.CopyExpr(spread), this.Chain.Path()) + e, liftOK = addPathToExpr(dag.CopyExpr(spread), this.Chain) return e } // Copy e1 so f and y don't share dag.Exprs. - e, liftOK = addPathToExpr(dag.CopyExpr(e1), this.Chain.Path()[1:]) + e, liftOK = addPathToExpr(dag.CopyExpr(e1), this.Chain[1:]) return e }) if liftOK { @@ -644,10 +644,10 @@ func mergeValuesOps(seq dag.Seq) dag.Seq { if v1TopLevelSpread == nil { return newErrorMissing() } - e, mergeOK = addPathToExpr(v1TopLevelSpread, this.Chain.Path()) + e, mergeOK = addPathToExpr(v1TopLevelSpread, this.Chain) return e } - e, mergeOK = addPathToExpr(v1Expr, this.Chain.Path()[1:]) + e, mergeOK = addPathToExpr(v1Expr, this.Chain[1:]) return e } var mergedOp dag.Op @@ -703,8 +703,8 @@ func hasThisWithEmptyPath(v any) bool { // - It returns a dag.This when possible. // - It descends to a dag.RecordExpr.Elem when possible. // - It returns false when it cannot descend to a dag.RecordExpr.Elem. -func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) { - if len(path) == 0 { +func addPathToExpr(e dag.Expr, chain field.Chain) (dag.Expr, bool) { + if len(chain) == 0 { return e, true } switch e := e.(type) { @@ -713,14 +713,14 @@ func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) { for _, elem := range slices.Backward(e.Elems) { switch elem := elem.(type) { case *dag.Field: - if elem.Name != path[0] { + if elem.Name != chain[0].ID { continue } if spread != nil { // Don't know which will win. return e, false } - return addPathToExpr(elem.Value, path[1:]) + return addPathToExpr(elem.Value, chain[1:]) case *dag.Spread: if spread != nil { // Don't know which will win. @@ -732,12 +732,12 @@ func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) { if spread == nil { return e, false } - return addPathToExpr(spread.Expr, path) + return addPathToExpr(spread.Expr, chain) case *dag.ThisExpr: - return dag.NewThis(slices.Concat(e.Chain, field.NewChain(path...))), true + return dag.NewThis(slices.Concat(e.Chain, chain)), true } - for _, elem := range path { - e = &dag.DotExpr{Kind: "DotExpr", LHS: e, RHS: elem} + for _, elem := range chain { + e = &dag.DotExpr{Kind: "DotExpr", LHS: e, RHS: elem.ID, Noneish: elem.Noneish, Nullish: elem.Nullish} } return e, true } diff --git a/compiler/rungen/vexpr.go b/compiler/rungen/vexpr.go index 3bd50fa07..eeb4dc6b0 100644 --- a/compiler/rungen/vexpr.go +++ b/compiler/rungen/vexpr.go @@ -155,7 +155,7 @@ func (b *Builder) compileVamDotExpr(dot *dag.DotExpr) (vamexpr.Evaluator, error) if err != nil { return nil, err } - return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish), nil + return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish, dot.Nullish), nil } func (b *Builder) compileVamIndexExpr(idx *dag.IndexExpr) (vamexpr.Evaluator, error) { diff --git a/compiler/semantic/dagen.go b/compiler/semantic/dagen.go index 69fdc72bd..fbf8eff31 100644 --- a/compiler/semantic/dagen.go +++ b/compiler/semantic/dagen.go @@ -402,6 +402,7 @@ func (d *dagen) expr(e sem.Expr) dag.Expr { LHS: d.expr(e.LHS), RHS: e.RHS, Noneish: e.Noneish, + Nullish: e.Nullish, } case *sem.IndexExpr: return &dag.IndexExpr{ diff --git a/compiler/semantic/schema.go b/compiler/semantic/schema.go index ce3a65be3..4ff39a5e5 100644 --- a/compiler/semantic/schema.go +++ b/compiler/semantic/schema.go @@ -395,7 +395,7 @@ func (j *joinUsingScope) star(n ast.Node, table string, path field.Path) ([]*sem return nil, err } p := append(append(path, "left"), left...) - this := sem.NewThis(n, field.NewChain(p...)) + this := sem.NewThis(n, field.NewChainNullish(p...)) out = append(out, this) } var err error @@ -430,7 +430,7 @@ func (s *staticTable) star(n ast.Node, table string, path field.Path) ([]*sem.Th var out []*sem.ThisExpr if table == "" || s.table == table { for _, col := range s.typ.Fields { - path := field.NewChain(path...).Append(col.Name, false) + path := field.NewChainNullish(path...).AppendNullish(col.Name) out = append(out, sem.NewThis(n, path)) } } diff --git a/compiler/semantic/scope.go b/compiler/semantic/scope.go index 5d1bfe8de..c8f99a715 100644 --- a/compiler/semantic/scope.go +++ b/compiler/semantic/scope.go @@ -198,7 +198,7 @@ func (s *Scope) resolve(t *translator, n ast.Node, path field.Path, inType super return badExpr, t.checker.unknown } } - this := sem.NewThis(n, field.NewChain(append(out, path[1:]...)...)) + this := sem.NewThis(n, field.NewChainNullish(append(out, path[1:]...)...)) return this, t.checker.this(n, this, inType) } if scope, ok := scope.(*selectScope); ok && scope.lateral && inputFirst { @@ -227,7 +227,7 @@ func (s *Scope) resolve(t *translator, n ast.Node, path field.Path, inType super return badExpr, t.checker.unknown } if out != nil { - this := sem.NewThis(n, field.NewChain(append(out, path[2:]...)...)) + this := sem.NewThis(n, field.NewChainNullish(append(out, path[2:]...)...)) return this, t.checker.this(n, this, inType) } if p, _, _ := scope.resolveTable(n, path[0], nil); p != nil { @@ -244,7 +244,7 @@ func extend(n ast.Node, e sem.Expr, rest []string) sem.Expr { return e } if this, ok := e.(*sem.ThisExpr); ok { - return sem.NewThis(n, append(this.Chain, field.NewChain(rest...)...)) + return sem.NewThis(n, append(this.Chain, field.NewChainNullish(rest...)...)) } out := &sem.DotExpr{ Node: n, @@ -253,9 +253,10 @@ func extend(n ast.Node, e sem.Expr, rest []string) sem.Expr { } for _, f := range rest[1:] { out = &sem.DotExpr{ - Node: n, - LHS: out, - RHS: f, + Node: n, + LHS: out, + RHS: f, + Nullish: true, } } return out diff --git a/compiler/semantic/sem/expr.go b/compiler/semantic/sem/expr.go index c3c950019..66ed27e1e 100644 --- a/compiler/semantic/sem/expr.go +++ b/compiler/semantic/sem/expr.go @@ -58,6 +58,7 @@ type ( LHS Expr RHS string Noneish bool + Nullish bool } IndexExpr struct { ast.Node @@ -336,6 +337,7 @@ func CopyExpr(e Expr) Expr { LHS: CopyExpr(e.LHS), RHS: e.RHS, Noneish: e.Noneish, + Nullish: e.Nullish, } case *IndexExpr: return &IndexExpr{ diff --git a/compiler/semantic/sql.go b/compiler/semantic/sql.go index 0cfa9ba04..88bac8067 100644 --- a/compiler/semantic/sql.go +++ b/compiler/semantic/sql.go @@ -413,7 +413,7 @@ func mapColumns(sctx *super.Context, in *super.TypeRecord, alias *ast.TableAlias elems = append(elems, &sem.FieldElem{ Node: alias.Columns[k], Name: out[k], - Value: sem.NewThis(alias.Columns[k], field.NewChain(in.Fields[k].Name)), + Value: sem.NewThis(alias.Columns[k], field.NewChainNullish(in.Fields[k].Name)), }) fields = append(fields, super.NewField(out[k], in.Fields[k].Type)) } @@ -634,8 +634,8 @@ func (t *translator) sqlJoinCond(cond ast.JoinCond, typ super.Type) sem.Expr { t.error(id, fmt.Errorf("column %q in USING clause does not exist in right table", id.Name)) continue } - lhs := sem.NewThis(id, field.NewChain(append([]string{"left"}, left...)...)) - rhs := sem.NewThis(id, field.NewChain(append([]string{"right"}, right...)...)) + lhs := sem.NewThis(id, field.NewChainNullish(append([]string{"left"}, left...)...)) + rhs := sem.NewThis(id, field.NewChainNullish(append([]string{"right"}, right...)...)) exprs = append(exprs, sem.NewBinaryExpr(id, "==", lhs, rhs)) } if len(exprs) == 0 { @@ -711,7 +711,7 @@ func (t *translator) resolveOrdinalOuter(ts tableScope, n ast.Node, prefix strin } else { path = []string{ts.typ.Fields[col-1].Name} } - return sem.NewThis(n, field.NewChain(path...)) + return sem.NewThis(n, field.NewChainNullish(path...)) default: panic(ts) } diff --git a/compiler/sfmt/shared.go b/compiler/sfmt/shared.go index bb98d6da2..67e99706c 100644 --- a/compiler/sfmt/shared.go +++ b/compiler/sfmt/shared.go @@ -34,6 +34,9 @@ func (s *shared) fieldchain(chain field.Chain) { if elem.Noneish { s.write("?") } + if elem.Nullish { + s.write("??") + } if sup.IsIdentifier(elem.ID) { if k != 0 { s.write(".") diff --git a/compiler/ztests/merge-values.yaml b/compiler/ztests/merge-values.yaml index ba7715fa1..2022d4205 100644 --- a/compiler/ztests/merge-values.yaml +++ b/compiler/ztests/merge-values.yaml @@ -49,7 +49,7 @@ outputs: === file f unordered fields a,b | aggregate - t0:=max(b) by k0:=a + t0:=max(??b) by k0:=??a | aggregate min:=min(t0) by a:=k0 | output main diff --git a/compiler/ztests/sql/agg-dups.yaml b/compiler/ztests/sql/agg-dups.yaml index 58d0c7bd5..adcf148ab 100644 --- a/compiler/ztests/sql/agg-dups.yaml +++ b/compiler/ztests/sql/agg-dups.yaml @@ -6,9 +6,9 @@ outputs: data: | null | values {c0:1}, {c0:2} - | values {a:c0} + | values {a:??c0} | aggregate - t0:=max(a) + t0:=max(??a) | values {g:this,out:{max:t0,max_1:t0}} | sort g.t0 asc nulls last | values out diff --git a/compiler/ztests/sql/as-implied.yaml b/compiler/ztests/sql/as-implied.yaml index f17b53af1..88c1ec552 100644 --- a/compiler/ztests/sql/as-implied.yaml +++ b/compiler/ztests/sql/as-implied.yaml @@ -31,9 +31,9 @@ outputs: {x:4,y:4,z:2} === file t.json format json fields a - | values {"a+1":a+1,"a+2":a+2,"a+1_1":a+1,"a+3":a+3,"a+1_2":a+1} + | values {"a+1":??a+1,"a+2":??a+2,"a+1_1":??a+1,"a+3":??a+3,"a+1_2":??a+1} | output main === file t.json format json fields a - | values {"a+1":a+1,"a+1_1":a+1} + | values {"a+1":??a+1,"a+1_1":??a+1} | output main diff --git a/compiler/ztests/sql/groupby.yaml b/compiler/ztests/sql/groupby.yaml index 540f08bc3..789fa7c52 100644 --- a/compiler/ztests/sql/groupby.yaml +++ b/compiler/ztests/sql/groupby.yaml @@ -1,7 +1,7 @@ script: | - super -s -c "select val.radius ?? error('missing') as radius,count() from shapes.json group by val.radius ?? error('missing') | sort this" + super -s -c "select val.radius.ok() ?? error('missing') as radius,count() from shapes.json group by val.radius.ok() ?? error('missing') | sort this" echo === - super -s -c 'select type,sum(val.radius) from shapes.json group by type | sort this' + super -s -c 'select type,sum(val.radius.ok()) from shapes.json group by type | sort this' inputs: - name: shapes.json diff --git a/compiler/ztests/sql/join-filter-pullup.yaml b/compiler/ztests/sql/join-filter-pullup.yaml index 25e36f573..e7888cdc3 100644 --- a/compiler/ztests/sql/join-filter-pullup.yaml +++ b/compiler/ztests/sql/join-filter-pullup.yaml @@ -23,14 +23,14 @@ outputs: | fork ( values {id:1,team:"badgers"} - | where team=="badgers" + | where ??team=="badgers" ) ( values {id:1,team_id:1,player:"blake",position:"P"} - | where position=="P" + | where ??position=="P" ) - | inner hashjoin as {left,right} on id==team_id - | values {id:left.id,team:left.team,id_1:right.id,team_id:right.team_id,player:right.player,position:right.position} + | inner hashjoin as {left,right} on ??id==??team_id + | values {id:??left??.id,team:??left??.team,id_1:??right??.id,team_id:??right??.team_id,player:??right??.player,position:??right??.position} | output main // === null @@ -39,19 +39,19 @@ outputs: fork ( values {a1:1} - | where a1 in [1,5,9] and a1 in set[1,5,9] + | where ??a1 in [1,5,9] and ??a1 in set[1,5,9] ) ( values {a2:1} - | where a2==1 or a2==2 + | where ??a2==1 or ??a2==2 ) | cross join as {left,right} ) ( values {a3:1} - | where a3 in {c0:1,c1:5,c2:[9,11]} + | where ??a3 in {c0:1,c1:5,c2:[9,11]} ) | cross join as {left,right} | where 1==1 - | values {a1:left.left.a1,a2:left.right.a2,a3:right.a3} + | values {a1:??left??.left??.a1,a2:??left??.right??.a2,a3:??right??.a3} | output main diff --git a/compiler/ztests/sql/join-nulls.yaml b/compiler/ztests/sql/join-nulls.yaml new file mode 100644 index 000000000..e82cc5792 --- /dev/null +++ b/compiler/ztests/sql/join-nulls.yaml @@ -0,0 +1,24 @@ +# from issue 5984 + +script: | + super -f parquet -o integers.parquet integers.sup + super -f parquet -o integers2.parquet integers2.sup + super -s -c "SELECT * FROM integers.parquet LEFT OUTER JOIN integers2.parquet ON integers.i=integers2.k ORDER BY i;" + +inputs: + - name: integers.sup + data: | + {i:1::int32,j:2::int32} + {i:2::int32,j:3::int32} + {i:3::int32,j:4::int32} + - name: integers2.sup + data: | + {k:1::int32,l:10::int32} + {k:2::int32,l:20::int32} + +outputs: + - name: stdout + data: | + {i:1::int32,j:2::int32,k:1::int32,l:10::int32} + {i:2::int32,j:3::int32,k:2::int32,l:20::int32} + {i:3::int32,j:4::int32,k:null,l:null} diff --git a/compiler/ztests/sql/select-missing.yaml b/compiler/ztests/sql/select-missing.yaml new file mode 100644 index 000000000..a1d2b4489 --- /dev/null +++ b/compiler/ztests/sql/select-missing.yaml @@ -0,0 +1,20 @@ +spq: SELECT T.y FROM (VALUES (1,2),(3)) T(x,y) + +output: | + {y:2} + {y:null} + +--- + +spq: SELECT y FROM (VALUES (1,2),(3)) T(x,y) + +output: | + {y:2} + {y:null} + +--- + +spq: SELECT * FROM (VALUES (1,2)) AS T(x,y) GROUP BY T.x, T.y + +output: | + {x:1,y:2} diff --git a/compiler/ztests/sql/select-star-static.yaml b/compiler/ztests/sql/select-star-static.yaml index 06cf0aa83..f9372ecc2 100644 --- a/compiler/ztests/sql/select-star-static.yaml +++ b/compiler/ztests/sql/select-star-static.yaml @@ -27,9 +27,9 @@ outputs: data: | null | values {c0:1,c1:2,c2:3} - | values {a:c0,b:c1,c:c2} + | values {a:??c0,b:??c1,c:??c2} | values {in:this} - | values {in:in,out:{a:in.a,a_1:in.a,b:in.b,c:in.c}} + | values {in:in,out:{a:??in??.a,a_1:??in??.a,b:??in??.b,c:??in??.c}} | values out | output main // === @@ -37,15 +37,15 @@ outputs: | fork ( values {c0:1,c1:"dodgers"} - | values {id:c0,team:c1} + | values {id:??c0,team:??c1} ) ( values {c0:1,c1:1,c2:"ted"} - | values {id:c0,team_id:c1,player:c2} + | values {id:??c0,team_id:??c1,player:??c2} ) - | inner join as {left,right} on left.id==right.team_id + | inner join as {left,right} on ??left??.id==??right??.team_id | values {in:this} - | values {in:in,out:{id:in.left.id,team:in.left.team,id_1:in.right.id,team_id:in.right.team_id,player:in.right.player}} + | values {in:in,out:{id:??in??.left??.id,team:??in??.left??.team,id_1:??in??.right??.id,team_id:??in??.right??.team_id,player:??in??.right??.player}} | values out | output main // === @@ -53,17 +53,17 @@ outputs: | fork ( values {c0:1,c1:2,c2:"a"} - | values {aid:c0,bid:c1,a:c2} + | values {aid:??c0,bid:??c1,a:??c2} ) ( values {c0:1,c1:2,c2:"b"} - | values {aid:c0,bid:c1,b:c2} + | values {aid:??c0,bid:??c1,b:??c2} ) - | inner join as {left,right} on left.aid==right.aid and left.bid==right.bid + | inner join as {left,right} on ??left??.aid==??right??.aid and ??left??.bid==??right??.bid | values {in:this} - | values {in:in,out:{aid:in.left.aid,bid:in.left.bid,a:in.left.a,b:in.right.b}} + | values {in:in,out:{aid:??in??.left??.aid,bid:??in??.left??.bid,a:??in??.left??.a,b:??in??.right??.b}} | values out | values {in:this} - | values {in:in,out:{aid:in.aid,bid:in.bid,a:in.a,b:in.b}} + | values {in:in,out:{aid:??in??.aid,bid:??in??.bid,a:??in??.a,b:??in??.b}} | values out | output main diff --git a/pkg/field/chain.go b/pkg/field/chain.go index c9189065d..4a71cad55 100644 --- a/pkg/field/chain.go +++ b/pkg/field/chain.go @@ -3,6 +3,7 @@ package field type ChainElem struct { ID string Noneish bool + Nullish bool } type Chain []ChainElem @@ -15,8 +16,20 @@ func NewChain(ids ...string) Chain { return chain } +func NewChainNullish(ids ...string) Chain { + chain := make([]ChainElem, 0, len(ids)) + for _, id := range ids { + chain = append(chain, ChainElem{ID: id, Nullish: true}) + } + return chain +} + func (c Chain) Append(id string, noneish bool) Chain { - return append(c, ChainElem{id, noneish}) + return append(c, ChainElem{id, noneish, false}) +} + +func (c Chain) AppendNullish(id string) Chain { + return append(c, ChainElem{id, false, true}) } func (c Chain) Path() Path { diff --git a/runtime/vam/expr/dot.go b/runtime/vam/expr/dot.go index 0c53fdb8d..95edd9cc8 100644 --- a/runtime/vam/expr/dot.go +++ b/runtime/vam/expr/dot.go @@ -2,6 +2,7 @@ package expr import ( "fmt" + "slices" "github.com/brimdata/super" "github.com/brimdata/super/pkg/field" @@ -17,72 +18,120 @@ func (*This) Eval(val vector.Any) vector.Any { type DotExpr struct { sctx *super.Context - record Evaluator - field string + defuse *Defuse + entity Evaluator + key string noneish bool + nullish bool + okPush bool } -func NewDotExpr(sctx *super.Context, record Evaluator, field string, noneish bool) *DotExpr { +func NewDotExpr(sctx *super.Context, record Evaluator, field string, noneish, nullish bool) *DotExpr { return &DotExpr{ sctx: sctx, - record: record, - field: field, + defuse: NewDefuse(sctx), + entity: record, + key: field, noneish: noneish, + nullish: nullish, } } func NewDottedExpr(sctx *super.Context, f field.Chain) Evaluator { ret := Evaluator(&This{}) for _, elem := range f { - ret = NewDotExpr(sctx, ret, elem.ID, elem.Noneish) + ret = NewDotExpr(sctx, ret, elem.ID, elem.Noneish, elem.Nullish) } return ret } func (d *DotExpr) Eval(vec vector.Any) vector.Any { - return vector.Apply(vector.ApplyRipFusions|vector.ApplyRipUnions, d.eval, d.record.Eval(vec)) + return vector.Apply(vector.ApplyNone, d.eval, d.entity.Eval(vec)) } -func (d *DotExpr) eval(vecs ...vector.Any) vector.Any { - switch val := vector.Under(vector.Super(vecs[0])).(type) { - case *vector.None: - return val - case *vector.Record: - i, ok := val.Typ.IndexOfField(d.field) - if !ok { - if d.noneish { - return vector.NewNone(val.Len()) +func (d *DotExpr) eval(outerVecs ...vector.Any) vector.Any { + vec := outerVecs[0] + var missing bool + eval := func(innerVecs ...vector.Any) vector.Any { + switch val := vector.Under(innerVecs[0]).(type) { + case *vector.Null: + if d.nullish { + return val } - return vector.NewWrappedError(d.sctx, fmt.Sprintf("no such field %s", sup.QuotedName(d.field)), val) - } - return val.Fields[i] - case *vector.TypeValue: - var errs []uint32 - typvals := vector.NewTypeValueEmpty() - for i := range val.Len() { - typ := val.Value(i) - if typ, ok := super.TypeUnder(typ).(*super.TypeRecord); ok { - if typ, ok := typ.TypeOfField(d.field); ok { - typvals.Append(typ) - continue + case *vector.None: + return val + case *vector.Record: + i, ok := val.Typ.IndexOfField(d.key) + if !ok { + if d.nullish { + return vector.NewNull(val.Len()) } + missing = true + return vector.NewWrappedError(d.sctx, fmt.Sprintf("no such field %s", sup.QuotedName(d.key)), innerVecs[0]) } - errs = append(errs, i) - } - if len(errs) > 0 { - return vector.NewCombinedError(d.sctx, fmt.Sprintf("no such field %s", sup.QuotedName(d.field)), typvals, val, errs) + out := val.Fields[i] + if hasNone(out) { + missing = true + } + return out + case *vector.TypeValue: + var errs []uint32 + typvals := vector.NewTypeValueEmpty() + for i := range val.Len() { + typ := val.Value(i) + if typ, ok := super.TypeUnder(typ).(*super.TypeRecord); ok { + if typ, ok := typ.TypeOfField(d.key); ok { + typvals.Append(typ) + continue + } + } + errs = append(errs, i) + } + if len(errs) > 0 { + return vector.NewCombinedError(d.sctx, fmt.Sprintf("no such field %s", sup.QuotedName(d.key)), typvals, val, errs) + } + return typvals + case *vector.Map: + keyVec := vector.NewConstString(d.key, val.Len()) + return indexMap(d.sctx, val, keyVec) + case *vector.View: + return vector.Pick(d.eval(val.Any), val.Index) } - return typvals - case *vector.Map: - keyVec := vector.NewConstString(d.field, val.Len()) - return indexMap(d.sctx, val, keyVec) - case *vector.View: - return vector.Pick(d.eval(val.Any), val.Index) - default: dot := "." if d.noneish { dot = "?." + } else if d.nullish { + dot = "??." } - return vector.NewWrappedError(d.sctx, fmt.Sprintf("'%s': applied to non-record", dot), vecs[0]) + return vector.NewWrappedError(d.sctx, fmt.Sprintf("'%s': applied to non-record", dot), innerVecs[0]) + } + out := vector.Apply(vector.ApplyRipFusions|vector.ApplyRipUnions, eval, vec) + // If there were any structured errors or none values (e.g., because we hit a none + // inside a fusion and thus should be an error), then we take the slow path + // by defusing and starting over. One simple optimization we can do is okPush + // to indicate that this reference is wrapped in an ok() or is_ok(), in which case, + // the on field of the structured error will be discarded and thus does not need + // to be correct. There are a number of other ways to avoid this slow path but let's + // get it working first before we make it fast. + // XXX we need to wire up okPush + if !d.okPush && missing && vec.Kind() == vector.KindFusion { + return vector.Apply(vector.ApplyRipFusions|vector.ApplyRipUnions, d.eval, d.defuse.Eval(vec)) + } + return out +} + +func hasNone(vec vector.Any) bool { + if _, ok := vec.(*vector.None); ok { + return true + } + if vec, ok := vec.(*vector.Fusion); ok { + return hasNone(vec.Values) + } + if vec, ok := vec.(*vector.Dynamic); ok { + return slices.IndexFunc(vec.Values, hasNone) >= 0 + } + if super.IsOptionType(vec.Type()) { + return hasNone(vec.(*vector.Union).Dynamic()) } + return false } diff --git a/runtime/ztests/expr/dot-fusion-none.yaml b/runtime/ztests/expr/dot-fusion-none.yaml new file mode 100644 index 000000000..5beab28c5 --- /dev/null +++ b/runtime/ztests/expr/dot-fusion-none.yaml @@ -0,0 +1,13 @@ +spq: fuse | values x + +input: | + {x:"foo"} + {x:1} + {y:1} + {} + +output: | + "foo" + 1 + error({message:"no such field x",on:{y:1}}) + error({message:"no such field x",on:{}}) diff --git a/runtime/ztests/expr/dot.yaml b/runtime/ztests/expr/dot.yaml index 120b6f65a..c3e6440ac 100644 --- a/runtime/ztests/expr/dot.yaml +++ b/runtime/ztests/expr/dot.yaml @@ -19,7 +19,7 @@ output: | 1 1 error({message:"'.': applied to non-record",on:null}) - none + error({message:"no such field b",on:{}}) error({message:"'.': applied to non-record",on:1}) error({message:"'.': applied to non-record",on:error({message:"no such field a",on:{}})}) error({message:"'.': applied to non-record",on:error({message:"'.': applied to non-record",on:null})}) diff --git a/runtime/ztests/op/aggregate/count-star.yaml b/runtime/ztests/op/aggregate/count-star.yaml index a98e496d6..d041ea308 100644 --- a/runtime/ztests/op/aggregate/count-star.yaml +++ b/runtime/ztests/op/aggregate/count-star.yaml @@ -6,4 +6,4 @@ input: | {a:null} output: | - {c1:3,c2:3,c3:2} + {c1:3,c2:3,c3:1} diff --git a/runtime/ztests/op/distinct.yaml b/runtime/ztests/op/distinct.yaml index 65bef8d38..fcf73c8da 100644 --- a/runtime/ztests/op/distinct.yaml +++ b/runtime/ztests/op/distinct.yaml @@ -1,5 +1,5 @@ script: | - super -s -c 'from "1.sup" | distinct x' + super -s -c 'from "1.sup" | distinct x.ok()' echo === super -s -c 'from "2.sup" | distinct abs(this)' echo === diff --git a/runtime/ztests/op/join-empty-inner.yaml b/runtime/ztests/op/join-empty-inner.yaml index bc0e94720..d2471249f 100644 --- a/runtime/ztests/op/join-empty-inner.yaml +++ b/runtime/ztests/op/join-empty-inner.yaml @@ -1,8 +1,8 @@ script: | echo === hash join - super -dynamic -s -c 'left join (from C.sup) on left.a=right.a | values {...left,hit:this?.right?.sc ?? error("missing")} | sort' A.sup + super -dynamic -s -c 'left join (from C.sup) on left.a=right.a | values {...left,hit:right.sc.ok() ?? error("missing")} | sort' A.sup echo === nested loop join - super -dynamic -s -c 'left join (from C.sup) on left.a