handle missing fields in SQL as null

book/src/super-sql/sql/join.md

@@ -99,7 +99,7 @@ JOIN U ON x=z
 ---
 
 _Left outer join_
-```mdtest-spq-skip
+```mdtest-spq
 # spq
 WITH T(x,y) AS (
     VALUES (1,2), (3,4), (5,6)
@@ -114,15 +114,15 @@ ORDER BY x
 # input
 
 # expected output
-{x:1,y:2,z:error("missing")}
+{x:1,y:2,z:null}
 {x:3,y:4,z:3}
-{x:5,y:6,z:error("missing")}
+{x:5,y:6,z:null}
 ```
 
 ---
 
 _Right outer join_
-```mdtest-spq-skip
+```mdtest-spq
 # spq
 WITH T(x,y) AS (
     VALUES (1,2), (3,4), (5,6)
@@ -138,7 +138,7 @@ ORDER BY x
 
 # expected output
 {x:3,y:4,z:3}
-{x:error("missing"),y:error("missing"),z:2}
+{x:null,y:null,z:2}
 ```
 
 ---

book/src/super-sql/sql/values.md

@@ -59,15 +59,15 @@ FROM (VALUES ('hello, world'),('to be or not to be')) T(message)
 
 ---
 
-_Column variation filled in with missing values_
-```mdtest-spq-skip
+_Column variation filled in with null values_
+```mdtest-spq
 # spq
 SELECT * FROM (VALUES (1,2),(3)) T(x,y)
 # input
 
 # expected output
 {x:1,y:2}
-{x:3,y:error("missing")}
+{x:3,y:null}
 ```
 
----
+---

compiler/dag/expr.go

@@ -66,6 +66,7 @@ type (
 		LHS     Expr   `json:"lhs"`
 		RHS     string `json:"rhs"`
 		Noneish bool   `json:"noneish"`
+		Nullish bool   `json:"nullish"`
 	}
 	IndexExpr struct {
 		Kind  string `json:"kind" unpack:""`

compiler/optimizer/optimizer.go

@@ -599,11 +599,11 @@ func liftFilterOps(seq dag.Seq) dag.Seq {
 						return newErrorMissing()
 					}
 					// Copy spread so f and y don't share dag.Exprs.
-					e, liftOK = addPathToExpr(dag.CopyExpr(spread), this.Chain.Path())
+					e, liftOK = addPathToExpr(dag.CopyExpr(spread), this.Chain)
 					return e
 				}
 				// Copy e1 so f and y don't share dag.Exprs.
-				e, liftOK = addPathToExpr(dag.CopyExpr(e1), this.Chain.Path()[1:])
+				e, liftOK = addPathToExpr(dag.CopyExpr(e1), this.Chain[1:])
 				return e
 			})
 			if liftOK {
@@ -644,10 +644,10 @@ func mergeValuesOps(seq dag.Seq) dag.Seq {
 					if v1TopLevelSpread == nil {
 						return newErrorMissing()
 					}
-					e, mergeOK = addPathToExpr(v1TopLevelSpread, this.Chain.Path())
+					e, mergeOK = addPathToExpr(v1TopLevelSpread, this.Chain)
 					return e
 				}
-				e, mergeOK = addPathToExpr(v1Expr, this.Chain.Path()[1:])
+				e, mergeOK = addPathToExpr(v1Expr, this.Chain[1:])
 				return e
 			}
 			var mergedOp dag.Op
@@ -703,8 +703,8 @@ func hasThisWithEmptyPath(v any) bool {
 //   - It returns a dag.This when possible.
 //   - It descends to a dag.RecordExpr.Elem when possible.
 //   - It returns false when it cannot descend to a dag.RecordExpr.Elem.
-func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) {
-	if len(path) == 0 {
+func addPathToExpr(e dag.Expr, chain field.Chain) (dag.Expr, bool) {
+	if len(chain) == 0 {
 		return e, true
 	}
 	switch e := e.(type) {
@@ -713,14 +713,14 @@ func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) {
 		for _, elem := range slices.Backward(e.Elems) {
 			switch elem := elem.(type) {
 			case *dag.Field:
-				if elem.Name != path[0] {
+				if elem.Name != chain[0].ID {
 					continue
 				}
 				if spread != nil {
 					// Don't know which will win.
 					return e, false
 				}
-				return addPathToExpr(elem.Value, path[1:])
+				return addPathToExpr(elem.Value, chain[1:])
 			case *dag.Spread:
 				if spread != nil {
 					// Don't know which will win.
@@ -732,12 +732,12 @@ func addPathToExpr(e dag.Expr, path []string) (dag.Expr, bool) {
 		if spread == nil {
 			return e, false
 		}
-		return addPathToExpr(spread.Expr, path)
+		return addPathToExpr(spread.Expr, chain)
 	case *dag.ThisExpr:
-		return dag.NewThis(slices.Concat(e.Chain, field.NewChain(path...))), true
+		return dag.NewThis(slices.Concat(e.Chain, chain)), true
 	}
-	for _, elem := range path {
-		e = &dag.DotExpr{Kind: "DotExpr", LHS: e, RHS: elem}
+	for _, elem := range chain {
+		e = &dag.DotExpr{Kind: "DotExpr", LHS: e, RHS: elem.ID, Noneish: elem.Noneish, Nullish: elem.Nullish}
 	}
 	return e, true
 }

compiler/rungen/vexpr.go

@@ -155,7 +155,7 @@ func (b *Builder) compileVamDotExpr(dot *dag.DotExpr) (vamexpr.Evaluator, error)
 	if err != nil {
 		return nil, err
 	}
-	return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish), nil
+	return vamexpr.NewDotExpr(b.sctx(), record, dot.RHS, dot.Noneish, dot.Nullish), nil
 }
 
 func (b *Builder) compileVamIndexExpr(idx *dag.IndexExpr) (vamexpr.Evaluator, error) {

compiler/semantic/dagen.go

@@ -402,6 +402,7 @@ func (d *dagen) expr(e sem.Expr) dag.Expr {
 			LHS:     d.expr(e.LHS),
 			RHS:     e.RHS,
 			Noneish: e.Noneish,
+			Nullish: e.Nullish,
 		}
 	case *sem.IndexExpr:
 		return &dag.IndexExpr{

compiler/semantic/scope.go

@@ -198,7 +198,7 @@ func (s *Scope) resolve(t *translator, n ast.Node, path field.Path, inType super
 				return badExpr, t.checker.unknown
 			}
 		}
-		this := sem.NewThis(n, field.NewChain(append(out, path[1:]...)...))
+		this := sem.NewThis(n, field.NewChainNullish(append(out, path[1:]...)...))
 		return this, t.checker.this(n, this, inType)
 	}
 	if scope, ok := scope.(*selectScope); ok && scope.lateral && inputFirst {
@@ -227,7 +227,7 @@ func (s *Scope) resolve(t *translator, n ast.Node, path field.Path, inType super
 			return badExpr, t.checker.unknown
 		}
 		if out != nil {
-			this := sem.NewThis(n, field.NewChain(append(out, path[2:]...)...))
+			this := sem.NewThis(n, field.NewChainNullish(append(out, path[2:]...)...))
 			return this, t.checker.this(n, this, inType)
 		}
 		if p, _, _ := scope.resolveTable(n, path[0], nil); p != nil {
@@ -244,7 +244,7 @@ func extend(n ast.Node, e sem.Expr, rest []string) sem.Expr {
 		return e
 	}
 	if this, ok := e.(*sem.ThisExpr); ok {
-		return sem.NewThis(n, append(this.Chain, field.NewChain(rest...)...))
+		return sem.NewThis(n, append(this.Chain, field.NewChainNullish(rest...)...))
 	}
 	out := &sem.DotExpr{
 		Node: n,
@@ -253,9 +253,10 @@ func extend(n ast.Node, e sem.Expr, rest []string) sem.Expr {
 	}
 	for _, f := range rest[1:] {
 		out = &sem.DotExpr{
-			Node: n,
-			LHS:  out,
-			RHS:  f,
+			Node:    n,
+			LHS:     out,
+			RHS:     f,
+			Nullish: true,
 		}
 	}
 	return out

compiler/semantic/sem/expr.go

@@ -58,6 +58,7 @@ type (
 		LHS     Expr
 		RHS     string
 		Noneish bool
+		Nullish bool
 	}
 	IndexExpr struct {
 		ast.Node
@@ -336,6 +337,7 @@ func CopyExpr(e Expr) Expr {
 			LHS:     CopyExpr(e.LHS),
 			RHS:     e.RHS,
 			Noneish: e.Noneish,
+			Nullish: e.Nullish,
 		}
 	case *IndexExpr:
 		return &IndexExpr{

compiler/semantic/sql.go

@@ -177,7 +177,7 @@ func (t *translator) formProjection(scope *selectScope, in []ast.SQLAsExpr, inTy
 			}
 			for _, p := range paths {
 				typ := t.checker.this(star, p, inType)
-				out = append(out, column{name: dedup(scores, t.asName(as.Label, nil, p.Chain.Path())), semExpr: p, typ: typ, astExpr: as.Expr})
+				out = append(out, column{name: dedup(scores, t.asName(as.Label, nil, p.Chain.Path())), semExpr: t.nullcheck(star, p), typ: typ, astExpr: as.Expr})
 			}
 			continue
 		}
@@ -187,6 +187,11 @@ func (t *translator) formProjection(scope *selectScope, in []ast.SQLAsExpr, inTy
 	return out
 }
 
+func (t *translator) nullcheck(loc ast.Node, e sem.Expr) sem.Expr {
+	null := sem.NewLiteral(loc, super.NewValue(super.TypeNull, nil), t.defs)
+	return sem.NewBinaryExpr(loc, "??", sem.NewCall(loc, "ok", []sem.Expr{e}), null)
+}
+
 func (t *translator) asName(label *ast.ID, expr ast.Expr, path []string) string {
 	var name string
 	if label != nil {
@@ -413,7 +418,7 @@ func mapColumns(sctx *super.Context, in *super.TypeRecord, alias *ast.TableAlias
 			elems = append(elems, &sem.FieldElem{
 				Node:  alias.Columns[k],
 				Name:  out[k],
-				Value: sem.NewThis(alias.Columns[k], field.NewChain(in.Fields[k].Name)),
+				Value: sem.NewThis(alias.Columns[k], field.NewChainNullish(in.Fields[k].Name)),
 			})
 			fields = append(fields, super.NewField(out[k], in.Fields[k].Type))
 		}
@@ -634,8 +639,8 @@ func (t *translator) sqlJoinCond(cond ast.JoinCond, typ super.Type) sem.Expr {
 				t.error(id, fmt.Errorf("column %q in USING clause does not exist in right table", id.Name))
 				continue
 			}
-			lhs := sem.NewThis(id, field.NewChain(append([]string{"left"}, left...)...))
-			rhs := sem.NewThis(id, field.NewChain(append([]string{"right"}, right...)...))
+			lhs := sem.NewThis(id, field.NewChainNullish(append([]string{"left"}, left...)...))
+			rhs := sem.NewThis(id, field.NewChainNullish(append([]string{"right"}, right...)...))
 			exprs = append(exprs, sem.NewBinaryExpr(id, "==", lhs, rhs))
 		}
 		if len(exprs) == 0 {
@@ -711,7 +716,7 @@ func (t *translator) resolveOrdinalOuter(ts tableScope, n ast.Node, prefix strin
 		} else {
 			path = []string{ts.typ.Fields[col-1].Name}
 		}
-		return sem.NewThis(n, field.NewChain(path...))
+		return sem.NewThis(n, field.NewChainNullish(path...))
 	default:
 		panic(ts)
 	}

compiler/sfmt/shared.go

@@ -34,6 +34,9 @@ func (s *shared) fieldchain(chain field.Chain) {
 		if elem.Noneish {
 			s.write("?")
 		}
+		if elem.Nullish {
+			s.write("??")
+		}
 		if sup.IsIdentifier(elem.ID) {
 			if k != 0 {
 				s.write(".")

compiler/ztests/merge-values.yaml

@@ -49,7 +49,7 @@ outputs:
       ===
       file f unordered fields a,b
       | aggregate
-          t0:=max(b) by k0:=a
+          t0:=max(??b) by k0:=??a
       | aggregate
           min:=min(t0) by a:=k0
       | output main

compiler/ztests/sql/agg-dups.yaml

@@ -6,9 +6,9 @@ outputs:
     data: |
       null
       | values {c0:1}, {c0:2}
-      | values {a:c0}
+      | values {a:??c0}
       | aggregate
-          t0:=max(a)
+          t0:=max(??a)
       | values {g:this,out:{max:t0,max_1:t0}}
       | sort g.t0 asc nulls last
       | values out

compiler/ztests/sql/as-implied.yaml

@@ -31,9 +31,9 @@ outputs:
       {x:4,y:4,z:2}
       ===
       file t.json format json fields a
-      | values {"a+1":a+1,"a+2":a+2,"a+1_1":a+1,"a+3":a+3,"a+1_2":a+1}
+      | values {"a+1":??a+1,"a+2":??a+2,"a+1_1":??a+1,"a+3":??a+3,"a+1_2":??a+1}
       | output main
       ===
       file t.json format json fields a
-      | values {"a+1":a+1,"a+1_1":a+1}
+      | values {"a+1":??a+1,"a+1_1":??a+1}
       | output main

compiler/ztests/sql/join-filter-pullup.yaml

@@ -23,14 +23,14 @@ outputs:
       | fork
         (
           values {id:1,team:"badgers"}
-          | where team=="badgers"
+          | where ??team=="badgers"
         )
         (
           values {id:1,team_id:1,player:"blake",position:"P"}
-          | where position=="P"
+          | where ??position=="P"
         )
-      | inner hashjoin as {left,right} on id==team_id
-      | values {id:left.id,team:left.team,id_1:right.id,team_id:right.team_id,player:right.player,position:right.position}
+      | inner hashjoin as {left,right} on ??id==??team_id
+      | values {id:ok(left.id)??null,team:ok(left.team)??null,id_1:ok(right.id)??null,team_id:ok(right.team_id)??null,player:ok(right.player)??null,position:ok(right.position)??null}
       | output main
       // ===
       null
@@ -39,19 +39,19 @@ outputs:
           fork
             (
               values {a1:1}
-              | where a1 in [1,5,9] and a1 in set[1,5,9]
+              | where ??a1 in [1,5,9] and ??a1 in set[1,5,9]
             )
             (
               values {a2:1}
-              | where a2==1 or a2==2
+              | where ??a2==1 or ??a2==2
             )
           | cross join as {left,right}
         )
         (
           values {a3:1}
-          | where a3 in {c0:1,c1:5,c2:[9,11]}
+          | where ??a3 in {c0:1,c1:5,c2:[9,11]}
         )
       | cross join as {left,right}
       | where 1==1
-      | values {a1:left.left.a1,a2:left.right.a2,a3:right.a3}
+      | values {a1:ok(left.left.a1)??null,a2:ok(left.right.a2)??null,a3:ok(right.a3)??null}
       | output main

compiler/ztests/sql/join-nulls.yaml

@@ -0,0 +1,24 @@
+# from issue 5984
+
+script: |
+  super -f parquet -o integers.parquet integers.sup
+  super -f parquet -o integers2.parquet integers2.sup
+  super -s -c "SELECT * FROM integers.parquet LEFT OUTER JOIN integers2.parquet ON integers.i=integers2.k ORDER BY i;"
+
+inputs:
+  - name: integers.sup
+    data: |
+      {i:1::int32,j:2::int32}
+      {i:2::int32,j:3::int32}
+      {i:3::int32,j:4::int32}
+  - name: integers2.sup
+    data: |
+      {k:1::int32,l:10::int32}
+      {k:2::int32,l:20::int32}
+
+outputs:
+  - name: stdout
+    data: |
+      {i:1::int32,j:2::int32,k:1::int32,l:10::int32}
+      {i:2::int32,j:3::int32,k:2::int32,l:20::int32}
+      {i:3::int32,j:4::int32,k:null,l:null}

compiler/ztests/sql/select-missing.yaml

@@ -0,0 +1,13 @@
+spq: SELECT T.y FROM (VALUES (1,2),(3)) T(x,y)
+
+output: |
+  {y:2}
+  {y:null}
+
+---
+
+spq: SELECT y FROM (VALUES (1,2),(3)) T(x,y)
+
+output: |
+  {y:2}
+  {y:null}