change pipe-level none handling, empty subquery, and add has()

book/src/SUMMARY.md

@@ -137,6 +137,7 @@
             - [regexp_replace](super-sql/functions/parsing/regexp_replace.md)
         - [Records](super-sql/functions/records/intro.md)
             - [flatten](super-sql/functions/records/flatten.md)
+            - [has](super-sql/functions/records/has.md)
             - [keys](super-sql/functions/records/keys.md)
             - [nest_dotted](super-sql/functions/records/nest_dotted.md)
             - [paths](super-sql/functions/records/paths.md)

book/src/super-sql/expressions/match.md

@@ -19,7 +19,8 @@ match expression.  A value matches a type by the same rules as a
 its underlying value and [named types](../types/named.md) are distinct.
 
 If no case matches, the result is `<else-result>` or, without an
-`else` clause, `none`, which disappears from a record, array, or map
+`else` clause, `none`, which disappears from a record, array, or map,
+is dropped from the output of the [values](../operators/values.md) operator,
 and is typically handled with the `??` operator.
 An [error](../types/error.md) or `missing` value is passed through
 unless a case has its type.

book/src/super-sql/expressions/subqueries.md

@@ -19,6 +19,9 @@ results in the value `{s:"HELLO, WORLD"}`.
 Except for subqueries appearing as the right-hand side of
 an [in](containment.md) operator, the result of a subquery must be a single value.
 When multiple values are generated, an [error](../types/error.md) is produced.
+When no values are generated, the result is `none`, which disappears from a
+record, array, or map and is dropped from the output of the
+[values](../operators/values.md) operator, or, in a SQL expression, `null`.
 
 For the [in](containment.md) operator, any subquery on the right-hand side is
 always treated as an [array subquery](#array-subqueries), thus
@@ -278,3 +281,20 @@ values walk(this, &addOne)
 [2,3,4]
 [{x:[2,"foo"]},{y:3}]
 ```
+
+---
+
+_A subquery that produces no values is none_
+
+```mdtest-spq
+# spq
+values {s:(unnest a | where this > 1)}, (unnest a | where this > 1) ?? 0
+# input
+{a:[1,2]}
+{a:[1]}
+# expected output
+{s:2}
+2
+{}
+0
+```

book/src/super-sql/expressions/type-assertions.md

@@ -7,7 +7,8 @@ A type assertion has the form
 where `<expr>` is any [expression](intro.md) and `<type>` is any
 [type](../types/intro.md).  Its value is the value of `<expr>` when that
 value has type `<type>` and is otherwise `none`,
-which disappears from a record, array, or map and is typically handled with
+which disappears from a record, array, or map, is dropped from the output of the
+[values](../operators/values.md) operator, and is typically handled with
 the `??` operator, e.g.,
 ```
 v.(string) ?? "default"

book/src/super-sql/functions/generics/ok.md

@@ -16,8 +16,9 @@ unmodified.
 
 This function is useful when errors are expected and are to be explicitly managed
 with `none` operators.  The `none` disappears from a record expression, an
-array, or a map and otherwise must be handled with `??` since a query may not
-output a `none`.
+array, or a map and is dropped from the output of the
+[values](../../operators/values.md) operator, so `??` is needed only to replace
+it with another value.
 
 ## Examples
 

book/src/super-sql/functions/records/has.md

@@ -0,0 +1,112 @@
+# has
+
+test whether a record or record type has a field or a map has a key
+
+## Synopsis
+
+```
+has(val: any, key: any) -> bool
+```
+
+## Description
+
+The `has` function returns true if
+* `val` is a record whose type has a field named `key`,
+* `val` is a type value of a record type with a field named `key`, or
+* `val` is a map with the key `key`
+
+and false otherwise, e.g., for a value that isn't a record, type value, or
+map.
+
+For a record, the result depends only on its type, not on the value of the
+field, so `has` is true for a field whose value is null, an error, or an
+option's none.  A value of a named type or a union type is tested by its
+underlying type.
+
+For a map, `key` may be of any type and is compared with the map's keys after
+removing named and union types, so a key of a different type (e.g., an
+`int32` for a map with `int64` keys) is never found.
+
+An error is returned if `val` is a record and `key` isn't a string.
+
+## Examples
+
+---
+
+_Test records for fields_
+
+```mdtest-spq
+# spq
+values {a:has(this,"a"),b:has(this,"b")}
+# input
+{a:1}
+{b:null}
+{a:error("x"),b:2}
+1
+# expected output
+{a:true,b:false}
+{a:false,b:true}
+{a:true,b:true}
+{a:false,b:false}
+```
+
+---
+
+_Filter on the presence of a field_
+
+```mdtest-spq
+# spq
+where has(this,"b")
+# input
+{a:1}
+{b:2}
+# expected output
+{b:2}
+```
+
+---
+
+_Test a record type_
+
+```mdtest-spq
+# spq
+values has(this,"a")
+# input
+<{a:int64}>
+<{b:int64}>
+# expected output
+true
+false
+```
+
+---
+
+_Test a map for a key_
+
+```mdtest-spq
+# spq
+values has(this,"a"), has(this,1)
+# input
+map{"a":1}
+map{1:"x"}
+# expected output
+true
+false
+false
+true
+```
+
+---
+
+_The field name may be computed_
+
+```mdtest-spq
+# spq
+values has(this, f)
+# input
+{a:1,f:"a"}
+{a:1,f:"b"}
+# expected output
+true
+false
+```

book/src/super-sql/operators/aggregate.md

@@ -55,6 +55,10 @@ The output values are records formed from the
 [field assignments](intro.md#field-assignment)
 first from the grouping expressions then from the aggregate functions
 in left-to-right order.
+As in a record expression, a grouping key whose value
+is a plain `none` (e.g., `ok(x)` when `x` is missing) disappears from its
+output record, so the values whose key is `none` form a group whose record
+has no field for that key.
 
 When the result of `aggregate` is a single value (e.g., a single aggregate
 function without grouping expressions or a single grouping expression without aggregates)

book/src/super-sql/operators/debug.md

@@ -16,6 +16,8 @@ each value in the input, evaluates the optional
 and transmits each result to a _debugging output_.
 If `<expr>` is omitted, then each input value is passed unmodified to the
 debugging output.
+As for the [values](values.md) operator, a result that is a plain `none`
+(e.g., from [ok](../functions/generics/ok.md) or `?.`) is dropped.
 
 An optional filter may be applied to the values sent to the debugging output,
 which is specified with Boolean-valued expression `<pred>`.

book/src/super-sql/operators/unnest.md

@@ -41,7 +41,8 @@ produces
 ```
 
 The value of `<expr>` must be an array (or a set), whose elements form the
-derived sequence.
+derived sequence.  A `null` or a plain `none` value (e.g., from
+[ok](../functions/generics/ok.md) or `?.`) produces no values.
 
 To carry other values along with each unnested element, give the unnested
 value a name with `:=` or list the values in a `with` clause.  Then each

book/src/super-sql/operators/values.md

@@ -19,6 +19,13 @@ in left-to-right order.  Each `<expr>` may be any valid
 The input order convolved with left-to-right evaluation order is preserved
 at the output.
 
+A result that is a plain `none`, e.g., from [ok](../functions/generics/ok.md),
+`?.`, a [type assertion](../expressions/type-assertions.md) that fails, or a
+[match](../expressions/match.md) with no matching case and no `else`,
+is dropped, so a `none` is never passed to a downstream operator.
+To output another value in its place, use the `??` operator.
+An option's `none` (e.g., `none::int64`) is a value and is output as is.
+
 The `values` operator name is optional since it can be used as a
 [shortcut](intro.md#shortcuts).  When used as a shortcut, only one expression
 may be present.
@@ -127,3 +134,18 @@ values [a,b],[b,a] | collect(this)
 # expected output
 [[1,2],[2,1]]
 ```
+
+---
+
+_A plain none is dropped unless it's replaced with ??_
+```mdtest-spq
+# spq
+values ok(a), ok(a) ?? 0
+# input
+{a:1}
+{b:2}
+# expected output
+1
+1
+0
+```

compiler/dag/expr.go

@@ -146,7 +146,11 @@ type (
 	SubqueryExpr struct {
 		Kind       string `json:"kind" unpack:""`
 		Correlated bool   `json:"correlated"`
-		Body       Seq    `json:"body"`
+		// SQL is true for a subquery in a SQL expression, whose value is
+		// null when the subquery produces no values.  Otherwise, the value
+		// is none.
+		SQL  bool `json:"sql,omitempty"`
+		Body Seq  `json:"body"`
 	}
 	ThisExpr struct {
 		Kind  string      `json:"kind" unpack:""`

compiler/dag/op.go

@@ -373,7 +373,7 @@ func NewFilterOp(e Expr) *FilterOp {
 }
 
 func NewValuesOp(exprs ...Expr) *ValuesOp {
-	return &ValuesOp{"ValuesOp", exprs}
+	return &ValuesOp{Kind: "ValuesOp", Exprs: exprs}
 }
 
 func CopySeq(seq Seq) Seq {

compiler/rungen/vexpr.go

@@ -313,11 +313,11 @@ func (b *Builder) compileVamSubquery(query *dag.SubqueryExpr) (expr.Evaluator, e
 			return nil, err
 		}
 		body := b.combineVam(exits)
-		return op.NewCachedSubquery(b.sctx(), body), nil
+		return op.NewCachedSubquery(b.sctx(), body, query.SQL), nil
 	}
 	var create func() *op.Subquery
 	create = func() *op.Subquery {
-		subquery := op.NewSubquery(b.rctx.Context, b.sctx(), func() *op.Subquery {
+		subquery := op.NewSubquery(b.rctx.Context, b.sctx(), query.SQL, func() *op.Subquery {
 			b.mu.Lock()
 			defer b.mu.Unlock()
 			return create()

compiler/rungen/vop.go

@@ -317,7 +317,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vio.Puller) (vio.Puller, error
 		return vio.NewPuller(vector.NewNull(1)), nil
 	case *dag.OutputOp:
 		if parent != nil {
-			parent = op.NewOutput(b.sctx(), parent)
+			parent = op.NewOutput(parent)
 		}
 		b.channels[o.Name] = append(b.channels[o.Name], parent)
 		return parent, nil

compiler/semantic/analyzer.go

@@ -79,9 +79,6 @@ type translator struct {
 	sctx     *super.Context
 	types    *sup.Analyzer
 	defs     *super.Context
-	// subqueries is the depth of expression subqueries being translated.
-	// Their results may be none (see checker.output).
-	subqueries int
 }
 
 func newTranslator(ctx context.Context, r reporter, env *exec.Environment) *translator {