Skip to content

Commit 776d225

Browse files
committed
Merge land/source-type-warning-5: warn when a source's column type can't be read, and the playground goes on
2 parents 18b937b + 93dc709 commit 776d225

6 files changed

Lines changed: 66 additions & 14 deletions

File tree

‎crates/sqlquack-semantics/src/types/_helpers/source_file.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
33
use serde::Deserialize;
44
use serde_json::{Map, Value, json};
5+
use sqlquack_core::provenance::models::ProvenanceLog;
6+
use sqlquack_snowflake_types::typing::main::cast_target::cast_target;
57

68
use crate::types::_helpers::yaml_value::yaml_value;
79
use crate::types::models::SourceTables;
@@ -75,19 +77,31 @@ fn folded_name(name: &str) -> String {
7577
}
7678
}
7779

78-
/// Why a table source is left out: it lists no columns, or a column has no text type.
80+
/// Why a table source is left out: it lists no columns, or a column has no type SQLQuack reads.
7981
fn left_out(source: &SourceEntry) -> Option<String> {
8082
if source.expression.is_some() {
8183
return None;
8284
}
8385
if source.columns.is_empty() {
8486
return Some("it lists no columns".to_owned());
8587
}
88+
let log = ProvenanceLog::default();
8689
source
8790
.columns
8891
.iter()
89-
.find(|column| !matches!(column.type_name, Some(Value::String(_))))
90-
.map(|column| format!("its column {} has no type", column.name))
92+
.find_map(|column| untyped(column, &log))
93+
}
94+
95+
/// Why `column` has no type SQLQuack reads: none is given, or the one given does not parse.
96+
fn untyped(column: &SourceColumn, log: &ProvenanceLog) -> Option<String> {
97+
let Some(Value::String(type_name)) = &column.type_name else {
98+
return Some(format!("its column {} has no type", column.name));
99+
};
100+
let error = cast_target(type_name, log).err()?;
101+
Some(format!(
102+
"its column {} has no type SQLQuack reads: {error}",
103+
column.name
104+
))
91105
}
92106

93107
/// A source as a table name and its columns; None for an inline query or an untyped column.

‎crates/sqlquack/tests/cli/test_project.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,22 @@ fn given_a_project_when_a_declaration_clashes_or_is_unconfirmed_then_the_error_s
331331
expected_exit: Some(1),
332332
expected_output: "sqlquack convert: warning: ./sources/raw.yml: the source raw__customers is left out, because its column email has no type",
333333
},
334+
ProjectTestCase {
335+
description: "a source with a column type SQLQuack cannot read is left out with a warning naming it",
336+
files: &[
337+
POLICY_FILE,
338+
ProjectFile {
339+
path: "sources/raw.yml",
340+
repository: &[],
341+
text: "sources:\n - name: raw__customers\n columns:\n - name: id\n type: INTEGER\n - name: place\n type: GEOGRAPHY\n",
342+
},
343+
],
344+
folder: "",
345+
flags: &[],
346+
sql: "SELECT id AS n FROM raw__customers",
347+
expected_exit: Some(1),
348+
expected_output: "sqlquack convert: warning: ./sources/raw.yml: the source raw__customers is left out, because its column place has no type SQLQuack reads: unknown Snowflake type `GEOGRAPHY`",
349+
},
334350
ProjectTestCase {
335351
description: "a Python function file not marked as SQLBuild marks one is an error",
336352
files: &[

‎website/public/playground/playground.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,14 @@ async function run() {
163163
for (const diagnostic of diagnostics ?? []) spans.push({ kind, span: diagnostic.span, text: rendered(diagnostic, chars) });
164164
};
165165

166+
// A source left out, as `sqlquack convert` warns about it; the other sources still declare.
167+
const warnings = (report.warnings ?? []).map((w) => terminal(`<span class="warn">warning:</span> ${escape(w)}`)).join('');
166168
const parse = report.parse;
167169
add(parse.diagnostics, 'error');
168170
setStage(
169171
'parse',
170172
parse.status,
171-
parse.diagnostics ? parse.diagnostics.map((d) => terminal(rendered(d, chars))).join('') : '',
173+
warnings + (parse.diagnostics ? parse.diagnostics.map((d) => terminal(rendered(d, chars))).join('') : ''),
172174
parse.statements ? `${parse.statements} statement${parse.statements === 1 ? '' : 's'}` : '',
173175
);
174176

‎website/src/content/docs/docs/project.mdx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ SQLBuild project's source files work unchanged.
4545

4646
A source with an `expression` is a query rather than a table, and is skipped.
4747

48-
A source with a column that has no `type` is left out whole, so a `SELECT *` never misses a
49-
column. SQLQuack warns about it:
48+
A source with a column that has no `type`, or a type SQLQuack can't read, is left out whole, so a
49+
`SELECT *` never misses a column. SQLQuack warns about it:
5050

5151
```console
5252
sqlquack convert: warning: ./sources/more.yml: the source customers is left out, because its column email has no type

‎website/tests/playground.test.mjs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,14 @@ test('an undeclared relation is refused by name, and the stub the page adds stay
188188
);
189189
// The stub's placeholder type is not a Snowflake type: no column is assumed, so it still refuses.
190190
const stub = `${PAGE_SOURCES} - name: t\n columns:\n - name: column_name\n type: SNOWFLAKE_TYPE\n`;
191-
assert.deepEqual(JSON.parse(analyze('SELECT * FROM t', stub)).translate.diagnostics.map((d) => d.code), ['SQ0004']);
191+
const stubbed = JSON.parse(analyze('SELECT * FROM t', stub));
192+
assert.deepEqual(stubbed.translate.diagnostics.map((d) => d.code), ['SQ0004']);
193+
// The page says why the stub is left out, as `sqlquack convert` warns, and analysis goes on.
194+
assert.deepEqual(stubbed.warnings, [
195+
'the source t is left out, because its column column_name has no type SQLQuack reads: unknown Snowflake type `SNOWFLAKE_TYPE`',
196+
]);
197+
// The other sources still declare beside the one left out.
198+
assert.equal(JSON.parse(analyze('SELECT * FROM orders', stub)).translate.status, 'pass');
192199
// Every table of the sample sources without semi-structured columns translates as declared.
193200
for (const relation of ['ORDERS', 'CUSTOMERS', 'PRODUCTS', 'ORDER_ITEMS']) {
194201
assert.equal(JSON.parse(analyze(`SELECT * FROM ${relation.toLowerCase()}`, PAGE_SOURCES)).translate.status, 'pass', relation);

‎website/wasm/src/lib.rs‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ pub fn fixture_tables(sources: &str) -> String {
122122
struct Catalog {
123123
schema: Schema,
124124
columns: BTreeMap<String, Vec<SchemaColumn>>,
125+
/// Why a source is left out, as `sqlquack convert` warns about it.
126+
warnings: Vec<String>,
125127
}
126128

127129
impl LineageSchema for Catalog {
@@ -130,16 +132,13 @@ impl LineageSchema for Catalog {
130132
}
131133
}
132134

133-
/// The source file `text`, read as `sqlquack convert` reads a project's `sources/*.yml`; a source
134-
/// left out for an untyped column is an error, so the playground says why.
135+
/// The source file `text`, read as `sqlquack convert` reads a project's `sources/*.yml`: a source
136+
/// left out for a column without a type SQLQuack reads is a warning, and the rest still declare.
135137
fn catalog(text: &str) -> Result<Catalog, String> {
136138
if text.trim().is_empty() {
137139
return Ok(Catalog::default());
138140
}
139141
let found = source_tables(text)?;
140-
if !found.warnings.is_empty() {
141-
return Err(found.warnings.join("; "));
142-
}
143142
let relations: serde_json::Map<String, Value> = found.tables.into_iter().collect();
144143
let schema =
145144
parse_schema(&json!({ "relations": relations }).to_string()).map_err(
@@ -163,7 +162,11 @@ fn catalog(text: &str) -> Result<Catalog, String> {
163162
(relation.clone(), named)
164163
})
165164
.collect();
166-
Ok(Catalog { schema, columns })
165+
Ok(Catalog {
166+
schema,
167+
columns,
168+
warnings: found.warnings,
169+
})
167170
}
168171

169172
/// The playground's policy and session settings.
@@ -249,11 +252,21 @@ fn placed(mut refusal: Refusal, statement: &Expression, lines: &SourceLines) ->
249252
refusal
250253
}
251254

255+
/// Every stage's result for `sql` over `sources`, with a warning for each source left out.
252256
fn report(sql: &str, sources: &str, settings: &str) -> Value {
253257
let catalog = match catalog(sources) {
254258
Ok(catalog) => catalog,
255259
Err(error) => return json!({ "sources_error": error }),
256260
};
261+
let mut report = stages(sql, &catalog, settings);
262+
if !catalog.warnings.is_empty() {
263+
report["warnings"] = json!(catalog.warnings);
264+
}
265+
report
266+
}
267+
268+
/// Each stage's result for `sql` over `catalog`, as JSON.
269+
fn stages(sql: &str, catalog: &Catalog, settings: &str) -> Value {
257270
let settings: Settings = match settings.trim() {
258271
"" => Settings::default(),
259272
text => match serde_json::from_str(text) {
@@ -293,7 +306,7 @@ fn report(sql: &str, sources: &str, settings: &str) -> Value {
293306
(false, 1) => {
294307
let context = LineageContext {
295308
profile: &SNOWFLAKE,
296-
schema: &catalog,
309+
schema: catalog,
297310
sql: Some(sql),
298311
};
299312
query_name_findings(sql, Dialect::Snowflake, &ParseOptions::default(), context)

0 commit comments

Comments
 (0)