diff --git a/pyproject.toml b/pyproject.toml index b9bc286c..57fd2643 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dev = [ "playwright>=1.50.0", "fastapi>=0.115.0", "httpx>=0.28.0", + "jsonschema>=4.23,<5", "pyarrow>=14.0.0", "uvicorn>=0.34.0", # fakesnow pinned out until it supports sqlglot 30.x @@ -61,6 +62,9 @@ charts = [ fast = [ "sqlglot[c]>=30.1.0", ] +ossie = [ + "jsonschema>=4.23,<5", +] serve = [ "riffq>=0.1.0", "pyarrow>=14.0.0", @@ -126,7 +130,7 @@ all-databases = [ "sidemantic[postgres,bigquery,snowflake,clickhouse,databricks,spark,adbc]", ] full = [ - "sidemantic[workbench,mcp,apps,charts,lsp,dax,lookml,malloy,metricflow,widget,api]", + "sidemantic[workbench,mcp,apps,charts,lsp,dax,lookml,malloy,metricflow,widget,api,ossie]", ] [build-system] @@ -193,6 +197,7 @@ force-exclude = true exclude = [ "sidemantic/adapters/malloy_grammar", # ANTLR-generated files "sidemantic/adapters/holistics_grammar", # ANTLR-generated files + "tests/ossie-fixtures/upstream/validation/validate.py", # Exact pinned Apache validator fixture ] [tool.ruff.lint] diff --git a/sidemantic/interchange/__init__.py b/sidemantic/interchange/__init__.py new file mode 100644 index 00000000..8f40cc2e --- /dev/null +++ b/sidemantic/interchange/__init__.py @@ -0,0 +1 @@ +"""Typed interchange contracts that are independent from executable semantic models.""" diff --git a/sidemantic/interchange/ossie/__init__.py b/sidemantic/interchange/ossie/__init__.py new file mode 100644 index 00000000..84dbddad --- /dev/null +++ b/sidemantic/interchange/ossie/__init__.py @@ -0,0 +1,106 @@ +"""Typed Apache Ossie source-contract foundation.""" + +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSchemaProvenance, + OssieSourceLocation, + diagnostic_sort_key, + sort_diagnostics, +) +from sidemantic.interchange.ossie.documents import ( + FrozenJSONObject, + FrozenJSONValue, + JSONScalar, + OssieDocument, + OssieDocumentSource, + OssieLogicalDocument, + OssieOntologyDocument, + UnsupportedOssieDocument, + freeze_json, + thaw_json, +) +from sidemantic.interchange.ossie.lowering import OssieLoweringResult, lower_ossie_document +from sidemantic.interchange.ossie.parser import ( + OssieParseOptions, + OssieParseResult, + parse_ossie_document, +) +from sidemantic.interchange.ossie.profiles import ( + DBT_1_12_0_1_0_ALIAS, + DBT_1_12_0_1_1, + OSSIE_CORE_0_1_1, + OSSIE_CORE_0_2_0_DEV0, + OSSIE_PROFILES, + OssieConsumerProfile, + OssieImportPolicy, + OssieOptions, + OssiePreservationPolicy, + OssieProfile, + OssieProfileError, + OssieSerialization, + resolve_ossie_profile, +) +from sidemantic.interchange.ossie.semantic_validation import ( + SemanticDocumentKind, + SemanticValidationResult, + validate_ossie_semantics, +) +from sidemantic.interchange.ossie.serialization import ( + OssieSerializationError, + OssieSerializationResult, + serialize_ossie_document, +) +from sidemantic.interchange.ossie.synthesis import ( + OssieSynthesisError, + OssieSynthesisResult, + require_synthesized_document, + synthesize_ossie_document, +) + +__all__ = [ + "DBT_1_12_0_1_0_ALIAS", + "DBT_1_12_0_1_1", + "OSSIE_CORE_0_1_1", + "OSSIE_CORE_0_2_0_DEV0", + "OSSIE_PROFILES", + "FrozenJSONValue", + "FrozenJSONObject", + "JSONScalar", + "OssieConsumerProfile", + "OssieDiagnostic", + "OssieDiagnosticSeverity", + "OssieDocument", + "OssieDocumentSource", + "OssieImportPolicy", + "OssieLogicalDocument", + "OssieLoweringResult", + "OssieOntologyDocument", + "OssieOptions", + "OssieParseOptions", + "OssieParseResult", + "OssiePreservationPolicy", + "OssieProfile", + "OssieProfileError", + "OssieSchemaProvenance", + "OssieSerialization", + "OssieSerializationError", + "OssieSerializationResult", + "OssieSourceLocation", + "OssieSynthesisError", + "OssieSynthesisResult", + "SemanticDocumentKind", + "SemanticValidationResult", + "UnsupportedOssieDocument", + "diagnostic_sort_key", + "freeze_json", + "lower_ossie_document", + "parse_ossie_document", + "resolve_ossie_profile", + "require_synthesized_document", + "serialize_ossie_document", + "sort_diagnostics", + "synthesize_ossie_document", + "thaw_json", + "validate_ossie_semantics", +] diff --git a/sidemantic/interchange/ossie/diagnostics.py b/sidemantic/interchange/ossie/diagnostics.py new file mode 100644 index 00000000..f62c5cfa --- /dev/null +++ b/sidemantic/interchange/ossie/diagnostics.py @@ -0,0 +1,112 @@ +"""Structured diagnostics for Apache Ossie parsing, validation, and lowering.""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from enum import Enum + +from sidemantic.interchange.ossie.profiles import OssieProfile + + +class OssieDiagnosticSeverity(str, Enum): + ERROR = "error" + WARNING = "warning" + INFO = "info" + + +@dataclass(frozen=True, slots=True) +class OssieSourceLocation: + """Source identity and optional one-based text coordinates.""" + + identifier: str + line: int | None = None + column: int | None = None + end_line: int | None = None + end_column: int | None = None + + def __post_init__(self) -> None: + if not self.identifier: + raise ValueError("source identifier must not be empty") + for field_name in ("line", "column", "end_line", "end_column"): + value = getattr(self, field_name) + if value is not None and value < 1: + raise ValueError(f"{field_name} must be one-based") + if self.column is not None and self.line is None: + raise ValueError("column requires line") + if self.end_column is not None and self.end_line is None: + raise ValueError("end_column requires end_line") + + +@dataclass(frozen=True, slots=True) +class OssieSchemaProvenance: + """Identity of the pinned schema used to produce a diagnostic.""" + + schema_id: str + version: str | None = None + commit: str | None = None + sha256: str | None = None + + def __post_init__(self) -> None: + if not self.schema_id: + raise ValueError("schema_id must not be empty") + + +@dataclass(frozen=True, slots=True) +class OssieDiagnostic: + """One stable, source-addressable Ossie diagnostic.""" + + severity: OssieDiagnosticSeverity + code: str + message: str + json_pointer: str = "" + source: OssieSourceLocation | None = None + scope: str | None = None + profile: OssieProfile | None = None + schema: OssieSchemaProvenance | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "severity", OssieDiagnosticSeverity(self.severity)) + if not self.code or any(character.isspace() for character in self.code): + raise ValueError("diagnostic code must be non-empty and contain no whitespace") + if not self.message: + raise ValueError("diagnostic message must not be empty") + if self.json_pointer and not self.json_pointer.startswith("/"): + raise ValueError("json_pointer must be empty or start with '/'") + + +_SEVERITY_ORDER = { + OssieDiagnosticSeverity.ERROR: 0, + OssieDiagnosticSeverity.WARNING: 1, + OssieDiagnosticSeverity.INFO: 2, +} + + +def diagnostic_sort_key(diagnostic: OssieDiagnostic) -> tuple[object, ...]: + """Return a stable ordering by source position, object path, severity, and identity.""" + + source = diagnostic.source + schema = diagnostic.schema + return ( + source.identifier if source else "", + source.line if source and source.line is not None else 0, + source.column if source and source.column is not None else 0, + source.end_line if source and source.end_line is not None else 0, + source.end_column if source and source.end_column is not None else 0, + diagnostic.json_pointer, + _SEVERITY_ORDER[diagnostic.severity], + diagnostic.code, + diagnostic.message, + diagnostic.scope or "", + diagnostic.profile.identifier if diagnostic.profile else "", + schema.schema_id if schema else "", + schema.version or "" if schema else "", + schema.commit or "" if schema else "", + schema.sha256 or "" if schema else "", + ) + + +def sort_diagnostics(diagnostics: Iterable[OssieDiagnostic]) -> tuple[OssieDiagnostic, ...]: + """Materialize diagnostics in deterministic presentation order.""" + + return tuple(sorted(diagnostics, key=diagnostic_sort_key)) diff --git a/sidemantic/interchange/ossie/documents.py b/sidemantic/interchange/ossie/documents.py new file mode 100644 index 00000000..abf6a227 --- /dev/null +++ b/sidemantic/interchange/ossie/documents.py @@ -0,0 +1,219 @@ +"""Immutable Apache Ossie source-document contracts. + +These contracts preserve the parsed JSON-compatible data model and optionally +the original source bytes. They do not claim to reconstruct YAML comments, +anchors, quoting, scalar style, whitespace, or other lexical formatting. +""" + +from __future__ import annotations + +import hashlib +import math +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from typing import ClassVar, TypeAlias + +from sidemantic.interchange.ossie.profiles import OssieSerialization + +JSONScalar: TypeAlias = str | int | float | bool | None +ParsedJSONValue: TypeAlias = JSONScalar | Mapping[str, object] | list[object] | tuple[object, ...] + + +@dataclass(frozen=True, slots=True) +class FrozenJSONObject(Mapping[str, "FrozenJSONValue"]): + """An insertion-ordered, deeply immutable JSON object.""" + + _entries: tuple[tuple[str, FrozenJSONValue], ...] = () + + def __post_init__(self) -> None: + keys = [key for key, _ in self._entries] + if any(not isinstance(key, str) for key in keys): + raise TypeError("JSON object keys must be strings") + if len(keys) != len(set(keys)): + raise ValueError("JSON object keys must be unique") + object.__setattr__(self, "_entries", tuple((key, freeze_json(value)) for key, value in self._entries)) + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> FrozenJSONObject: + return cls(tuple((key, freeze_json(item)) for key, item in value.items())) + + def __getitem__(self, key: str) -> FrozenJSONValue: + for candidate, value in self._entries: + if candidate == key: + return value + raise KeyError(key) + + def __iter__(self) -> Iterator[str]: + return (key for key, _ in self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def to_dict(self) -> dict[str, object]: + """Return a mutable JSON-compatible copy for serialization.""" + + return {key: thaw_json(value) for key, value in self._entries} + + +FrozenJSONValue: TypeAlias = JSONScalar | FrozenJSONObject | tuple["FrozenJSONValue", ...] + + +def freeze_json(value: object) -> FrozenJSONValue: + """Copy JSON-compatible data into an immutable representation.""" + + if isinstance(value, FrozenJSONObject): + return value + if value is None or isinstance(value, (str, bool, int)): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("JSON numbers must be finite") + return value + if isinstance(value, Mapping): + return FrozenJSONObject.from_mapping(value) + if isinstance(value, (list, tuple)): + return tuple(freeze_json(item) for item in value) + raise TypeError(f"Unsupported parsed data type: {type(value).__name__}") + + +def thaw_json(value: FrozenJSONValue) -> object: + """Return a detached mutable JSON-compatible copy.""" + + if isinstance(value, FrozenJSONObject): + return value.to_dict() + if isinstance(value, tuple): + return [thaw_json(item) for item in value] + return value + + +def _escape_json_pointer_token(token: str) -> str: + return token.replace("~", "~0").replace("/", "~1") + + +def _present_fields(value: FrozenJSONValue, pointer: str = "") -> set[str]: + fields: set[str] = set() + if isinstance(value, FrozenJSONObject): + for key, child in value._entries: + child_pointer = f"{pointer}/{_escape_json_pointer_token(key)}" + fields.add(child_pointer) + fields.update(_present_fields(child, child_pointer)) + elif isinstance(value, tuple): + for index, child in enumerate(value): + fields.update(_present_fields(child, f"{pointer}/{index}")) + return fields + + +@dataclass(frozen=True, slots=True) +class OssieDocumentSource: + """Optional source identity and exact bytes retained with a parsed document.""" + + identifier: str | None = None + original_bytes: bytes | None = None + media_type: str | None = None + + def __post_init__(self) -> None: + if self.identifier is not None and not self.identifier: + raise ValueError("source identifier must not be empty") + if self.original_bytes is not None and not isinstance(self.original_bytes, bytes): + raise TypeError("original_bytes must be bytes") + + @property + def sha256(self) -> str | None: + if self.original_bytes is None: + return None + return hashlib.sha256(self.original_bytes).hexdigest() + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _OssieDocumentBase: + canonical_data: ParsedJSONValue | FrozenJSONObject + serialization: OssieSerialization + source: OssieDocumentSource | None = None + + _known_root_fields: ClassVar[frozenset[str]] = frozenset() + + def __post_init__(self) -> None: + object.__setattr__(self, "canonical_data", freeze_json(self.canonical_data)) + object.__setattr__(self, "serialization", OssieSerialization(self.serialization)) + + @property + def version(self) -> str | None: + if not isinstance(self.canonical_data, FrozenJSONObject): + return None + value = self.canonical_data.get("version") + return value if isinstance(value, str) else None + + @property + def present_fields(self) -> frozenset[str]: + """JSON pointers for fields explicitly present in the parsed data.""" + + return frozenset(_present_fields(self.canonical_data)) + + def is_field_present(self, json_pointer: str) -> bool: + return json_pointer in self.present_fields + + @property + def unknown_data(self) -> FrozenJSONObject: + """Unknown root fields; nested unknown data remains in ``canonical_data``.""" + + if not isinstance(self.canonical_data, FrozenJSONObject): + return FrozenJSONObject() + return FrozenJSONObject( + tuple((key, value) for key, value in self.canonical_data._entries if key not in self._known_root_fields) + ) + + def to_parsed_data(self) -> object: + return thaw_json(self.canonical_data) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class OssieLogicalDocument(_OssieDocumentBase): + """A logical-layer Ossie document, before validation or runtime lowering.""" + + _known_root_fields: ClassVar[frozenset[str]] = frozenset({"version", "dialects", "vendors", "semantic_model"}) + + def __post_init__(self) -> None: + super(OssieLogicalDocument, self).__post_init__() + if not isinstance(self.canonical_data, FrozenJSONObject): + raise TypeError("An Ossie logical document must have an object root") + + @property + def semantic_model_value(self) -> FrozenJSONValue | None: + return self.canonical_data.get("semantic_model") + + @property + def semantic_models(self) -> tuple[FrozenJSONValue, ...]: + value = self.semantic_model_value + return value if isinstance(value, tuple) else () + + +@dataclass(frozen=True, slots=True, kw_only=True) +class OssieOntologyDocument(_OssieDocumentBase): + """An ontology-layer Ossie document, preserved without reasoning semantics.""" + + _known_root_fields: ClassVar[frozenset[str]] = frozenset( + {"version", "name", "description", "ai_context", "ontology", "ontology_mappings"} + ) + + def __post_init__(self) -> None: + super(OssieOntologyDocument, self).__post_init__() + if not isinstance(self.canonical_data, FrozenJSONObject): + raise TypeError("An Ossie ontology document must have an object root") + + @property + def ontology(self) -> FrozenJSONValue | None: + return self.canonical_data.get("ontology") + + @property + def ontology_mappings(self) -> FrozenJSONValue | None: + return self.canonical_data.get("ontology_mappings") + + +@dataclass(frozen=True, slots=True, kw_only=True) +class UnsupportedOssieDocument(_OssieDocumentBase): + """Parsed data that cannot yet be classified as a supported Ossie document family.""" + + reason: str | None = None + + +OssieDocument: TypeAlias = OssieLogicalDocument | OssieOntologyDocument | UnsupportedOssieDocument diff --git a/sidemantic/interchange/ossie/expression_validation.py b/sidemantic/interchange/ossie/expression_validation.py new file mode 100644 index 00000000..3296e3e6 --- /dev/null +++ b/sidemantic/interchange/ossie/expression_validation.py @@ -0,0 +1,55 @@ +"""Structural validation for executable Apache Ossie SQL expressions. + +The expression-language proposal is versioned independently from the JSON +schemas. This validator deliberately enforces only its stable scalar-expression +boundary. Dialect-specific functions remain extensions and are not rejected by +an ANSI function allowlist. +""" + +from __future__ import annotations + +import sqlglot +from sqlglot import expressions as exp + +OSSIE_EXPRESSION_PROPOSAL_COMMIT = "88e0011148283302c9a04cd0287e00e0b9d87354" +OSSIE_EXPRESSION_PROPOSAL_PATH = "core-spec/expression_language.md" + +_FORBIDDEN_NODE_NAMES = ( + "Query", + "DDL", + "DML", + "Command", + "Drop", + "Transaction", + "Commit", + "Rollback", + "Where", + "Group", + "Join", + "With", +) +_FORBIDDEN_NODE_TYPES = tuple( + node_type for name in _FORBIDDEN_NODE_NAMES if isinstance((node_type := getattr(exp, name, None)), type) +) + + +def scalar_sql_expression_error(expression: str, *, sqlglot_dialect: str | None) -> str | None: + """Return why SQL is not one scalar Ossie expression, otherwise ``None``. + + Parsing uses the selected executable dialect. The structural gate rejects + queries, subqueries, clauses, set operations, DDL, DML, and commands even + when SQLGlot accepts them as valid warehouse SQL. + """ + + try: + parsed = sqlglot.parse(expression, read=sqlglot_dialect) + except sqlglot.errors.SqlglotError as exc: + return str(exc) + if len(parsed) != 1 or parsed[0] is None: + return "expression must contain exactly one SQL expression" + + root = parsed[0] + for node in root.walk(): + if isinstance(node, _FORBIDDEN_NODE_TYPES): + return f"Ossie expressions cannot contain {type(node).__name__}" + return None diff --git a/sidemantic/interchange/ossie/identifier.py b/sidemantic/interchange/ossie/identifier.py new file mode 100644 index 00000000..22aa738e --- /dev/null +++ b/sidemantic/interchange/ossie/identifier.py @@ -0,0 +1,35 @@ +"""Identifier comparison rules from the Apache Ossie expression proposal.""" + +from __future__ import annotations + +OSSIE_IDENTIFIER_MAX_LENGTH = 128 + + +def is_quoted_identifier(identifier: str) -> bool: + """Return whether *identifier* uses Ossie's ANSI double-quote form.""" + + return len(identifier) >= 2 and identifier.startswith('"') and identifier.endswith('"') + + +def normalize_identifier(identifier: str) -> str: + """Return the proposal's exact comparison key without changing source text. + + Regular identifiers compare after upper-casing. Double-quoted identifiers + compare after removing their outer quotes and unescaping doubled quotes. + """ + + if is_quoted_identifier(identifier): + return identifier[1:-1].replace('""', '"') + return identifier.upper() + + +def identifier_length(identifier: str) -> int: + """Return the identifier-body length used by the proposal's 128-char limit.""" + + return len(normalize_identifier(identifier)) if is_quoted_identifier(identifier) else len(identifier) + + +def identifier_within_limit(identifier: str) -> bool: + """Return whether *identifier* satisfies Ossie's proposed size limit.""" + + return identifier_length(identifier) <= OSSIE_IDENTIFIER_MAX_LENGTH diff --git a/sidemantic/interchange/ossie/lowering.py b/sidemantic/interchange/ossie/lowering.py new file mode 100644 index 00000000..4dc21a62 --- /dev/null +++ b/sidemantic/interchange/ossie/lowering.py @@ -0,0 +1,649 @@ +"""Loss-aware lowering from Apache Ossie documents into executable scopes.""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +import sqlglot +from sqlglot import expressions as exp + +from sidemantic.core.dimension import Dimension +from sidemantic.core.metric import Metric +from sidemantic.core.model import Model +from sidemantic.core.registry import reset_current_layer, set_current_layer +from sidemantic.core.relationship import Relationship +from sidemantic.core.semantic_catalog import CompiledSemanticScope, SemanticCatalog +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSourceLocation, + sort_diagnostics, +) +from sidemantic.interchange.ossie.documents import OssieLogicalDocument, OssieOntologyDocument +from sidemantic.interchange.ossie.expression_validation import scalar_sql_expression_error +from sidemantic.interchange.ossie.identifier import identifier_within_limit, normalize_identifier +from sidemantic.interchange.ossie.parser import OssieParseResult +from sidemantic.interchange.ossie.profiles import OssieImportPolicy +from sidemantic.interchange.ossie.semantic_validation import ( + SemanticValidationResult, + validate_ossie_semantics, +) +from sidemantic.interchange.ossie.validation import SchemaValidationResult, validate_ossie_schema + +JSONObject = Mapping[str, object] +_TEMPORAL_TYPES = frozenset({"Date", "Time", "DateTime", "DateTimeTz"}) +_NUMERIC_TYPES = frozenset({"Integer", "Decimal", "Float"}) +_DIALECT_LABELS = { + "bigquery": "BIGQUERY", + "databricks": "DATABRICKS", + "snowflake": "SNOWFLAKE", +} +_SQLGLOT_DIALECTS = { + "bigquery": "bigquery", + "databricks": "databricks", + "duckdb": "duckdb", + "postgres": "postgres", + "postgresql": "postgres", + "snowflake": "snowflake", + "spark": "spark", +} +_LOWERING_IMPLEMENTATION = "sidemantic-ossie-lowering-v1" + + +@dataclass(frozen=True, slots=True) +class OssieLoweringResult: + """A preserved source document plus any safely executable scope projections.""" + + parse_result: OssieParseResult + catalog: SemanticCatalog + schema_validation: SchemaValidationResult | None + semantic_validation: SemanticValidationResult | None + lowering_diagnostics: tuple[OssieDiagnostic, ...] + + @property + def document(self): + return self.parse_result.document + + @property + def diagnostics(self) -> tuple[OssieDiagnostic, ...]: + schema = ( + self.schema_validation.diagnostics + if self.schema_validation is not None and self.schema_validation is not self.parse_result.schema_validation + else () + ) + semantic = self.semantic_validation.diagnostics if self.semantic_validation else () + return sort_diagnostics((*self.parse_result.diagnostics, *schema, *semantic, *self.lowering_diagnostics)) + + @property + def executable(self) -> bool: + return len(self.catalog) > 0 + + @property + def valid(self) -> bool: + return not any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in self.diagnostics) + + +def _mapping(value: object) -> JSONObject | None: + return value if isinstance(value, Mapping) else None + + +def _array(value: object) -> Sequence[object] | None: + return value if isinstance(value, (list, tuple)) else None + + +def _name(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _source_location(result: OssieParseResult) -> OssieSourceLocation | None: + source = result.document.source + if source is None or source.identifier is None: + return None + return OssieSourceLocation(identifier=source.identifier) + + +def _diagnostic( + result: OssieParseResult, + *, + code: str, + message: str, + pointer: str, + scope: str | None = None, +) -> OssieDiagnostic: + return OssieDiagnostic( + severity=OssieDiagnosticSeverity.ERROR, + code=code, + message=message, + json_pointer=pointer, + source=_source_location(result), + scope=scope, + profile=result.profile, + ) + + +def _canonical_hash(value: object) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def _document_id(result: OssieParseResult) -> str: + source = result.document.source + if source and source.sha256: + return f"sha256:{source.sha256}" + return _canonical_hash(result.document.to_parsed_data()) + + +def _normalize_dialect(value: str) -> str: + return value.strip().lower().replace("-", "_") + + +def _expression_for_target(expression: object, target_dialect: str) -> tuple[str, str] | None: + expression_object = _mapping(expression) + variants = _array(expression_object.get("dialects")) if expression_object else None + if not variants: + return None + + by_dialect: dict[str, str] = {} + for value in variants: + variant = _mapping(value) + if variant is None: + continue + dialect = variant.get("dialect") + text = variant.get("expression") + if isinstance(dialect, str) and dialect and isinstance(text, str) and text.strip(): + by_dialect.setdefault(dialect.upper(), text) + + normalized = _normalize_dialect(target_dialect) + target_label = _DIALECT_LABELS.get(normalized) + if target_label and target_label in by_dialect: + return by_dialect[target_label], target_label + if "ANSI_SQL" in by_dialect: + return by_dialect["ANSI_SQL"], "ANSI_SQL" + return None + + +def _sql_expression_error(expression: str, target_dialect: str) -> str | None: + dialect = _SQLGLOT_DIALECTS.get(_normalize_dialect(target_dialect)) + if dialect is None: + return f"target dialect {target_dialect!r} has no configured SQL parser" + return scalar_sql_expression_error(expression, sqlglot_dialect=dialect) + + +def _classify_source(source: str, source_dialect: str | None) -> tuple[str, str] | None: + dialect = _SQLGLOT_DIALECTS.get(_normalize_dialect(source_dialect)) if source_dialect else None + try: + parsed = sqlglot.parse_one(source, read=dialect) + except sqlglot.errors.ParseError: + parsed = None + if isinstance(parsed, (exp.Query, exp.Subquery)): + return "query", source + + try: + sqlglot.parse_one(source, read=dialect, into=exp.Table) + except sqlglot.errors.ParseError: + return None + return "table", source + + +def _runtime_dimension_type(logical_type: str | None, is_time: bool) -> str: + if is_time: + return "time" + if logical_type == "Boolean": + return "boolean" + if logical_type in _NUMERIC_TYPES: + return "numeric" + return "categorical" + + +def _key_value(columns: Sequence[object] | None) -> str | list[str] | None: + if not columns or not all(isinstance(column, str) and column for column in columns): + return None + values = list(columns) + return values[0] if len(values) == 1 else values + + +def _canonical_name_lookup(names: Sequence[str]) -> dict[str, str]: + """Map comparison keys to exact declarations, excluding ambiguous names.""" + + counts = Counter(normalize_identifier(name) for name in names if identifier_within_limit(name)) + return { + normalize_identifier(name): name + for name in names + if identifier_within_limit(name) and counts[normalize_identifier(name)] == 1 + } + + +def _canonical_columns(columns: Sequence[object] | None, declarations: Mapping[str, str]) -> list[str] | None: + if not columns or not all(isinstance(column, str) and column for column in columns): + return None + resolved: list[str] = [] + seen: set[str] = set() + for column in columns: + if not identifier_within_limit(column): + return None + declaration = declarations.get(normalize_identifier(column)) + if declaration is None or declaration in seen: + return None + resolved.append(declaration) + seen.add(declaration) + return resolved + + +def _unique_named_items(values: Sequence[object] | None) -> list[tuple[int, JSONObject, str]]: + entries: list[tuple[int, JSONObject, str]] = [] + names: list[str] = [] + for index, value in enumerate(values or ()): + item = _mapping(value) + item_name = _name(item.get("name")) if item else None + if item is not None and item_name is not None: + entries.append((index, item, item_name)) + names.append(item_name) + counts = Counter(normalize_identifier(name) for name in names if identifier_within_limit(name)) + return [ + entry for entry in entries if identifier_within_limit(entry[2]) and counts[normalize_identifier(entry[2])] == 1 + ] + + +def _lower_scope( + result: OssieParseResult, + semantic_model: JSONObject, + *, + scope_id: str, + scope_index: int, + document_id: str, + target_dialect: str, + diagnostics: list[OssieDiagnostic], + document_diagnostics: tuple[OssieDiagnostic, ...], +) -> CompiledSemanticScope: + graph = SemanticGraph() + source_dialect = result.options.source_dialect if result.options else None + dataset_values = _array(semantic_model.get("datasets")) + lowered_models: dict[str, Model] = {} + + # Model/Metric construction has a legacy auto-registration hook. Lowering + # must be isolated from any ambient user layer. + registration_token = set_current_layer(None) + try: + for dataset_index, dataset, dataset_name in _unique_named_items(dataset_values): + pointer = f"/semantic_model/{scope_index}/datasets/{dataset_index}" + source = dataset.get("source") + if not isinstance(source, str) or not source.strip(): + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.source_unusable", + message=f"Dataset {dataset_name!r} has no executable source.", + pointer=f"{pointer}/source", + scope=scope_id, + ) + ) + continue + classified_source = _classify_source(source, source_dialect) + if classified_source is None: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.source_ambiguous", + message=f"Dataset source for {dataset_name!r} is neither a table reference nor a SQL query.", + pointer=f"{pointer}/source", + scope=scope_id, + ) + ) + continue + + dimensions: list[Dimension] = [] + fields = _array(dataset.get("fields")) + for field_index, field, field_name in _unique_named_items(fields): + field_pointer = f"{pointer}/fields/{field_index}" + selected = _expression_for_target(field.get("expression"), target_dialect) + if selected is None: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.expression_unavailable", + message=( + f"Field {dataset_name}.{field_name} has no expression for " + f"target dialect {target_dialect!r} or ANSI_SQL." + ), + pointer=f"{field_pointer}/expression", + scope=scope_id, + ) + ) + continue + expression, _ = selected + expression_error = _sql_expression_error(expression, target_dialect) + if expression_error is not None: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.expression_invalid", + message=( + f"Field {dataset_name}.{field_name} is not one executable " + f"{target_dialect} SQL expression: {expression_error}" + ), + pointer=f"{field_pointer}/expression", + scope=scope_id, + ) + ) + continue + logical_type = field.get("datatype") if isinstance(field.get("datatype"), str) else None + dimension = _mapping(field.get("dimension")) + declared_is_time = dimension.get("is_time") if dimension and "is_time" in dimension else None + declared_is_time = declared_is_time if isinstance(declared_is_time, bool) else None + effective_is_time = ( + declared_is_time if declared_is_time is not None else logical_type in _TEMPORAL_TYPES + ) + try: + runtime_dimension = Dimension( + name=field_name, + type=_runtime_dimension_type(logical_type, effective_is_time), + logical_data_type=logical_type, + declared_is_time=declared_is_time, + sql=expression, + description=field.get("description") if isinstance(field.get("description"), str) else None, + label=field.get("label") if isinstance(field.get("label"), str) else None, + ) + except (TypeError, ValueError) as exc: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.field_unexecutable", + message=f"Field {dataset_name}.{field_name} cannot be represented safely: {exc}", + pointer=field_pointer, + scope=scope_id, + ) + ) + continue + dimensions.append(runtime_dimension) + + field_declarations = _canonical_name_lookup([dimension.name for dimension in dimensions]) + primary_columns = _canonical_columns(_array(dataset.get("primary_key")), field_declarations) + primary_key = _key_value(primary_columns) + unique_keys: list[list[str]] = [] + for value in _array(dataset.get("unique_keys")) or (): + resolved_key = _canonical_columns(_array(value), field_declarations) + if resolved_key is not None: + unique_keys.append(resolved_key) + source_kind, source_text = classified_source + try: + model = Model( + name=dataset_name, + table=source_text if source_kind == "table" else None, + sql=source_text if source_kind == "query" else None, + description=dataset.get("description") if isinstance(dataset.get("description"), str) else None, + primary_key=primary_key, + unique_keys=unique_keys or None, + dimensions=dimensions, + default_time_dimension=None, + metadata={"ossie_source_kind": source_kind, "ossie_pointer": pointer}, + ) + except (TypeError, ValueError) as exc: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.dataset_unexecutable", + message=f"Dataset {dataset_name!r} cannot be represented safely: {exc}", + pointer=pointer, + scope=scope_id, + ) + ) + continue + graph.add_model(model) + lowered_models[normalize_identifier(dataset_name)] = model + + relationships = _array(semantic_model.get("relationships")) + for relationship_index, relationship, edge_id in _unique_named_items(relationships): + pointer = f"/semantic_model/{scope_index}/relationships/{relationship_index}" + from_name = _name(relationship.get("from")) + to_name = _name(relationship.get("to")) + from_columns = _array(relationship.get("from_columns")) + to_columns = _array(relationship.get("to_columns")) + from_model = ( + lowered_models.get(normalize_identifier(from_name)) + if from_name and identifier_within_limit(from_name) + else None + ) + to_model = ( + lowered_models.get(normalize_identifier(to_name)) + if to_name and identifier_within_limit(to_name) + else None + ) + from_declarations = ( + _canonical_name_lookup([dimension.name for dimension in from_model.dimensions]) if from_model else {} + ) + to_declarations = ( + _canonical_name_lookup([dimension.name for dimension in to_model.dimensions]) if to_model else {} + ) + canonical_from_columns = _canonical_columns(from_columns, from_declarations) + canonical_to_columns = _canonical_columns(to_columns, to_declarations) + from_key = _key_value(canonical_from_columns) + to_key = _key_value(canonical_to_columns) + target_keys = {tuple(to_model.primary_key_columns)} if to_model and to_model.primary_key_columns else set() + if to_model: + target_keys.update(tuple(key) for key in to_model.unique_keys or ()) + safe = ( + from_model is not None + and to_model is not None + and from_key is not None + and to_key is not None + and len(canonical_from_columns or ()) == len(canonical_to_columns or ()) + and tuple(canonical_to_columns or ()) in target_keys + and all(from_model.get_dimension(column) is not None for column in canonical_from_columns or ()) + and all(to_model.get_dimension(column) is not None for column in canonical_to_columns or ()) + ) + if not safe: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.relationship_unsafe", + message=f"Relationship {edge_id!r} is preserved but excluded from executable topology.", + pointer=pointer, + scope=scope_id, + ) + ) + continue + from_model.relationships.append( + Relationship( + name=to_model.name, + edge_id=edge_id, + type="many_to_one", + foreign_key=from_key, + primary_key=to_key, + ) + ) + + metrics = _array(semantic_model.get("metrics")) + for metric_index, metric, metric_name in _unique_named_items(metrics): + pointer = f"/semantic_model/{scope_index}/metrics/{metric_index}" + selected = _expression_for_target(metric.get("expression"), target_dialect) + if selected is None: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.expression_unavailable", + message=( + f"Metric {metric_name!r} has no expression for target dialect " + f"{target_dialect!r} or ANSI_SQL." + ), + pointer=f"{pointer}/expression", + scope=scope_id, + ) + ) + continue + expression, _ = selected + expression_error = _sql_expression_error(expression, target_dialect) + if expression_error is not None: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.expression_invalid", + message=( + f"Metric {metric_name!r} is not one executable {target_dialect} " + f"SQL expression: {expression_error}" + ), + pointer=f"{pointer}/expression", + scope=scope_id, + ) + ) + continue + try: + metric_object = Metric( + name=metric_name, + sql=expression, + logical_data_type=(metric.get("datatype") if isinstance(metric.get("datatype"), str) else None), + description=metric.get("description") if isinstance(metric.get("description"), str) else None, + ) + except (TypeError, ValueError) as exc: + diagnostics.append( + _diagnostic( + result, + code="ossie.lowering.metric_unexecutable", + message=f"Metric {metric_name!r} cannot be represented safely: {exc}", + pointer=pointer, + scope=scope_id, + ) + ) + continue + graph.add_metric(metric_object) + finally: + reset_current_layer(registration_token) + + graph.build_adjacency() + scope_data = semantic_model + content_id = _canonical_hash(scope_data) + compilation_id = _canonical_hash( + { + "implementation": _LOWERING_IMPLEMENTATION, + "document_version": result.document.version, + "consumer_profile": result.profile.identifier if result.profile else None, + "schema_commit": result.schema_validation.schema_commit if result.schema_validation else None, + "source_dialect": _normalize_dialect(source_dialect) if source_dialect else None, + "target_dialect": _normalize_dialect(target_dialect), + "lowering_policy": result.options.import_policy.value if result.options else "unknown", + } + ) + scope_diagnostics = sort_diagnostics( + ( + *(diagnostic for diagnostic in document_diagnostics if diagnostic.scope in (None, scope_id)), + *(diagnostic for diagnostic in diagnostics if diagnostic.scope == scope_id), + ) + ) + scope_valid = not any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in scope_diagnostics) + return CompiledSemanticScope( + scope_id=scope_id, + document_id=document_id, + content_id=content_id, + compilation_id=compilation_id, + runtime=graph, + target_dialect=target_dialect, + lowering_policy=result.options.import_policy.value if result.options else "unknown", + valid=scope_valid, + diagnostics=scope_diagnostics, + provenance={ + "document_version": result.document.version, + "semantic_model_index": scope_index, + "source_identifier": result.document.source.identifier if result.document.source else None, + }, + ) + + +def lower_ossie_document( + parse_result: OssieParseResult, + *, + target_dialect: str | None = None, +) -> OssieLoweringResult: + """Validate and project each logical semantic model into an isolated scope.""" + + if not isinstance(parse_result, OssieParseResult): + raise TypeError("parse_result must be an OssieParseResult") + + document = parse_result.document + schema_validation = parse_result.schema_validation + if schema_validation is None and isinstance(document, (OssieLogicalDocument, OssieOntologyDocument)): + schema_validation = validate_ossie_schema( + document.to_parsed_data(), + profile=parse_result.profile, + consumer_profile=parse_result.parse_options.consumer_profile, + ) + + semantic_validation = None + if isinstance(document, (OssieLogicalDocument, OssieOntologyDocument)): + semantic_validation = validate_ossie_semantics(document, profile=parse_result.profile) + + diagnostics: list[OssieDiagnostic] = [] + selected_target = target_dialect or (parse_result.options.target_dialect if parse_result.options else None) + if selected_target is None: + diagnostics.append( + _diagnostic( + parse_result, + code="ossie.lowering.target_dialect_required", + message="Executable lowering requires an explicit target dialect.", + pointer="", + ) + ) + + all_pre_lowering = [*parse_result.diagnostics] + if schema_validation and schema_validation is not parse_result.schema_validation: + all_pre_lowering.extend(schema_validation.diagnostics) + if semantic_validation: + all_pre_lowering.extend(semantic_validation.diagnostics) + strict = parse_result.options is None or parse_result.options.import_policy is OssieImportPolicy.STRICT + has_errors = any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in all_pre_lowering) + if ( + selected_target is None + or not isinstance(document, OssieLogicalDocument) + or parse_result.options is None + or (strict and has_errors) + ): + return OssieLoweringResult( + parse_result=parse_result, + catalog=SemanticCatalog(), + schema_validation=schema_validation, + semantic_validation=semantic_validation, + lowering_diagnostics=tuple(diagnostics), + ) + + parsed = document.to_parsed_data() + root = _mapping(parsed) + semantic_models = _array(root.get("semantic_model")) if root else None + named_models = [ + (index, model, model_name) + for index, value in enumerate(semantic_models or ()) + if (model := _mapping(value)) is not None + if (model_name := _name(model.get("name"))) is not None + ] + name_counts = Counter(normalize_identifier(name) for _, _, name in named_models if identifier_within_limit(name)) + document_id = _document_id(parse_result) + scopes = [] + for index, semantic_model, name in named_models: + if not identifier_within_limit(name): + continue + scope_id = name if name_counts[normalize_identifier(name)] == 1 else f"{name}@{index}" + scopes.append( + _lower_scope( + parse_result, + semantic_model, + scope_id=scope_id, + scope_index=index, + document_id=document_id, + target_dialect=selected_target, + diagnostics=diagnostics, + document_diagnostics=tuple(all_pre_lowering), + ) + ) + + if strict and any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in diagnostics): + scopes = [] + + return OssieLoweringResult( + parse_result=parse_result, + catalog=SemanticCatalog(scopes), + schema_validation=schema_validation, + semantic_validation=semantic_validation, + lowering_diagnostics=sort_diagnostics(diagnostics), + ) diff --git a/sidemantic/interchange/ossie/parser.py b/sidemantic/interchange/ossie/parser.py new file mode 100644 index 00000000..b93416c0 --- /dev/null +++ b/sidemantic/interchange/ossie/parser.py @@ -0,0 +1,574 @@ +"""Exact-byte parsing and document-family classification for Apache Ossie.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import PurePosixPath +from urllib.parse import urlsplit + +import yaml + +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSourceLocation, + sort_diagnostics, +) +from sidemantic.interchange.ossie.documents import ( + OssieDocument, + OssieDocumentSource, + OssieLogicalDocument, + OssieOntologyDocument, + UnsupportedOssieDocument, +) +from sidemantic.interchange.ossie.profiles import ( + OssieConsumerProfile, + OssieImportPolicy, + OssieOptions, + OssiePreservationPolicy, + OssieProfile, + OssieProfileError, + OssieSerialization, +) +from sidemantic.interchange.ossie.validation import SchemaValidationResult, validate_ossie_schema + +_MAX_SOURCE_BYTES = 16 * 1024 * 1024 +_MAX_NESTING_DEPTH = 256 + +try: # pragma: no cover - depends on how PyYAML was built + from yaml import CSafeLoader as _SafeLoader +except ImportError: # pragma: no cover - Pyodide does not ship libyaml + from yaml import SafeLoader as _SafeLoader + + +@dataclass(frozen=True, slots=True) +class OssieParseOptions: + """Options known before the document supplies its schema version.""" + + serialization: OssieSerialization | None = None + consumer_profile: OssieConsumerProfile = OssieConsumerProfile.OSSIE_CORE + import_policy: OssieImportPolicy = OssieImportPolicy.STRICT + source_dialect: str | None = None + target_dialect: str | None = None + preservation_policy: OssiePreservationPolicy = OssiePreservationPolicy.CANONICAL_DATA + validate_schema: bool = False + + def __post_init__(self) -> None: + try: + if self.serialization is not None: + object.__setattr__(self, "serialization", OssieSerialization(self.serialization)) + object.__setattr__(self, "consumer_profile", OssieConsumerProfile(self.consumer_profile)) + object.__setattr__(self, "import_policy", OssieImportPolicy(self.import_policy)) + object.__setattr__(self, "preservation_policy", OssiePreservationPolicy(self.preservation_policy)) + except ValueError as exc: + raise OssieProfileError(f"Invalid Ossie parse option: {exc}") from exc + + for field_name in ("source_dialect", "target_dialect"): + value = getattr(self, field_name) + if value is not None and (not isinstance(value, str) or not value.strip()): + raise OssieProfileError(f"{field_name} must be a non-empty string when provided") + if not isinstance(self.validate_schema, bool): + raise OssieProfileError("validate_schema must be a boolean") + + +@dataclass(frozen=True, slots=True) +class OssieParseResult: + """Immutable result of parsing, classifying, and optionally validating bytes.""" + + document: OssieDocument + parse_options: OssieParseOptions + options: OssieOptions | None + profile: OssieProfile | None + parse_diagnostics: tuple[OssieDiagnostic, ...] + schema_validation: SchemaValidationResult | None = None + + def __post_init__(self) -> None: + if self.options is None and self.profile is not None: + raise ValueError("profile cannot be resolved when options are unresolved") + if self.options is not None and self.profile is not self.options.profile: + raise ValueError("profile must match the resolved options") + object.__setattr__(self, "parse_diagnostics", sort_diagnostics(self.parse_diagnostics)) + + @property + def diagnostics(self) -> tuple[OssieDiagnostic, ...]: + """Return parser and schema diagnostics in stable presentation order.""" + + schema_diagnostics = self.schema_validation.diagnostics if self.schema_validation else () + return sort_diagnostics((*self.parse_diagnostics, *schema_diagnostics)) + + @property + def valid(self) -> bool: + return not any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in self.diagnostics) + + @property + def blocks_lowering(self) -> bool: + """Whether the selected import policy would block a later lowering stage.""" + + return self.parse_options.import_policy is OssieImportPolicy.STRICT and not self.valid + + +class _DuplicateKeyError(ValueError): + def __init__(self, key: object, *, line: int | None = None, column: int | None = None) -> None: + super().__init__(f"Duplicate mapping key {key!r}") + self.key = key + self.line = line + self.column = column + + +class _NonFiniteJSONNumberError(ValueError): + pass + + +class _UniqueKeySafeLoader(_SafeLoader): + """Safe YAML loader that rejects duplicate mapping keys at every depth.""" + + def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[object, object]: + self.flatten_mapping(node) + mapping: dict[object, object] = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + duplicate = key in mapping + except TypeError as exc: + mark = key_node.start_mark + raise _DuplicateKeyError( + "", + line=mark.line + 1, + column=mark.column + 1, + ) from exc + if duplicate: + mark = key_node.start_mark + raise _DuplicateKeyError(key, line=mark.line + 1, column=mark.column + 1) + mapping[key] = self.construct_object(value_node, deep=deep) + return mapping + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + value: dict[str, object] = {} + for key, item in pairs: + if key in value: + raise _DuplicateKeyError(key) + value[key] = item + return value + + +def _reject_non_finite_json_number(value: str) -> None: + raise _NonFiniteJSONNumberError(f"JSON number {value!r} is not finite") + + +def _source_location( + identifier: str, + *, + line: int | None = None, + column: int | None = None, +) -> OssieSourceLocation: + return OssieSourceLocation(identifier=identifier, line=line, column=column) + + +def _diagnostic( + *, + code: str, + message: str, + identifier: str, + json_pointer: str = "", + line: int | None = None, + column: int | None = None, +) -> OssieDiagnostic: + return OssieDiagnostic( + severity=OssieDiagnosticSeverity.ERROR, + code=code, + message=message, + json_pointer=json_pointer, + source=_source_location(identifier, line=line, column=column), + ) + + +def _serialization_from_suffix(identifier: str) -> OssieSerialization | None: + path = urlsplit(identifier).path + suffix = PurePosixPath(path).suffix.lower() + if suffix == ".json": + return OssieSerialization.JSON + if suffix in {".yaml", ".yml"}: + return OssieSerialization.YAML + return None + + +def _infer_serialization(source_bytes: bytes, identifier: str) -> OssieSerialization: + from_suffix = _serialization_from_suffix(identifier) + if from_suffix is not None: + return from_suffix + + try: + text = source_bytes.decode("utf-8-sig") + except UnicodeDecodeError: + return OssieSerialization.YAML + + significant = text.lstrip() + if significant.startswith(("{", "[")): + return OssieSerialization.JSON + try: + json.loads(text) + except (json.JSONDecodeError, UnicodeDecodeError): + return OssieSerialization.YAML + return OssieSerialization.JSON + + +def _parse_serialized_data(text: str, serialization: OssieSerialization) -> object: + if serialization is OssieSerialization.JSON: + return json.loads( + text, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_non_finite_json_number, + ) + return yaml.load(text, Loader=_UniqueKeySafeLoader) + + +def _classify_document( + parsed: object, + *, + serialization: OssieSerialization, + source: OssieDocumentSource, +) -> tuple[OssieDocument, str | None]: + if not isinstance(parsed, Mapping): + reason = "The Ossie document root must be an object" + return ( + UnsupportedOssieDocument( + canonical_data=parsed, + serialization=serialization, + source=source, + reason=reason, + ), + "ossie.document.root_type", + ) + + has_logical_root = "semantic_model" in parsed + has_ontology_root = "ontology" in parsed or "ontology_mappings" in parsed + if has_logical_root and has_ontology_root: + reason = "The document mixes logical semantic_model and ontology root families" + return ( + UnsupportedOssieDocument( + canonical_data=parsed, + serialization=serialization, + source=source, + reason=reason, + ), + "ossie.document.family_mixed", + ) + if has_logical_root: + return ( + OssieLogicalDocument(canonical_data=parsed, serialization=serialization, source=source), + None, + ) + if has_ontology_root: + return ( + OssieOntologyDocument(canonical_data=parsed, serialization=serialization, source=source), + None, + ) + + reason = "The document contains none of semantic_model, ontology, or ontology_mappings" + return ( + UnsupportedOssieDocument( + canonical_data=parsed, + serialization=serialization, + source=source, + reason=reason, + ), + "ossie.document.family_missing", + ) + + +def _resolve_options( + document: OssieDocument, + *, + serialization: OssieSerialization, + parse_options: OssieParseOptions, + identifier: str, +) -> tuple[OssieOptions | None, OssieDiagnostic | None]: + version = document.version + if version is None: + canonical = document.canonical_data + if isinstance(canonical, Mapping) and "version" in canonical: + diagnostic = _diagnostic( + code="ossie.profile.version_type", + message="The Ossie document version must be a string", + identifier=identifier, + json_pointer="/version", + ) + else: + diagnostic = _diagnostic( + code="ossie.profile.version_missing", + message="The Ossie document does not declare a version", + identifier=identifier, + ) + return None, diagnostic + + try: + options = OssieOptions( + schema_version=version, + serialization=serialization, + consumer_profile=parse_options.consumer_profile, + import_policy=parse_options.import_policy, + source_dialect=parse_options.source_dialect, + target_dialect=parse_options.target_dialect, + preservation_policy=parse_options.preservation_policy, + ) + except OssieProfileError as exc: + return ( + None, + _diagnostic( + code="ossie.profile.unsupported", + message=str(exc), + identifier=identifier, + json_pointer="/version", + ), + ) + return options, None + + +def _schema_profile_name(document: OssieDocument) -> str | None: + version = document.version + if version is None: + return None + if isinstance(document, OssieLogicalDocument): + return f"logical-{version}" + if isinstance(document, OssieOntologyDocument): + return f"ontology-{version}" + return None + + +def parse_ossie_document( + source_bytes: bytes, + *, + source_identifier: str = "", + options: OssieParseOptions | None = None, +) -> OssieParseResult: + """Parse exact YAML or JSON bytes without lowering into runtime objects. + + Syntax, document-family, profile, and requested schema-validation failures + are returned as diagnostics rather than escaping as parser tracebacks. + """ + + if not isinstance(source_bytes, bytes): + raise TypeError("source_bytes must be exact immutable bytes") + if not source_identifier: + raise ValueError("source_identifier must not be empty") + + parse_options = options or OssieParseOptions() + serialization = parse_options.serialization or _infer_serialization(source_bytes, source_identifier) + media_type = "application/json" if serialization is OssieSerialization.JSON else "application/yaml" + source = OssieDocumentSource( + identifier=source_identifier, + original_bytes=( + source_bytes if parse_options.preservation_policy is OssiePreservationPolicy.SOURCE_BYTES else None + ), + media_type=media_type, + ) + + diagnostics: list[OssieDiagnostic] = [] + if len(source_bytes) > _MAX_SOURCE_BYTES: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="The source exceeds the parser input budget", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.limit", + message=f"Apache Ossie source exceeds the {_MAX_SOURCE_BYTES}-byte parser limit", + identifier=source_identifier, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + try: + text = source_bytes.decode("utf-8-sig") + except UnicodeDecodeError as exc: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="The source is not valid UTF-8", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.encoding", + message=f"Apache Ossie sources must be UTF-8: {exc}", + identifier=source_identifier, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + + try: + parsed = _parse_serialized_data(text, serialization) + except _DuplicateKeyError as exc: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason=str(exc), + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.duplicate_key", + message=str(exc), + identifier=source_identifier, + line=exc.line, + column=exc.column, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + except json.JSONDecodeError as exc: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="Invalid JSON syntax", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.syntax", + message=exc.msg, + identifier=source_identifier, + line=exc.lineno, + column=exc.colno, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + except yaml.MarkedYAMLError as exc: + mark = exc.problem_mark + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="Invalid YAML syntax", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.syntax", + message=exc.problem or str(exc), + identifier=source_identifier, + line=mark.line + 1 if mark else None, + column=mark.column + 1 if mark else None, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + except _NonFiniteJSONNumberError as exc: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason=str(exc), + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.non_json_value", + message=str(exc), + identifier=source_identifier, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + except RecursionError: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="The source exceeds the parser nesting budget", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.limit", + message=f"Apache Ossie source exceeds the {_MAX_NESTING_DEPTH}-level nesting limit", + identifier=source_identifier, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + + stack = [(parsed, 0)] + nesting_exceeded = False + while stack: + value, depth = stack.pop() + if depth > _MAX_NESTING_DEPTH: + nesting_exceeded = True + break + if isinstance(value, dict): + stack.extend((child, depth + 1) for child in value.values()) + elif isinstance(value, (list, tuple)): + stack.extend((child, depth + 1) for child in value) + if nesting_exceeded: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="The source exceeds the parser nesting budget", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.limit", + message=f"Apache Ossie source exceeds the {_MAX_NESTING_DEPTH}-level nesting limit", + identifier=source_identifier, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + + try: + document, family_diagnostic_code = _classify_document( + parsed, + serialization=serialization, + source=source, + ) + except (TypeError, ValueError, RecursionError) as exc: + document = UnsupportedOssieDocument( + canonical_data=None, + serialization=serialization, + source=source, + reason="Parsed input is outside Ossie's JSON-compatible data model", + ) + diagnostics.append( + _diagnostic( + code="ossie.parse.non_json_value", + message=str(exc), + identifier=source_identifier, + ) + ) + return OssieParseResult(document, parse_options, None, None, tuple(diagnostics)) + + if family_diagnostic_code is not None: + diagnostics.append( + _diagnostic( + code=family_diagnostic_code, + message=document.reason or "Unsupported Ossie document family", + identifier=source_identifier, + ) + ) + + resolved_options, profile_diagnostic = _resolve_options( + document, + serialization=serialization, + parse_options=parse_options, + identifier=source_identifier, + ) + if profile_diagnostic is not None: + diagnostics.append(profile_diagnostic) + + schema_validation = None + if parse_options.validate_schema: + validation_profile = ( + resolved_options.profile if resolved_options is not None else _schema_profile_name(document) + ) + schema_validation = validate_ossie_schema( + document.to_parsed_data(), + profile=validation_profile, + consumer_profile=parse_options.consumer_profile, + ) + + profile = resolved_options.profile if resolved_options is not None else None + return OssieParseResult( + document=document, + parse_options=parse_options, + options=resolved_options, + profile=profile, + parse_diagnostics=tuple(diagnostics), + schema_validation=schema_validation, + ) diff --git a/sidemantic/interchange/ossie/profiles.py b/sidemantic/interchange/ossie/profiles.py new file mode 100644 index 00000000..2da04805 --- /dev/null +++ b/sidemantic/interchange/ossie/profiles.py @@ -0,0 +1,167 @@ +"""Apache Ossie profile and import/export option contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class OssieProfileError(ValueError): + """Raised when an Ossie option combination does not identify a supported profile.""" + + +class OssieSerialization(str, Enum): + """Source or output serialization, independent from the Ossie schema version.""" + + YAML = "yaml" + JSON = "json" + + +class OssieConsumerProfile(str, Enum): + """The contract against which an Ossie document is interpreted.""" + + OSSIE_CORE = "ossie-core" + DBT_1_12 = "dbt-1.12" + + +class OssieImportPolicy(str, Enum): + """How validation diagnostics affect later lowering.""" + + STRICT = "strict" + PERMISSIVE = "permissive" + + +class OssiePreservationPolicy(str, Enum): + """The round-trip material retained in addition to typed projections.""" + + CANONICAL_DATA = "canonical-data" + SOURCE_BYTES = "source-bytes" + + +@dataclass(frozen=True, slots=True) +class OssieProfile: + """One supported schema-version and consumer-profile contract.""" + + schema_version: str + consumer_profile: OssieConsumerProfile + upstream_schema_version: str | None + compatibility_alias_for: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "consumer_profile", OssieConsumerProfile(self.consumer_profile)) + if not self.schema_version or self.schema_version != self.schema_version.strip(): + raise OssieProfileError("profile schema_version must be a non-empty value without surrounding whitespace") + if self.compatibility_alias_for is not None and self.upstream_schema_version is not None: + raise OssieProfileError("a compatibility alias cannot also claim an upstream schema version") + if self.compatibility_alias_for is None and self.upstream_schema_version is None: + raise OssieProfileError("a non-alias profile must identify its upstream schema version") + + @property + def identifier(self) -> str: + return f"{self.consumer_profile.value}:{self.schema_version}" + + @property + def is_compatibility_alias(self) -> bool: + return self.compatibility_alias_for is not None + + @property + def validation_schema_version(self) -> str: + """Return the pinned logical schema version used to validate this profile. + + Compatibility aliases retain their declared version in source and output; + this value is only the schema version used during validation. + """ + + return self.compatibility_alias_for or self.schema_version + + +OSSIE_CORE_0_1_1 = OssieProfile( + schema_version="0.1.1", + consumer_profile=OssieConsumerProfile.OSSIE_CORE, + upstream_schema_version="0.1.1", +) +OSSIE_CORE_0_2_0_DEV0 = OssieProfile( + schema_version="0.2.0.dev0", + consumer_profile=OssieConsumerProfile.OSSIE_CORE, + upstream_schema_version="0.2.0.dev0", +) +DBT_1_12_0_1_0_ALIAS = OssieProfile( + schema_version="0.1.0", + consumer_profile=OssieConsumerProfile.DBT_1_12, + upstream_schema_version=None, + compatibility_alias_for="0.1.1", +) +DBT_1_12_0_1_1 = OssieProfile( + schema_version="0.1.1", + consumer_profile=OssieConsumerProfile.DBT_1_12, + upstream_schema_version="0.1.1", +) + +OSSIE_PROFILES = ( + OSSIE_CORE_0_1_1, + OSSIE_CORE_0_2_0_DEV0, + DBT_1_12_0_1_0_ALIAS, + DBT_1_12_0_1_1, +) +_PROFILE_INDEX = {(profile.schema_version, profile.consumer_profile): profile for profile in OSSIE_PROFILES} + + +def resolve_ossie_profile(schema_version: str, consumer_profile: OssieConsumerProfile | str) -> OssieProfile: + """Resolve a supported profile without treating serialization as a version selector.""" + + try: + normalized_consumer = OssieConsumerProfile(consumer_profile) + except ValueError as exc: + raise OssieProfileError(f"Unsupported Ossie consumer profile: {consumer_profile!r}") from exc + + profile = _PROFILE_INDEX.get((schema_version, normalized_consumer)) + if profile is not None: + return profile + + if schema_version == "0.1.0": + raise OssieProfileError( + "Ossie version 0.1.0 is only a dbt-1.12 compatibility alias; it is not an upstream Ossie schema" + ) + + supported = ", ".join( + profile.schema_version for profile in OSSIE_PROFILES if profile.consumer_profile is normalized_consumer + ) + raise OssieProfileError( + f"Unsupported schema version {schema_version!r} for {normalized_consumer.value}; supported: {supported}" + ) + + +@dataclass(frozen=True, slots=True) +class OssieOptions: + """Validated options for parsing, preserving, lowering, or exporting Ossie data.""" + + schema_version: str + serialization: OssieSerialization + consumer_profile: OssieConsumerProfile = OssieConsumerProfile.OSSIE_CORE + import_policy: OssieImportPolicy = OssieImportPolicy.STRICT + source_dialect: str | None = None + target_dialect: str | None = None + preservation_policy: OssiePreservationPolicy = OssiePreservationPolicy.CANONICAL_DATA + + def __post_init__(self) -> None: + try: + object.__setattr__(self, "serialization", OssieSerialization(self.serialization)) + object.__setattr__(self, "consumer_profile", OssieConsumerProfile(self.consumer_profile)) + object.__setattr__(self, "import_policy", OssieImportPolicy(self.import_policy)) + object.__setattr__(self, "preservation_policy", OssiePreservationPolicy(self.preservation_policy)) + except ValueError as exc: + raise OssieProfileError(f"Invalid Ossie option: {exc}") from exc + + if not self.schema_version or self.schema_version != self.schema_version.strip(): + raise OssieProfileError("schema_version must be a non-empty value without surrounding whitespace") + + for field_name in ("source_dialect", "target_dialect"): + value = getattr(self, field_name) + if value is not None and (not isinstance(value, str) or not value.strip()): + raise OssieProfileError(f"{field_name} must be a non-empty string when provided") + + resolve_ossie_profile(self.schema_version, self.consumer_profile) + + @property + def profile(self) -> OssieProfile: + return resolve_ossie_profile(self.schema_version, self.consumer_profile) diff --git a/sidemantic/interchange/ossie/schemas/README.md b/sidemantic/interchange/ossie/schemas/README.md new file mode 100644 index 00000000..06dbec21 --- /dev/null +++ b/sidemantic/interchange/ossie/schemas/README.md @@ -0,0 +1,14 @@ +# Vendored Apache Ossie schemas + +`manifest.json` is the source of truth for schema identity, origin, commit, and +SHA-256 integrity. Logical schema files are byte-for-byte copies of their pinned +upstream assets. + +The ontology directory contains both the untouched upstream schema and the +runtime schema. The runtime copy has two deterministic rewrites recorded in the +manifest: a unique ontology `$id`, and local URN references to the pinned logical +schema. Runtime validation registers only vendored resources and has no remote +resource retriever. + +Do not update an asset without updating its source commit, checksums, fixture +expectations, and validation tests together. diff --git a/sidemantic/interchange/ossie/schemas/logical/0.1.1/schema.json b/sidemantic/interchange/ossie/schemas/logical/0.1.1/schema.json new file mode 100644 index 00000000..a4a6f92a --- /dev/null +++ b/sidemantic/interchange/ossie/schemas/logical/0.1.1/schema.json @@ -0,0 +1,344 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", + "title": "OSI Core Metadata Specification", + "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.1.1", + "description": "OSI specification version" + }, + "dialects": { + "type": "array", + "description": "Supported expression language dialects (enumeration definition)", + "items": { + "$ref": "#/$defs/Dialect" + } + }, + "vendors": { + "type": "array", + "description": "Supported vendors for custom extensions (enumeration definition)", + "items": { + "$ref": "#/$defs/Vendor" + } + }, + "semantic_model": { + "type": "array", + "description": "Collection of semantic model definitions", + "items": { + "$ref": "#/$defs/SemanticModel" + } + } + }, + "required": ["version", "semantic_model"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL"], + "description": "Supported SQL and expression language dialects" + }, + "Vendor": { + "type": "string", + "enum": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], + "description": "Supported vendors for custom extensions" + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "CustomExtension": { + "type": "object", + "description": "Vendor-specific attributes for extensibility", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "string", + "description": "JSON string containing vendor-specific data" + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "required": ["dialects"], + "additionalProperties": false + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Indicates if this is a time-based dimension for temporal filtering" + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Primary key columns (single or composite)" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": "Array of unique key definitions (each can be single or composite)" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Foreign key columns in the 'from' dataset" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Primary/unique key columns in the 'to' dataset" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Quantifiable measures spanning datasets" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/sidemantic/interchange/ossie/schemas/logical/0.2.0.dev0/schema.json b/sidemantic/interchange/ossie/schemas/logical/0.2.0.dev0/schema.json new file mode 100644 index 00000000..f24e45f1 --- /dev/null +++ b/sidemantic/interchange/ossie/schemas/logical/0.2.0.dev0/schema.json @@ -0,0 +1,352 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/core-spec/osi-schema.json", + "title": "Apache Ossie Core Metadata Specification", + "description": "JSON Schema for validating Apache Ossie semantic model definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.2.0.dev0", + "description": "Apache Ossie specification version" + }, + "semantic_model": { + "type": "array", + "description": "Collection of semantic model definitions", + "items": { + "$ref": "#/$defs/SemanticModel" + } + } + }, + "required": ["version", "semantic_model"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY"], + "description": "Supported SQL and expression language dialects" + }, + "Vendor": { + "type": "string", + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], + "description": "Vendor name for custom extensions. Any string value is accepted." + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "CustomExtension": { + "type": "object", + "description": "Vendor-specific attributes for extensibility", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "string", + "description": "JSON string containing vendor-specific data" + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "required": ["dialects"], + "additionalProperties": false + }, + "DataType": { + "type": "string", + "enum": [ + "String", + "Integer", + "Decimal", + "Float", + "Boolean", + "Date", + "Time", + "DateTime", + "DateTimeTz", + "Opaque" + ], + "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown; use `Opaque` plus `custom_extensions` for a known type outside the portable vocabulary." + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, or `DateTimeTz`, and `false` otherwise. Set `is_time: false` explicitly to opt a temporal-typed column (such as an audit timestamp) out of time-dimension treatment." + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Primary key columns (single or composite)" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": "Array of unique key definitions (each can be single or composite)" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Foreign key columns in the 'from' dataset" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Primary/unique key columns in the 'to' dataset" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Quantifiable measures spanning datasets" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/sidemantic/interchange/ossie/schemas/manifest.json b/sidemantic/interchange/ossie/schemas/manifest.json new file mode 100644 index 00000000..87de1c09 --- /dev/null +++ b/sidemantic/interchange/ossie/schemas/manifest.json @@ -0,0 +1,75 @@ +{ + "manifest_version": 1, + "profiles": { + "logical-0.1.1": { + "document_kind": "logical", + "version": "0.1.1", + "schema_dialect": "https://json-schema.org/draft/2020-12/schema", + "resource_uri": "urn:sidemantic:ossie:schema:logical:0.1.1", + "runtime_path": "logical/0.1.1/schema.json", + "runtime_sha256": "c1e9adec39562786aa78809665fba568797b15f4c53a0847d9cbcf2dead1bc94", + "upstream_path": "logical/0.1.1/schema.json", + "upstream_sha256": "c1e9adec39562786aa78809665fba568797b15f4c53a0847d9cbcf2dead1bc94", + "source": { + "repository": "https://github.com/open-semantic-interchange/OSI", + "commit": "faf581054dcf7964d5fe0ceae7d6f415c8ce32a5", + "path": "core-spec/osi-schema.json", + "url": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/faf581054dcf7964d5fe0ceae7d6f415c8ce32a5/core-spec/osi-schema.json", + "license": "Apache-2.0" + }, + "transformations": [] + }, + "logical-0.2.0.dev0": { + "document_kind": "logical", + "version": "0.2.0.dev0", + "schema_dialect": "https://json-schema.org/draft/2020-12/schema", + "resource_uri": "urn:sidemantic:ossie:schema:logical:0.2.0.dev0", + "runtime_path": "logical/0.2.0.dev0/schema.json", + "runtime_sha256": "8ce9f82aa92080265f9ae119e31cda5bef062f489674d3c467245c2d4c5ff264", + "upstream_path": "logical/0.2.0.dev0/schema.json", + "upstream_sha256": "8ce9f82aa92080265f9ae119e31cda5bef062f489674d3c467245c2d4c5ff264", + "source": { + "repository": "https://github.com/apache/ossie", + "commit": "88e0011148283302c9a04cd0287e00e0b9d87354", + "path": "core-spec/osi-schema.json", + "url": "https://raw.githubusercontent.com/apache/ossie/88e0011148283302c9a04cd0287e00e0b9d87354/core-spec/osi-schema.json", + "license": "Apache-2.0" + }, + "transformations": [] + }, + "ontology-0.2.0.dev0": { + "document_kind": "ontology", + "version": "0.2.0.dev0", + "schema_dialect": "https://json-schema.org/draft/2020-12/schema", + "resource_uri": "urn:sidemantic:ossie:schema:ontology:0.2.0.dev0", + "runtime_path": "ontology/0.2.0.dev0/schema.json", + "runtime_sha256": "555820756a7d30bc6986ce1b57feaa9937ec4ddd3288878b8af0e3361d824a41", + "upstream_path": "ontology/0.2.0.dev0/upstream.json", + "upstream_sha256": "c0ce26ff658aff52307f01bdc564061d194c1987e930d61ff498e63456b9b41d", + "source": { + "repository": "https://github.com/apache/ossie", + "commit": "88e0011148283302c9a04cd0287e00e0b9d87354", + "path": "ontology/ontology.json", + "url": "https://raw.githubusercontent.com/apache/ossie/88e0011148283302c9a04cd0287e00e0b9d87354/ontology/ontology.json", + "license": "Apache-2.0" + }, + "dependencies": [ + "logical-0.2.0.dev0" + ], + "transformations": [ + { + "operation": "replace_id", + "from": "https://github.com/apache/ossie/core-spec/osi-schema.json", + "to": "urn:sidemantic:ossie:schema:ontology:0.2.0.dev0", + "reason": "The upstream ontology schema duplicates the logical schema ID." + }, + { + "operation": "replace_reference_base", + "from": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/osi-schema.json", + "to": "urn:sidemantic:ossie:schema:logical:0.2.0.dev0", + "reason": "Resolve ontology references against the pinned logical schema without network access." + } + ] + } + } +} diff --git a/sidemantic/interchange/ossie/schemas/ontology/0.2.0.dev0/schema.json b/sidemantic/interchange/ossie/schemas/ontology/0.2.0.dev0/schema.json new file mode 100644 index 00000000..84717876 --- /dev/null +++ b/sidemantic/interchange/ossie/schemas/ontology/0.2.0.dev0/schema.json @@ -0,0 +1,298 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:sidemantic:ossie:schema:ontology:0.2.0.dev0", + "title": "Apache Ossie Ontology Metadata Specification", + "description": "JSON Schema for validating Apache Ossie ontology definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.2.0.dev0", + "description": "Ontology specification version" + }, + "name": { + "type": "string", + "description": "Unique identifier for the ontology" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "urn:sidemantic:ossie:schema:logical:0.2.0.dev0#/$defs/AIContext" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this ontology" + }, + "ontology": { + "type": "array", + "items": { + "$ref": "#/$defs/OntologyComponent" + }, + "minItems": 1, + "description": "Components that define the concepts and relationships in this ontology" + }, + "ontology_mappings": { + "type": "array", + "description": "Collection of ontology maps from logical models", + "items": { + "$ref": "#/$defs/OntologyMap" + } + } + }, + "required": ["version", "name", "ontology"], + "additionalProperties": false, + "$defs": { + "OntologyComponent": { + "type": "object", + "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", + "properties": { + "concept": { + "type": "string", + "description": "Unique name of the concept defined by this component" + }, + "type": { + "$ref": "#/$defs/ConceptType" + }, + "description": { + "type": "string", + "description": "Human-readable description of the concept" + }, + "extends": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Indicates that this concept extends one or more other concepts" + }, + "derived_by": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that define how this concept is derived" + }, + "identify_by": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of relationships to use as the preferred identifier of this concept" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this concept" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines relationships that pertain primarily to the concept defined in this component" + } + }, + "required": ["concept", "type"], + "additionalProperties": false + }, + "Expression": { + "type": "string", + "description": "ANSI SQL expression" + }, + "Relationship": { + "type": "object", + "description": "Relationship between concepts in the ontology", + "properties": { + "name": { + "type": "string", + "description": "Name of the relationship" + }, + "description": { + "type": "string", + "description": "Human-readable description of the relationship" + }, + "roles": { + "type": "array", + "items": { + "$ref": "#/$defs/Role" + }, + "description": "Additional roles in this relationship" + }, + "multiplicity": { + "$ref": "#/$defs/Multiplicity" + }, + "derived_by": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that define how this concept is derived" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this relationship" + }, + "verbalizes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Natural language expressions that verbalize this relationship" + } + }, + "required": ["name", "verbalizes"], + "additionalProperties": false + }, + "ConceptMapping": { + "type": "object", + "description": "Mappings from logical model constructs to some ontology component", + "properties": { + "concept": { + "type": "string", + "description": "Name of the concept whose part of the ontology we are mapping to" + }, + "object_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ObjectMapping" + }, + "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" + }, + "link_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/LinkMapping" + }, + "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" + } + }, + "required": ["concept"], + "additionalProperties": false + }, + "ConceptType": { + "type": "string", + "enum": [ "EntityType", "ValueType" ], + "description": "A concept is either an entity type or a value type" + }, + "ReferentMapping": { + "type": "object", + "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", + "properties": { + "relationship": { + "type": "string", + "description": "Name of referent relationship" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "referent_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMapping" + } + } + }, + "required": ["relationship"], + "additionalProperties": false + }, + "Role": { + "type": "object", + "description": "Role in some relationship (the container)", + "properties": { + "concept": { + "type": "string", + "description": "Name of the concept playing this role" + }, + "name": { + "type": "string", + "description": "Optional name of this role, used when the same concept plays multiple roles in the same relationship" + } + }, + "required": ["concept"], + "additionalProperties": false + }, + "ObjectMapping": { + "type": "object", + "description": "Pattern of logical-level expressions for identifying objects of some concept using the values in one or more fields", + "properties": { + "concept": { + "type": "string", + "description": "Name of the concept whose objects we are mapping to" + }, + "referent_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMapping" + }, + "description": "Maps logical-model constructs to referent relationships of this entity type" + }, + "expression": { + "$ref": "#/$defs/Expression" + } + }, + "additionalProperties": false + }, + "LinkMapping": { + "type": "object", + "description": "Mapping from logical schema to the links of relationships in the ontology", + "properties": { + "relationship": { + "type": "string", + "description": "Name of relationship being populated by this mapping node" + }, + "object_mapping": { + "$ref": "#/$defs/ObjectMapping" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/LinkMapping" + }, + "description": "Relationship maps at the next level in this hierarchy" + } + }, + "required": ["object_mapping"], + "additionalProperties": false + }, + "Multiplicity": { + "type": "string", + "enum": [ "ManyToOne", "OneToOne" ], + "description": "Relationship multiplicity" + }, + "OntologyMap": { + "type": "object", + "description": "Map from the constructs of some logical model to some ontology", + "properties": { + "name": { + "type": "string", + "description": "Name of this ontology map" + }, + "description": { + "type": "string", + "description": "Human-readable description of this ontology map" + }, + "semantic_model": { + "$ref": "urn:sidemantic:ossie:schema:logical:0.2.0.dev0#/$defs/SemanticModel" + }, + "concept_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ConceptMapping" + }, + "description": "Maps logical model constructs to some concept and its relationships in the ontology" + } + }, + "required": ["semantic_model", "concept_mappings"], + "additionalProperties": false + } + } +} diff --git a/sidemantic/interchange/ossie/schemas/ontology/0.2.0.dev0/upstream.json b/sidemantic/interchange/ossie/schemas/ontology/0.2.0.dev0/upstream.json new file mode 100644 index 00000000..81578a39 --- /dev/null +++ b/sidemantic/interchange/ossie/schemas/ontology/0.2.0.dev0/upstream.json @@ -0,0 +1,298 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/core-spec/osi-schema.json", + "title": "Apache Ossie Ontology Metadata Specification", + "description": "JSON Schema for validating Apache Ossie ontology definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.2.0.dev0", + "description": "Ontology specification version" + }, + "name": { + "type": "string", + "description": "Unique identifier for the ontology" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/osi-schema.json#/$defs/AIContext" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this ontology" + }, + "ontology": { + "type": "array", + "items": { + "$ref": "#/$defs/OntologyComponent" + }, + "minItems": 1, + "description": "Components that define the concepts and relationships in this ontology" + }, + "ontology_mappings": { + "type": "array", + "description": "Collection of ontology maps from logical models", + "items": { + "$ref": "#/$defs/OntologyMap" + } + } + }, + "required": ["version", "name", "ontology"], + "additionalProperties": false, + "$defs": { + "OntologyComponent": { + "type": "object", + "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", + "properties": { + "concept": { + "type": "string", + "description": "Unique name of the concept defined by this component" + }, + "type": { + "$ref": "#/$defs/ConceptType" + }, + "description": { + "type": "string", + "description": "Human-readable description of the concept" + }, + "extends": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Indicates that this concept extends one or more other concepts" + }, + "derived_by": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that define how this concept is derived" + }, + "identify_by": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Names of relationships to use as the preferred identifier of this concept" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this concept" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines relationships that pertain primarily to the concept defined in this component" + } + }, + "required": ["concept", "type"], + "additionalProperties": false + }, + "Expression": { + "type": "string", + "description": "ANSI SQL expression" + }, + "Relationship": { + "type": "object", + "description": "Relationship between concepts in the ontology", + "properties": { + "name": { + "type": "string", + "description": "Name of the relationship" + }, + "description": { + "type": "string", + "description": "Human-readable description of the relationship" + }, + "roles": { + "type": "array", + "items": { + "$ref": "#/$defs/Role" + }, + "description": "Additional roles in this relationship" + }, + "multiplicity": { + "$ref": "#/$defs/Multiplicity" + }, + "derived_by": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that define how this concept is derived" + }, + "requires": { + "type": "array", + "items": { + "$ref": "#/$defs/Expression" + }, + "description": "Expressions that constrain the population of this relationship" + }, + "verbalizes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Natural language expressions that verbalize this relationship" + } + }, + "required": ["name", "verbalizes"], + "additionalProperties": false + }, + "ConceptMapping": { + "type": "object", + "description": "Mappings from logical model constructs to some ontology component", + "properties": { + "concept": { + "type": "string", + "description": "Name of the concept whose part of the ontology we are mapping to" + }, + "object_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ObjectMapping" + }, + "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" + }, + "link_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/LinkMapping" + }, + "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" + } + }, + "required": ["concept"], + "additionalProperties": false + }, + "ConceptType": { + "type": "string", + "enum": [ "EntityType", "ValueType" ], + "description": "A concept is either an entity type or a value type" + }, + "ReferentMapping": { + "type": "object", + "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", + "properties": { + "relationship": { + "type": "string", + "description": "Name of referent relationship" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "referent_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMapping" + } + } + }, + "required": ["relationship"], + "additionalProperties": false + }, + "Role": { + "type": "object", + "description": "Role in some relationship (the container)", + "properties": { + "concept": { + "type": "string", + "description": "Name of the concept playing this role" + }, + "name": { + "type": "string", + "description": "Optional name of this role, used when the same concept plays multiple roles in the same relationship" + } + }, + "required": ["concept"], + "additionalProperties": false + }, + "ObjectMapping": { + "type": "object", + "description": "Pattern of logical-level expressions for identifying objects of some concept using the values in one or more fields", + "properties": { + "concept": { + "type": "string", + "description": "Name of the concept whose objects we are mapping to" + }, + "referent_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ReferentMapping" + }, + "description": "Maps logical-model constructs to referent relationships of this entity type" + }, + "expression": { + "$ref": "#/$defs/Expression" + } + }, + "additionalProperties": false + }, + "LinkMapping": { + "type": "object", + "description": "Mapping from logical schema to the links of relationships in the ontology", + "properties": { + "relationship": { + "type": "string", + "description": "Name of relationship being populated by this mapping node" + }, + "object_mapping": { + "$ref": "#/$defs/ObjectMapping" + }, + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/LinkMapping" + }, + "description": "Relationship maps at the next level in this hierarchy" + } + }, + "required": ["object_mapping"], + "additionalProperties": false + }, + "Multiplicity": { + "type": "string", + "enum": [ "ManyToOne", "OneToOne" ], + "description": "Relationship multiplicity" + }, + "OntologyMap": { + "type": "object", + "description": "Map from the constructs of some logical model to some ontology", + "properties": { + "name": { + "type": "string", + "description": "Name of this ontology map" + }, + "description": { + "type": "string", + "description": "Human-readable description of this ontology map" + }, + "semantic_model": { + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/osi-schema.json#/$defs/SemanticModel" + }, + "concept_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/ConceptMapping" + }, + "description": "Maps logical model constructs to some concept and its relationships in the ontology" + } + }, + "required": ["semantic_model", "concept_mappings"], + "additionalProperties": false + } + } +} diff --git a/sidemantic/interchange/ossie/semantic_validation.py b/sidemantic/interchange/ossie/semantic_validation.py new file mode 100644 index 00000000..c16acc3f --- /dev/null +++ b/sidemantic/interchange/ossie/semantic_validation.py @@ -0,0 +1,859 @@ +"""Semantic validation for parsed Apache Ossie documents. + +This module is the second validation stage. JSON Schema owns document shape, +required properties, primitive types, and enum membership; these checks own +cross-object identity, references, and the invariants required to lower a +logical scope safely. Callers decide whether diagnostics block lowering. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Literal, TypeAlias + +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSourceLocation, + sort_diagnostics, +) +from sidemantic.interchange.ossie.documents import ( + OssieLogicalDocument, + OssieOntologyDocument, +) +from sidemantic.interchange.ossie.identifier import ( + OSSIE_IDENTIFIER_MAX_LENGTH, + identifier_length, + identifier_within_limit, + normalize_identifier, +) +from sidemantic.interchange.ossie.profiles import ( + OssieConsumerProfile, + OssieProfile, + OssieProfileError, + resolve_ossie_profile, +) + +SemanticDocumentKind = Literal["logical", "ontology", "unsupported"] +SemanticFailureStage = Literal["semantic"] +JSONObject: TypeAlias = Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class SemanticValidationResult: + """Immutable result of the semantic validation stage.""" + + valid: bool + document_kind: SemanticDocumentKind + checked_scopes: tuple[str, ...] + diagnostics: tuple[OssieDiagnostic, ...] + failure_stage: SemanticFailureStage | None = None + stage: Literal["semantic"] = "semantic" + + def to_dict(self) -> dict[str, Any]: + """Return a detached, serialization-friendly result.""" + + return { + "stage": self.stage, + "failure_stage": self.failure_stage, + "valid": self.valid, + "document_kind": self.document_kind, + "checked_scopes": list(self.checked_scopes), + "diagnostics": [ + { + "code": diagnostic.code, + "severity": diagnostic.severity.value, + "message": diagnostic.message, + "json_pointer": diagnostic.json_pointer, + "scope": diagnostic.scope, + "profile": diagnostic.profile.identifier if diagnostic.profile else None, + "source": ( + { + "identifier": diagnostic.source.identifier, + "line": diagnostic.source.line, + "column": diagnostic.source.column, + "end_line": diagnostic.source.end_line, + "end_column": diagnostic.source.end_column, + } + if diagnostic.source + else None + ), + } + for diagnostic in self.diagnostics + ], + } + + +def _escape_json_pointer_token(value: object) -> str: + return str(value).replace("~", "~0").replace("/", "~1") + + +def _pointer(parent: str, *parts: object) -> str: + suffix = "/".join(_escape_json_pointer_token(part) for part in parts) + if not suffix: + return parent + return f"{parent}/{suffix}" if parent else f"/{suffix}" + + +def _mapping(value: object) -> JSONObject | None: + return value if isinstance(value, Mapping) else None + + +def _array(value: object) -> Sequence[object] | None: + if isinstance(value, (list, tuple)): + return value + return None + + +def _name(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _profile_for( + document: JSONObject, + explicit_profile: OssieProfile | None = None, +) -> OssieProfile | None: + if explicit_profile is not None: + return explicit_profile + version = document.get("version") + if not isinstance(version, str): + return None + try: + return resolve_ossie_profile(version, OssieConsumerProfile.OSSIE_CORE) + except OssieProfileError: + return None + + +class _SemanticValidator: + def __init__( + self, + *, + profile: OssieProfile | None, + source: OssieSourceLocation | None, + ) -> None: + self.profile = profile + self.source = source + self.diagnostics: list[OssieDiagnostic] = [] + self.checked_scopes: list[str] = [] + + def emit( + self, + code: str, + message: str, + json_pointer: str, + *, + scope: str | None = None, + ) -> None: + self.diagnostics.append( + OssieDiagnostic( + severity=OssieDiagnosticSeverity.ERROR, + code=code, + message=message, + json_pointer=json_pointer, + source=self.source, + scope=scope, + profile=self.profile, + ) + ) + + def duplicate_names( + self, + values: Sequence[object] | None, + *, + parent_pointer: str, + namespace: str, + code: str, + scope: str | None = None, + ) -> Counter[str]: + counts: Counter[str] = Counter() + if values is None: + return counts + + first_pointer: dict[str, str] = {} + for index, value in enumerate(values): + item = _mapping(value) + if item is None: + continue + item_name = _name(item.get("name")) + if item_name is None: + continue + name_pointer = _pointer(parent_pointer, index, "name") + self.validate_identifier_length(item_name, name_pointer, scope=scope) + normalized_name = normalize_identifier(item_name) + counts[normalized_name] += 1 + if normalized_name in first_pointer: + self.emit( + code, + ( + f"Duplicate {namespace} name {item_name!r} after Ossie identifier normalization; " + f"first declared at {first_pointer[normalized_name]}." + ), + name_pointer, + scope=scope, + ) + else: + first_pointer[normalized_name] = name_pointer + return counts + + def validate_identifier_length( + self, + identifier: str, + json_pointer: str, + *, + scope: str | None = None, + ) -> bool: + length = identifier_length(identifier) + if length <= OSSIE_IDENTIFIER_MAX_LENGTH: + return True + self.emit( + "ossie.semantic.identifier.length_exceeded", + ( + f"Identifier {identifier!r} is {length} characters after quoted-identifier decoding; " + f"Apache Ossie identifiers are limited to {OSSIE_IDENTIFIER_MAX_LENGTH} characters." + ), + json_pointer, + scope=scope, + ) + return False + + def validate_expression( + self, + expression: object, + *, + expression_pointer: str, + scope: str, + ) -> None: + expression_object = _mapping(expression) + if expression_object is None or "dialects" not in expression_object: + return + + dialects_pointer = _pointer(expression_pointer, "dialects") + dialects = _array(expression_object.get("dialects")) + if dialects is None: + return + if not dialects: + self.emit( + "ossie.semantic.expression.dialects_empty", + "An executable expression must provide at least one dialect variant.", + dialects_pointer, + scope=scope, + ) + return + + first_dialect_pointer: dict[str, str] = {} + for index, value in enumerate(dialects): + variant = _mapping(value) + if variant is None: + continue + variant_pointer = _pointer(dialects_pointer, index) + dialect = variant.get("dialect") + if isinstance(dialect, str): + dialect_pointer = _pointer(variant_pointer, "dialect") + if not dialect.strip(): + self.emit( + "ossie.semantic.expression.dialect_empty", + "Expression dialect labels must not be empty.", + dialect_pointer, + scope=scope, + ) + elif dialect.strip().upper() in first_dialect_pointer: + normalized_dialect = dialect.strip().upper() + self.emit( + "ossie.semantic.expression.dialect_duplicate", + ( + f"Duplicate expression dialect {dialect!r}; first declared at " + f"{first_dialect_pointer[normalized_dialect]}." + ), + dialect_pointer, + scope=scope, + ) + else: + first_dialect_pointer[dialect.strip().upper()] = dialect_pointer + + text = variant.get("expression") + if isinstance(text, str) and not text.strip(): + self.emit( + "ossie.semantic.expression.text_empty", + "Expression text must not be empty.", + _pointer(variant_pointer, "expression"), + scope=scope, + ) + + def validate_semantic_models( + self, + semantic_models: Sequence[object] | None, + *, + parent_pointer: str, + ) -> None: + name_counts = self.duplicate_names( + semantic_models, + parent_pointer=parent_pointer, + namespace="semantic-model", + code="ossie.semantic.semantic_model.name_duplicate", + ) + if semantic_models is None: + return + + for index, value in enumerate(semantic_models): + semantic_model = _mapping(value) + if semantic_model is None: + continue + semantic_model_pointer = _pointer(parent_pointer, index) + semantic_model_name = _name(semantic_model.get("name")) + scope = semantic_model_name or f"semantic_model[{index}]" + if semantic_model_name and name_counts[normalize_identifier(semantic_model_name)] > 1: + scope = f"{semantic_model_name}@{index}" + self.checked_scopes.append(scope) + self.validate_scope(semantic_model, semantic_model_pointer, scope) + + def validate_scope( + self, + semantic_model: JSONObject, + semantic_model_pointer: str, + scope: str, + ) -> None: + datasets_pointer = _pointer(semantic_model_pointer, "datasets") + metrics_pointer = _pointer(semantic_model_pointer, "metrics") + relationships_pointer = _pointer(semantic_model_pointer, "relationships") + datasets = _array(semantic_model.get("datasets")) + metrics = _array(semantic_model.get("metrics")) + relationships = _array(semantic_model.get("relationships")) + + dataset_name_counts = self.duplicate_names( + datasets, + parent_pointer=datasets_pointer, + namespace="dataset", + code="ossie.semantic.dataset.name_duplicate", + scope=scope, + ) + self.duplicate_names( + metrics, + parent_pointer=metrics_pointer, + namespace="metric", + code="ossie.semantic.metric.name_duplicate", + scope=scope, + ) + self.duplicate_names( + relationships, + parent_pointer=relationships_pointer, + namespace="relationship", + code="ossie.semantic.relationship.name_duplicate", + scope=scope, + ) + + dataset_by_name: dict[str, tuple[JSONObject, str]] = {} + if datasets is not None: + for dataset_index, value in enumerate(datasets): + dataset = _mapping(value) + if dataset is None: + continue + dataset_pointer = _pointer(datasets_pointer, dataset_index) + dataset_name = _name(dataset.get("name")) + fields_pointer = _pointer(dataset_pointer, "fields") + fields = _array(dataset.get("fields")) + self.duplicate_names( + fields, + parent_pointer=fields_pointer, + namespace="field", + code="ossie.semantic.field.name_duplicate", + scope=scope, + ) + if fields is not None: + for field_index, field_value in enumerate(fields): + field = _mapping(field_value) + if field is not None and "expression" in field: + self.validate_expression( + field.get("expression"), + expression_pointer=_pointer(fields_pointer, field_index, "expression"), + scope=scope, + ) + if dataset_name is not None: + self.validate_declared_keys( + dataset, + dataset_pointer=dataset_pointer, + scope=scope, + ) + if ( + dataset_name is not None + and identifier_within_limit(dataset_name) + and dataset_name_counts[normalize_identifier(dataset_name)] == 1 + ): + dataset_by_name[normalize_identifier(dataset_name)] = (dataset, dataset_pointer) + + if metrics is not None: + for metric_index, metric_value in enumerate(metrics): + metric = _mapping(metric_value) + if metric is not None and "expression" in metric: + self.validate_expression( + metric.get("expression"), + expression_pointer=_pointer(metrics_pointer, metric_index, "expression"), + scope=scope, + ) + + if relationships is not None: + for relationship_index, relationship_value in enumerate(relationships): + relationship = _mapping(relationship_value) + if relationship is not None: + self.validate_relationship( + relationship, + relationship_pointer=_pointer(relationships_pointer, relationship_index), + dataset_by_name=dataset_by_name, + scope=scope, + ) + + def validate_relationship( + self, + relationship: JSONObject, + *, + relationship_pointer: str, + dataset_by_name: Mapping[str, tuple[JSONObject, str]], + scope: str, + ) -> None: + from_name = _name(relationship.get("from")) + to_name = _name(relationship.get("to")) + from_valid = ( + self.validate_identifier_length(from_name, _pointer(relationship_pointer, "from"), scope=scope) + if from_name is not None + else False + ) + to_valid = ( + self.validate_identifier_length(to_name, _pointer(relationship_pointer, "to"), scope=scope) + if to_name is not None + else False + ) + from_dataset = dataset_by_name.get(normalize_identifier(from_name)) if from_name and from_valid else None + to_dataset = dataset_by_name.get(normalize_identifier(to_name)) if to_name and to_valid else None + + if from_name is not None and from_dataset is None: + self.emit( + "ossie.semantic.relationship.from_dataset_unknown", + f"Relationship source dataset {from_name!r} does not exist uniquely within scope {scope!r}.", + _pointer(relationship_pointer, "from"), + scope=scope, + ) + if to_name is not None and to_dataset is None: + self.emit( + "ossie.semantic.relationship.to_dataset_unknown", + f"Relationship target dataset {to_name!r} does not exist uniquely within scope {scope!r}.", + _pointer(relationship_pointer, "to"), + scope=scope, + ) + + has_from_columns = "from_columns" in relationship + has_to_columns = "to_columns" in relationship + if not has_from_columns or not has_to_columns: + missing = [ + name + for name, present in ( + ("from_columns", has_from_columns), + ("to_columns", has_to_columns), + ) + if not present + ] + self.emit( + "ossie.semantic.relationship.keys_incomplete", + f"Executable relationships require both key arrays; missing {', '.join(missing)}.", + relationship_pointer, + scope=scope, + ) + return + + from_columns = _array(relationship.get("from_columns")) + to_columns = _array(relationship.get("to_columns")) + if from_columns is None or to_columns is None: + return + + empty_keys = False + if not from_columns: + empty_keys = True + self.emit( + "ossie.semantic.relationship.keys_empty", + "Relationship source keys must not be empty.", + _pointer(relationship_pointer, "from_columns"), + scope=scope, + ) + if not to_columns: + empty_keys = True + self.emit( + "ossie.semantic.relationship.keys_empty", + "Relationship target keys must not be empty.", + _pointer(relationship_pointer, "to_columns"), + scope=scope, + ) + if empty_keys: + return + + if len(from_columns) != len(to_columns): + self.emit( + "ossie.semantic.relationship.key_arity_mismatch", + ( + "Relationship source and target key arrays must have equal length; " + f"received {len(from_columns)} and {len(to_columns)}." + ), + relationship_pointer, + scope=scope, + ) + + self.validate_relationship_fields( + from_columns, + dataset=from_dataset, + columns_pointer=_pointer(relationship_pointer, "from_columns"), + side="from", + scope=scope, + ) + to_key = self.validate_relationship_fields( + to_columns, + dataset=to_dataset, + columns_pointer=_pointer(relationship_pointer, "to_columns"), + side="to", + scope=scope, + ) + + if to_dataset is not None and to_key is not None: + target, _ = to_dataset + declared_keys = self.declared_unique_keys(target) + if to_key not in declared_keys: + self.emit( + "ossie.semantic.relationship.target_key_not_unique", + ( + f"Relationship target key {list(to_key)!r} is not the target dataset's " + "declared primary key or one of its declared unique keys." + ), + _pointer(relationship_pointer, "to_columns"), + scope=scope, + ) + + def validate_relationship_fields( + self, + columns: Sequence[object], + *, + dataset: tuple[JSONObject, str] | None, + columns_pointer: str, + side: Literal["from", "to"], + scope: str, + ) -> tuple[str, ...] | None: + if dataset is None or not all(isinstance(column, str) and column for column in columns): + return None + dataset_object, _ = dataset + fields = _array(dataset_object.get("fields")) + field_names = { + normalize_identifier(field_name) + for value in fields or () + if (field := _mapping(value)) is not None + if (field_name := _name(field.get("name"))) is not None + if identifier_within_limit(field_name) + } + + valid = True + for index, column in enumerate(columns): + column_pointer = _pointer(columns_pointer, index) + if not self.validate_identifier_length(column, column_pointer, scope=scope): + valid = False + continue + if normalize_identifier(column) not in field_names: + valid = False + self.emit( + f"ossie.semantic.relationship.{side}_key_field_unknown", + f"Relationship {side} key field {column!r} does not exist in dataset {dataset_object.get('name')!r}.", + column_pointer, + scope=scope, + ) + return tuple(normalize_identifier(column) for column in columns) if valid else None + + @staticmethod + def declared_unique_keys(dataset: JSONObject) -> set[tuple[str, ...]]: + keys: set[tuple[str, ...]] = set() + primary_key = _array(dataset.get("primary_key")) + if primary_key and all( + isinstance(column, str) and column and identifier_within_limit(column) for column in primary_key + ): + keys.add(tuple(normalize_identifier(column) for column in primary_key)) + unique_keys = _array(dataset.get("unique_keys")) + for value in unique_keys or (): + key = _array(value) + if key and all(isinstance(column, str) and column and identifier_within_limit(column) for column in key): + keys.add(tuple(normalize_identifier(column) for column in key)) + return keys + + def validate_declared_keys( + self, + dataset: JSONObject, + *, + dataset_pointer: str, + scope: str, + ) -> None: + fields = _array(dataset.get("fields")) + field_names = { + normalize_identifier(field_name) + for value in fields or () + if (field := _mapping(value)) is not None + if (field_name := _name(field.get("name"))) is not None + if identifier_within_limit(field_name) + } + key_groups: list[tuple[str, Sequence[object]]] = [] + primary_key = _array(dataset.get("primary_key")) + if primary_key is not None: + key_groups.append(("primary_key", primary_key)) + unique_keys = _array(dataset.get("unique_keys")) + for key_index, value in enumerate(unique_keys or ()): + key = _array(value) + if key is not None: + key_groups.append((f"unique_keys/{key_index}", key)) + + first_key_pointer: dict[tuple[str, ...], str] = {} + for key_path, columns in key_groups: + key_pointer = _pointer(dataset_pointer, *key_path.split("/")) + if not columns: + self.emit( + "ossie.semantic.dataset.key_empty", + "Declared primary and unique keys must contain at least one field.", + key_pointer, + scope=scope, + ) + continue + normalized_columns: list[str] = [] + seen_columns: set[str] = set() + valid = True + for column_index, column in enumerate(columns): + if not isinstance(column, str) or not column: + valid = False + continue + column_pointer = _pointer(dataset_pointer, *key_path.split("/"), column_index) + if not self.validate_identifier_length(column, column_pointer, scope=scope): + valid = False + continue + normalized = normalize_identifier(column) + normalized_columns.append(normalized) + if normalized not in field_names: + valid = False + self.emit( + "ossie.semantic.dataset.key_field_unknown", + f"Declared key field {column!r} does not exist uniquely in dataset {dataset.get('name')!r}.", + column_pointer, + scope=scope, + ) + if normalized in seen_columns: + valid = False + self.emit( + "ossie.semantic.dataset.key_column_duplicate", + f"Declared key repeats field {column!r} after Ossie identifier normalization.", + column_pointer, + scope=scope, + ) + seen_columns.add(normalized) + normalized_key = tuple(normalized_columns) + is_unique_key = key_path.startswith("unique_keys/") + if valid and is_unique_key and normalized_key in first_key_pointer: + self.emit( + "ossie.semantic.dataset.key_duplicate", + f"Declared key duplicates the key first declared at {first_key_pointer[normalized_key]}.", + key_pointer, + scope=scope, + ) + elif valid and is_unique_key: + first_key_pointer[normalized_key] = key_pointer + + def validate_ontology(self, document: JSONObject) -> None: + ontology = _array(document.get("ontology")) + concept_names = { + concept + for value in ontology or () + if (component := _mapping(value)) is not None + if (concept := _name(component.get("concept"))) is not None + } + ontology_mappings = _array(document.get("ontology_mappings")) + embedded_models: list[object] = [] + embedded_model_pointers: list[str] = [] + + for mapping_index, value in enumerate(ontology_mappings or ()): + ontology_mapping = _mapping(value) + if ontology_mapping is None: + continue + mapping_pointer = _pointer("/ontology_mappings", mapping_index) + if "semantic_model" in ontology_mapping: + embedded_models.append(ontology_mapping.get("semantic_model")) + embedded_model_pointers.append(_pointer(mapping_pointer, "semantic_model")) + + concept_mappings = _array(ontology_mapping.get("concept_mappings")) + for concept_mapping_index, concept_value in enumerate(concept_mappings or ()): + concept_mapping = _mapping(concept_value) + if concept_mapping is None: + continue + concept_mapping_pointer = _pointer(mapping_pointer, "concept_mappings", concept_mapping_index) + self.validate_concept_reference( + concept_mapping.get("concept"), + pointer=_pointer(concept_mapping_pointer, "concept"), + concept_names=concept_names, + ) + object_mappings = _array(concept_mapping.get("object_mappings")) + for object_index, object_value in enumerate(object_mappings or ()): + self.validate_object_mapping( + object_value, + pointer=_pointer(concept_mapping_pointer, "object_mappings", object_index), + concept_names=concept_names, + ) + link_mappings = _array(concept_mapping.get("link_mappings")) + for link_index, link_value in enumerate(link_mappings or ()): + self.validate_link_mapping( + link_value, + pointer=_pointer(concept_mapping_pointer, "link_mappings", link_index), + concept_names=concept_names, + ) + + # Ontology maps embed complete logical SemanticModel objects. Validate + # each as an isolated scope while keeping ontology itself preservation-only. + self.validate_embedded_semantic_models(embedded_models, embedded_model_pointers) + + def validate_concept_reference( + self, + concept: object, + *, + pointer: str, + concept_names: set[str], + ) -> None: + if isinstance(concept, str) and concept and concept not in concept_names: + self.emit( + "ossie.semantic.ontology.concept_unknown", + f"Ontology mapping references unknown concept {concept!r}.", + pointer, + ) + + def validate_object_mapping( + self, + value: object, + *, + pointer: str, + concept_names: set[str], + ) -> None: + object_mapping = _mapping(value) + if object_mapping is None: + return + if "concept" in object_mapping: + self.validate_concept_reference( + object_mapping.get("concept"), + pointer=_pointer(pointer, "concept"), + concept_names=concept_names, + ) + + def validate_link_mapping( + self, + value: object, + *, + pointer: str, + concept_names: set[str], + ) -> None: + link_mapping = _mapping(value) + if link_mapping is None: + return + if "object_mapping" in link_mapping: + self.validate_object_mapping( + link_mapping.get("object_mapping"), + pointer=_pointer(pointer, "object_mapping"), + concept_names=concept_names, + ) + children = _array(link_mapping.get("children")) + for child_index, child in enumerate(children or ()): + self.validate_link_mapping( + child, + pointer=_pointer(pointer, "children", child_index), + concept_names=concept_names, + ) + + def validate_embedded_semantic_models( + self, + semantic_models: Sequence[object], + pointers: Sequence[str], + ) -> None: + name_counts: Counter[str] = Counter() + first_pointer: dict[str, str] = {} + for semantic_model, pointer in zip(semantic_models, pointers, strict=True): + model = _mapping(semantic_model) + if model is None: + continue + model_name = _name(model.get("name")) + if model_name is None: + continue + name_pointer = _pointer(pointer, "name") + self.validate_identifier_length(model_name, name_pointer) + normalized_name = normalize_identifier(model_name) + name_counts[normalized_name] += 1 + if normalized_name in first_pointer: + self.emit( + "ossie.semantic.semantic_model.name_duplicate", + ( + f"Duplicate semantic-model name {model_name!r} after Ossie identifier normalization; " + f"first declared at {first_pointer[normalized_name]}." + ), + name_pointer, + ) + else: + first_pointer[normalized_name] = name_pointer + + for index, (semantic_model, pointer) in enumerate(zip(semantic_models, pointers, strict=True)): + model = _mapping(semantic_model) + if model is None: + continue + model_name = _name(model.get("name")) + scope = model_name or f"ontology_mapping[{index}]" + if model_name and name_counts[normalize_identifier(model_name)] > 1: + scope = f"{model_name}@{index}" + self.checked_scopes.append(scope) + self.validate_scope(model, pointer, scope) + + +def validate_ossie_semantics( + document: Mapping[str, object] | OssieLogicalDocument | OssieOntologyDocument, + *, + profile: OssieProfile | None = None, +) -> SemanticValidationResult: + """Validate scope-local Apache Ossie semantics after schema validation. + + The function intentionally does not run JSON Schema validation. It accepts + schema-invalid mappings so permissive pipelines can collect independent + semantic diagnostics, but skips checks whose prerequisite shape is absent. + """ + + source: OssieSourceLocation | None = None + if isinstance(document, OssieLogicalDocument): + canonical_data = document.canonical_data + document_kind: SemanticDocumentKind = "logical" + if document.source and document.source.identifier: + source = OssieSourceLocation(identifier=document.source.identifier) + elif isinstance(document, OssieOntologyDocument): + canonical_data = document.canonical_data + document_kind = "ontology" + if document.source and document.source.identifier: + source = OssieSourceLocation(identifier=document.source.identifier) + elif isinstance(document, Mapping): + canonical_data = document + has_logical_root = "semantic_model" in canonical_data + has_ontology_root = "ontology" in canonical_data or "ontology_mappings" in canonical_data + if has_logical_root and not has_ontology_root: + document_kind = "logical" + elif has_ontology_root and not has_logical_root: + document_kind = "ontology" + else: + document_kind = "unsupported" + else: + raise TypeError("Semantic validation requires a parsed mapping or a supported Ossie document") + + validator = _SemanticValidator(profile=_profile_for(canonical_data, profile), source=source) + if document_kind == "logical": + validator.validate_semantic_models( + _array(canonical_data.get("semantic_model")), + parent_pointer="/semantic_model", + ) + elif document_kind == "ontology": + validator.validate_ontology(canonical_data) + + diagnostics = sort_diagnostics(validator.diagnostics) + valid = not any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in diagnostics) + return SemanticValidationResult( + valid=valid, + document_kind=document_kind, + checked_scopes=tuple(validator.checked_scopes), + diagnostics=diagnostics, + failure_stage=None if valid else "semantic", + ) diff --git a/sidemantic/interchange/ossie/serialization.py b/sidemantic/interchange/ossie/serialization.py new file mode 100644 index 00000000..0e896c8e --- /dev/null +++ b/sidemantic/interchange/ossie/serialization.py @@ -0,0 +1,258 @@ +"""Validated source serialization for Apache Ossie documents. + +Canonical serialization is deterministic, but it is not a lexical YAML +round-trip: comments, anchors, aliases, quoting, scalar styles, and whitespace +are not preserved. Exact retained bytes are returned only through the explicit +exact-source mode and only after they parse to the document's canonical data. +""" + +from __future__ import annotations + +import importlib +import json +from dataclasses import dataclass + +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSourceLocation, + sort_diagnostics, +) +from sidemantic.interchange.ossie.documents import ( + OssieDocument, + OssieLogicalDocument, + OssieOntologyDocument, + UnsupportedOssieDocument, +) +from sidemantic.interchange.ossie.profiles import OssieConsumerProfile, OssieProfile, OssieSerialization + + +@dataclass(frozen=True, slots=True) +class OssieSerializationResult: + """Immutable serialized bytes and diagnostics produced without file I/O.""" + + data: bytes + serialization: OssieSerialization + exact_source_reused: bool + diagnostics: tuple[OssieDiagnostic, ...] = () + + def __post_init__(self) -> None: + if not isinstance(self.data, bytes): + raise TypeError("serialized data must be bytes") + object.__setattr__(self, "serialization", OssieSerialization(self.serialization)) + object.__setattr__(self, "diagnostics", sort_diagnostics(self.diagnostics)) + + +class OssieSerializationError(ValueError): + """Serialization refusal with stable, structured diagnostics.""" + + def __init__(self, diagnostics: tuple[OssieDiagnostic, ...]) -> None: + self.diagnostics = sort_diagnostics(diagnostics) + message = "; ".join(diagnostic.message for diagnostic in self.diagnostics) + super().__init__(message or "Apache Ossie serialization failed") + + +def _source_location(document: OssieDocument) -> OssieSourceLocation | None: + source = document.source + if source is None or source.identifier is None: + return None + return OssieSourceLocation(identifier=source.identifier) + + +def _diagnostic( + document: OssieDocument, + *, + code: str, + message: str, + severity: OssieDiagnosticSeverity = OssieDiagnosticSeverity.ERROR, +) -> OssieDiagnostic: + return OssieDiagnostic( + code=code, + severity=severity, + message=message, + source=_source_location(document), + ) + + +def _schema_profile_name(document: OssieDocument) -> str | None: + if document.version is None: + return None + if isinstance(document, OssieLogicalDocument): + return f"logical-{document.version}" + if isinstance(document, OssieOntologyDocument): + return f"ontology-{document.version}" + return None + + +def _validate_canonical_data( + document: OssieDocument, + canonical_data: object, + *, + profile: OssieProfile | None, + consumer_profile: OssieConsumerProfile | str | None, +) -> tuple[OssieDiagnostic, ...]: + if isinstance(document, UnsupportedOssieDocument): + reason = f": {document.reason}" if document.reason else "" + raise OssieSerializationError( + ( + _diagnostic( + document, + code="ossie.serialization.unsupported_document", + message=f"Unsupported Apache Ossie document cannot be serialized{reason}", + ), + ) + ) + + profile_name = _schema_profile_name(document) + if profile_name is None: + raise OssieSerializationError( + ( + _diagnostic( + document, + code="ossie.serialization.version_missing", + message="Apache Ossie document must declare a version before serialization", + ), + ) + ) + + # Validation owns the pinned offline schema bundle and loads jsonschema only + # when this operation is requested. + from sidemantic.interchange.ossie.validation import validate_ossie_schema + + validation_profile = profile + if validation_profile is None and document.version != "0.1.0": + validation_profile = profile_name + validation = validate_ossie_schema( + canonical_data, + profile=validation_profile, + consumer_profile=consumer_profile, + ) + if not validation.valid: + raise OssieSerializationError(validation.diagnostics) + return validation.diagnostics + + +def _exact_source_matches( + document: OssieDocument, + source_bytes: bytes, + consumer_profile: OssieConsumerProfile | str | None, +) -> bool: + # Reuse the source parser's duplicate-key and JSON-compatibility checks. The + # import remains off the canonical JSON/YAML serialization path. + from sidemantic.interchange.ossie.parser import OssieParseOptions, parse_ossie_document + + parsed = parse_ossie_document( + source_bytes, + source_identifier=document.source.identifier or "", + options=OssieParseOptions( + serialization=document.serialization, + consumer_profile=consumer_profile or OssieConsumerProfile.OSSIE_CORE, + ), + ) + return parsed.valid and parsed.document.to_parsed_data() == document.to_parsed_data() + + +def _canonical_json(canonical_data: object) -> bytes: + text = json.dumps( + canonical_data, + allow_nan=False, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + return f"{text}\n".encode() + + +def _canonical_yaml(document: OssieDocument, canonical_data: object) -> bytes: + try: + yaml = importlib.import_module("yaml") + except ImportError as exc: + raise OssieSerializationError( + ( + _diagnostic( + document, + code="ossie.serialization.yaml_unavailable", + message="Canonical YAML serialization requires PyYAML", + ), + ) + ) from exc + + text = yaml.safe_dump( + canonical_data, + allow_unicode=True, + default_flow_style=False, + sort_keys=True, + ) + return f"{text.rstrip(chr(10))}\n".encode() + + +def serialize_ossie_document( + document: OssieDocument, + serialization: OssieSerialization | str, + *, + exact_source: bool = False, + profile: OssieProfile | None = None, + consumer_profile: OssieConsumerProfile | str | None = None, +) -> OssieSerializationResult: + """Serialize a document without selecting or changing its Ossie profile. + + The document's existing family and version identify the pinned offline + schema used for validation. ``serialization`` selects only YAML versus JSON; + it never selects, infers, or rewrites a schema version. The dbt 1.12 0.1.0 + compatibility alias requires an explicit ``profile`` or + ``consumer_profile`` context. + """ + + if not isinstance(document, (OssieLogicalDocument, OssieOntologyDocument, UnsupportedOssieDocument)): + raise TypeError("document must be an OssieDocument") + if not isinstance(exact_source, bool): + raise TypeError("exact_source must be a boolean") + + output_serialization = OssieSerialization(serialization) + if profile is not None and not isinstance(profile, OssieProfile): + raise TypeError("profile must be an OssieProfile") + canonical_data = document.to_parsed_data() + diagnostics = list( + _validate_canonical_data( + document, + canonical_data, + profile=profile, + consumer_profile=consumer_profile, + ) + ) + + source = document.source + if ( + exact_source + and output_serialization is document.serialization + and source is not None + and source.original_bytes is not None + ): + exact_source_consumer = profile.consumer_profile if profile is not None else consumer_profile + if _exact_source_matches(document, source.original_bytes, exact_source_consumer): + return OssieSerializationResult( + data=source.original_bytes, + serialization=output_serialization, + exact_source_reused=True, + diagnostics=tuple(diagnostics), + ) + diagnostics.append( + _diagnostic( + document, + code="ossie.serialization.exact_source_mismatch", + message="Retained source bytes no longer match canonical data; canonical serialization was used", + severity=OssieDiagnosticSeverity.WARNING, + ) + ) + + if output_serialization is OssieSerialization.JSON: + data = _canonical_json(canonical_data) + else: + data = _canonical_yaml(document, canonical_data) + + return OssieSerializationResult( + data=data, + serialization=output_serialization, + exact_source_reused=False, + diagnostics=tuple(diagnostics), + ) diff --git a/sidemantic/interchange/ossie/synthesis.py b/sidemantic/interchange/ossie/synthesis.py new file mode 100644 index 00000000..08789ecd --- /dev/null +++ b/sidemantic/interchange/ossie/synthesis.py @@ -0,0 +1,398 @@ +"""Fail-closed synthesis of Apache Ossie documents from runtime graphs. + +This path is intentionally distinct from source-document serialization. A +``SemanticGraph`` is a lowered runtime projection and cannot reproduce lexical +source form, alternate dialect expressions, ontology documents, or every Ossie +field. Synthesis therefore requires the caller to name the output scope and the +actual dialect of every emitted expression, and refuses constructs that would +need invented semantic meaning. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sidemantic.core.metric import Metric +from sidemantic.core.model import Model +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + sort_diagnostics, +) +from sidemantic.interchange.ossie.documents import OssieLogicalDocument +from sidemantic.interchange.ossie.expression_validation import scalar_sql_expression_error +from sidemantic.interchange.ossie.profiles import ( + OssieConsumerProfile, + OssieProfileError, + OssieSerialization, + resolve_ossie_profile, +) +from sidemantic.interchange.ossie.semantic_validation import validate_ossie_semantics +from sidemantic.interchange.ossie.validation import validate_ossie_schema + +_SQL_DIALECTS = frozenset({"ANSI_SQL", "BIGQUERY", "DATABRICKS", "SNOWFLAKE"}) +_SQLGLOT_DIALECTS = {"ANSI_SQL": None, "BIGQUERY": "bigquery", "DATABRICKS": "databricks", "SNOWFLAKE": "snowflake"} +_DATA_TYPES = frozenset( + {"String", "Integer", "Decimal", "Float", "Boolean", "Date", "Time", "DateTime", "DateTimeTz", "Opaque"} +) + + +@dataclass(frozen=True, slots=True) +class OssieSynthesisResult: + """A schema-valid synthesized document, or diagnostics explaining refusal.""" + + document: OssieLogicalDocument | None + diagnostics: tuple[OssieDiagnostic, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "diagnostics", sort_diagnostics(self.diagnostics)) + + @property + def valid(self) -> bool: + return self.document is not None and not any( + diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in self.diagnostics + ) + + +class OssieSynthesisError(ValueError): + """Graph synthesis refusal carrying stable structured diagnostics.""" + + def __init__(self, diagnostics: tuple[OssieDiagnostic, ...]) -> None: + self.diagnostics = sort_diagnostics(diagnostics) + super().__init__("; ".join(diagnostic.message for diagnostic in self.diagnostics)) + + +def _error(code: str, message: str, pointer: str = "") -> OssieDiagnostic: + return OssieDiagnostic( + severity=OssieDiagnosticSeverity.ERROR, + code=code, + message=message, + json_pointer=pointer, + ) + + +def _warning(code: str, message: str, pointer: str = "") -> OssieDiagnostic: + return OssieDiagnostic( + severity=OssieDiagnosticSeverity.WARNING, + code=code, + message=message, + json_pointer=pointer, + ) + + +def _expression(text: str, dialect: str) -> dict[str, object]: + return {"dialects": [{"dialect": dialect, "expression": text}]} + + +def _scalar_expression_error(text: str, dialect: str) -> str | None: + return scalar_sql_expression_error(text, sqlglot_dialect=_SQLGLOT_DIALECTS[dialect]) + + +def _metric_expression(metric: Metric, model_name: str | None) -> str | None: + if metric.sql_is_complete and metric.sql: + return metric.sql + if metric.type == "ratio": + if not metric.numerator or not metric.denominator: + return None + return f"{metric.numerator} / NULLIF({metric.denominator}, 0)" + if metric.type == "derived": + return metric.sql + if metric.type is not None: + return None + if metric.agg: + inner = metric.sql or "*" + if model_name and inner != "*" and "." not in inner: + inner = f"{model_name}.{inner}" + if metric.agg == "count_distinct": + return f"COUNT(DISTINCT {inner})" + aggregate = {"variance_pop": "VAR_POP"}.get(metric.agg, metric.agg.upper()) + return f"{aggregate}({inner})" + return metric.sql + + +def _dataset(model: Model, dialect: str, index: int, diagnostics: list[OssieDiagnostic]) -> dict[str, object] | None: + pointer = f"/semantic_model/0/datasets/{index}" + if model.table and model.sql: + diagnostics.append( + _error( + "ossie.synthesis.source_ambiguous", + f"Model {model.name!r} has both table and SQL sources; choose one before Ossie synthesis.", + f"{pointer}/source", + ) + ) + return None + source = model.sql or model.table + if not source: + diagnostics.append( + _error( + "ossie.synthesis.source_missing", + f"Model {model.name!r} has no source; Ossie requires one and Sidemantic will not invent it.", + f"{pointer}/source", + ) + ) + return None + if model.has_untranslated_dax: + diagnostics.append( + _error( + "ossie.synthesis.source_language_unsupported", + f"Model {model.name!r} contains untranslated DAX and cannot be emitted as SQL.", + f"{pointer}/source", + ) + ) + return None + + dataset: dict[str, object] = {"name": model.name, "source": source} + if model.primary_key is not None: + dataset["primary_key"] = model.primary_key_columns + if model.unique_keys is not None: + dataset["unique_keys"] = model.unique_keys + if model.description is not None: + dataset["description"] = model.description + + fields: list[dict[str, object]] = [] + for field_index, dimension in enumerate(model.dimensions): + field_pointer = f"{pointer}/fields/{field_index}" + if dimension.has_untranslated_dax: + diagnostics.append( + _error( + "ossie.synthesis.expression_language_unsupported", + f"Field {model.name}.{dimension.name} contains untranslated DAX.", + f"{field_pointer}/expression", + ) + ) + continue + expression_error = _scalar_expression_error(dimension.sql_expr, dialect) + if expression_error is not None: + diagnostics.append( + _error( + "ossie.synthesis.expression_invalid", + f"Field {model.name}.{dimension.name} is not one {dialect} SQL expression: {expression_error}", + f"{field_pointer}/expression", + ) + ) + continue + field: dict[str, object] = { + "name": dimension.name, + "expression": _expression(dimension.sql_expr, dialect), + } + if dimension.logical_data_type is not None: + if dimension.logical_data_type not in _DATA_TYPES: + diagnostics.append( + _error( + "ossie.synthesis.datatype_unsupported", + f"Field {model.name}.{dimension.name} has unsupported logical datatype {dimension.logical_data_type!r}.", + f"{field_pointer}/datatype", + ) + ) + else: + field["datatype"] = dimension.logical_data_type + if dimension.declared_is_time is not None: + field["dimension"] = {"is_time": dimension.declared_is_time} + elif dimension.type == "time" and dimension.logical_data_type not in {"Date", "Time", "DateTime", "DateTimeTz"}: + # Preserve runtime time-role semantics when datatype omission would + # otherwise make Ossie default this field to non-time. + field["dimension"] = {"is_time": True} + if dimension.description is not None: + field["description"] = dimension.description + if dimension.label is not None: + field["label"] = dimension.label + fields.append(field) + if fields: + dataset["fields"] = fields + return dataset + + +def _relationships(models: dict[str, Model], diagnostics: list[OssieDiagnostic]) -> list[dict[str, object]]: + result: list[dict[str, object]] = [] + used_names: set[str] = set() + relationship_index = 0 + for from_model in models.values(): + for relationship in from_model.relationships: + pointer = f"/semantic_model/0/relationships/{relationship_index}" + relationship_index += 1 + if relationship.type != "many_to_one": + diagnostics.append( + _error( + "ossie.synthesis.relationship_cardinality_unsupported", + f"Relationship from {from_model.name!r} to {relationship.name!r} is {relationship.type!r}; Ossie core requires from-many/to-one.", + pointer, + ) + ) + continue + edge_id = relationship.edge_id or (relationship.metadata or {}).get("osi_name") + if not isinstance(edge_id, str) or not edge_id: + diagnostics.append( + _error( + "ossie.synthesis.relationship_identity_missing", + f"Relationship from {from_model.name!r} to {relationship.name!r} has no declared edge identity.", + f"{pointer}/name", + ) + ) + continue + if edge_id in used_names: + diagnostics.append( + _error( + "ossie.synthesis.relationship_identity_duplicate", + f"Relationship identity {edge_id!r} is duplicated.", + f"{pointer}/name", + ) + ) + continue + used_names.add(edge_id) + target = models.get(relationship.name) + from_columns = relationship.foreign_key_columns + to_columns = relationship.primary_key_columns + if target is None or not from_columns or not to_columns or len(from_columns) != len(to_columns): + diagnostics.append( + _error( + "ossie.synthesis.relationship_keys_unusable", + f"Relationship {edge_id!r} lacks a valid target or equal non-empty key lists; keys will not be invented.", + pointer, + ) + ) + continue + target_keys = {tuple(target.primary_key_columns)} if target.primary_key_columns else set() + target_keys.update(tuple(key) for key in target.unique_keys or ()) + if tuple(to_columns) not in target_keys: + diagnostics.append( + _error( + "ossie.synthesis.relationship_target_not_unique", + f"Relationship {edge_id!r} targets columns that are not declared primary or unique.", + f"{pointer}/to_columns", + ) + ) + continue + result.append( + { + "name": edge_id, + "from": from_model.name, + "to": relationship.name, + "from_columns": from_columns, + "to_columns": to_columns, + } + ) + return result + + +def _metrics( + graph: SemanticGraph, + models: dict[str, Model], + dialect: str, + diagnostics: list[OssieDiagnostic], +) -> list[dict[str, object]]: + candidates: list[tuple[Metric, str | None]] = [ + (metric, graph.metric_owners.get(name)) for name, metric in graph.metrics.items() + ] + candidates.extend((metric, model.name) for model in models.values() for metric in model.metrics) + result: list[dict[str, object]] = [] + seen: dict[str, str] = {} + for metric, owner in candidates: + expression = _metric_expression(metric, owner) + if expression is None: + diagnostics.append( + _error( + "ossie.synthesis.metric_unrepresentable", + f"Metric {metric.name!r} cannot be represented as one Ossie SQL expression.", + ) + ) + continue + expression_error = _scalar_expression_error(expression, dialect) + if expression_error is not None: + diagnostics.append( + _error( + "ossie.synthesis.expression_invalid", + f"Metric {metric.name!r} is not one {dialect} SQL expression: {expression_error}", + ) + ) + continue + previous = seen.get(metric.name) + if previous is not None: + if previous != expression: + diagnostics.append( + _error( + "ossie.synthesis.metric_name_collision", + f"Metric {metric.name!r} has multiple distinct runtime definitions.", + ) + ) + continue + seen[metric.name] = expression + value: dict[str, object] = {"name": metric.name, "expression": _expression(expression, dialect)} + if metric.logical_data_type is not None: + if metric.logical_data_type not in _DATA_TYPES: + diagnostics.append( + _error( + "ossie.synthesis.datatype_unsupported", + f"Metric {metric.name!r} has unsupported logical datatype {metric.logical_data_type!r}.", + ) + ) + else: + value["datatype"] = metric.logical_data_type + if metric.description is not None: + value["description"] = metric.description + result.append(value) + return result + + +def synthesize_ossie_document( + graph: SemanticGraph, + *, + scope_name: str, + expression_dialect: str, + schema_version: str = "0.2.0.dev0", + serialization: OssieSerialization | str = OssieSerialization.YAML, + consumer_profile: OssieConsumerProfile | str = OssieConsumerProfile.OSSIE_CORE, +) -> OssieSynthesisResult: + """Synthesize one logical document without guessing scope or dialect.""" + + if not isinstance(graph, SemanticGraph): + raise TypeError("graph must be a SemanticGraph") + if not isinstance(scope_name, str) or not scope_name.strip(): + raise ValueError("scope_name must be an explicit non-empty string") + if not isinstance(expression_dialect, str): + raise TypeError("expression_dialect must be a string") + dialect = expression_dialect.strip().upper() + if dialect not in _SQL_DIALECTS: + supported = ", ".join(sorted(_SQL_DIALECTS)) + raise ValueError(f"expression_dialect must identify emitted SQL exactly; supported: {supported}") + + diagnostics: list[OssieDiagnostic] = [] + try: + profile = resolve_ossie_profile(schema_version, consumer_profile) + except OssieProfileError as exc: + return OssieSynthesisResult( + document=None, + diagnostics=(_error("ossie.synthesis.profile_unsupported", str(exc), "/version"),), + ) + models = dict(graph.models) + datasets = [ + dataset + for index, model in enumerate(models.values()) + if (dataset := _dataset(model, dialect, index, diagnostics)) is not None + ] + semantic_model: dict[str, object] = {"name": scope_name, "datasets": datasets} + relationships = _relationships(models, diagnostics) + metrics = _metrics(graph, models, dialect, diagnostics) + if relationships: + semantic_model["relationships"] = relationships + if metrics: + semantic_model["metrics"] = metrics + data = {"version": schema_version, "semantic_model": [semantic_model]} + + validation = validate_ossie_schema(data, profile=profile, consumer_profile=profile.consumer_profile) + diagnostics.extend(validation.diagnostics) + semantic_validation = validate_ossie_semantics(data, profile=profile) + diagnostics.extend(semantic_validation.diagnostics) + if any(diagnostic.severity is OssieDiagnosticSeverity.ERROR for diagnostic in diagnostics): + return OssieSynthesisResult(document=None, diagnostics=tuple(diagnostics)) + + document = OssieLogicalDocument(canonical_data=data, serialization=serialization) + return OssieSynthesisResult(document=document, diagnostics=tuple(diagnostics)) + + +def require_synthesized_document(result: OssieSynthesisResult) -> OssieLogicalDocument: + """Return a successful document or raise its structured refusal.""" + + if result.document is None: + raise OssieSynthesisError(result.diagnostics) + return result.document diff --git a/sidemantic/interchange/ossie/validation.py b/sidemantic/interchange/ossie/validation.py new file mode 100644 index 00000000..8411e913 --- /dev/null +++ b/sidemantic/interchange/ossie/validation.py @@ -0,0 +1,593 @@ +"""Offline JSON Schema validation for pinned Apache Ossie profiles. + +The optional ``jsonschema`` and ``referencing`` packages are imported only when +validation is requested. Importing Sidemantic or this module remains safe for +the core Pyodide installation. +""" + +from __future__ import annotations + +import hashlib +import importlib +import json +from collections.abc import Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +from functools import cache, lru_cache +from importlib import resources +from typing import Any, Literal + +from sidemantic.interchange.ossie.diagnostics import ( + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSchemaProvenance, + sort_diagnostics, +) +from sidemantic.interchange.ossie.profiles import ( + OssieConsumerProfile, + OssieProfile, + resolve_ossie_profile, +) + +DocumentKind = Literal["logical", "ontology"] +DiagnosticStage = Literal["profile", "availability", "integrity", "schema"] +JsonPathPart = str | int + +_INSTALL_GUIDANCE = ( + "Apache Ossie schema validation requires the optional 'ossie' extra. " + "Install it with `uv add 'sidemantic[ossie]'`, or use " + "`uv sync --extra ossie` in a Sidemantic checkout." +) + + +@dataclass(frozen=True, slots=True) +class SchemaProfile: + """Identity and provenance for one pinned schema profile.""" + + name: str + document_kind: DocumentKind + version: str + schema_dialect: str + resource_uri: str + runtime_path: str + runtime_sha256: str + upstream_path: str + upstream_sha256: str + source_repository: str + source_commit: str + source_path: str + source_url: str + transformations: tuple[Mapping[str, str], ...] + dependencies: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class SchemaValidationResult: + """Result of the schema stage; later semantic stages can compose with it.""" + + valid: bool + profile: str | None + schema_commit: str | None + diagnostics: tuple[OssieDiagnostic, ...] + failure_stage: DiagnosticStage | None = None + stage: Literal["schema"] = "schema" + + def to_dict(self) -> dict[str, Any]: + diagnostics = [] + for diagnostic in self.diagnostics: + schema = diagnostic.schema + diagnostics.append( + { + "code": diagnostic.code, + "severity": diagnostic.severity.value, + "message": diagnostic.message, + "json_pointer": diagnostic.json_pointer, + "profile": diagnostic.profile.identifier if diagnostic.profile else None, + "schema": ( + { + "schema_id": schema.schema_id, + "version": schema.version, + "commit": schema.commit, + "sha256": schema.sha256, + } + if schema + else None + ), + } + ) + return { + "stage": self.stage, + "failure_stage": self.failure_stage, + "valid": self.valid, + "profile": self.profile, + "schema_commit": self.schema_commit, + "diagnostics": diagnostics, + } + + +class SchemaAssetIntegrityError(RuntimeError): + """A vendored schema does not match the checksum in its manifest.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + super().__init__(message) + + +def _schema_root(): + return resources.files("sidemantic").joinpath("interchange", "ossie", "schemas") + + +@lru_cache(maxsize=1) +def _manifest() -> Mapping[str, Any]: + manifest_path = _schema_root().joinpath("manifest.json") + return json.loads(manifest_path.read_text(encoding="utf-8")) + + +def _profile_from_record(name: str, record: Mapping[str, Any]) -> SchemaProfile: + source = record["source"] + return SchemaProfile( + name=name, + document_kind=record["document_kind"], + version=record["version"], + schema_dialect=record["schema_dialect"], + resource_uri=record["resource_uri"], + runtime_path=record["runtime_path"], + runtime_sha256=record["runtime_sha256"], + upstream_path=record["upstream_path"], + upstream_sha256=record["upstream_sha256"], + source_repository=source["repository"], + source_commit=source["commit"], + source_path=source["path"], + source_url=source["url"], + transformations=tuple(record.get("transformations", ())), + dependencies=tuple(record.get("dependencies", ())), + ) + + +def _replace_reference_base(value: Any, old: str, new: str) -> None: + if isinstance(value, dict): + reference = value.get("$ref") + if isinstance(reference, str) and reference.startswith(old): + value["$ref"] = f"{new}{reference[len(old) :]}" + for child in value.values(): + _replace_reference_base(child, old, new) + elif isinstance(value, list): + for child in value: + _replace_reference_base(child, old, new) + + +def _apply_declared_transformations(profile: SchemaProfile, upstream: Any) -> Any: + transformed = deepcopy(upstream) + for index, transformation in enumerate(profile.transformations): + operation = transformation.get("operation") + old = transformation.get("from") + new = transformation.get("to") + reason = transformation.get("reason") + if set(transformation) != {"operation", "from", "to", "reason"} or not all( + isinstance(value, str) and value for value in (operation, old, new, reason) + ): + raise SchemaAssetIntegrityError( + "ossie.schema.provenance_integrity", + f"Invalid transformation declaration {index} for {profile.name}", + ) + if operation == "replace_id": + if not isinstance(transformed, dict) or transformed.get("$id") != old: + raise SchemaAssetIntegrityError( + "ossie.schema.transformation_integrity", + f"Transformation {index} for {profile.name} does not match the upstream $id", + ) + transformed["$id"] = new + elif operation == "replace_reference_base": + _replace_reference_base(transformed, old, new) + else: + raise SchemaAssetIntegrityError( + "ossie.schema.provenance_integrity", + f"Unknown transformation operation {operation!r} for {profile.name}", + ) + return transformed + + +def _read_verified_asset(profile: SchemaProfile, *, upstream: bool) -> tuple[bytes, Any]: + path = profile.upstream_path if upstream else profile.runtime_path + expected_sha256 = profile.upstream_sha256 if upstream else profile.runtime_sha256 + asset_kind = "upstream" if upstream else "runtime" + asset_path = _schema_root().joinpath(*path.split("/")) + asset_bytes = asset_path.read_bytes() + actual_sha256 = hashlib.sha256(asset_bytes).hexdigest() + if actual_sha256 != expected_sha256: + raise SchemaAssetIntegrityError( + f"ossie.schema.{asset_kind}_asset_integrity", + f"Checksum mismatch for {asset_kind} asset {path}: expected {expected_sha256}, got {actual_sha256}", + ) + try: + parsed = json.loads(asset_bytes) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SchemaAssetIntegrityError( + f"ossie.schema.{asset_kind}_asset_integrity", + f"Invalid JSON in {asset_kind} asset {path}: {exc}", + ) from exc + return asset_bytes, parsed + + +def _verify_bundle_integrity(profile: SchemaProfile) -> Mapping[str, Any]: + runtime_bytes, runtime = _read_verified_asset(profile, upstream=False) + upstream_bytes, upstream = _read_verified_asset(profile, upstream=True) + + expected_name = f"{profile.document_kind}-{profile.version}" + repository_prefix = "https://github.com/" + repository_slug = profile.source_repository.removeprefix(repository_prefix).rstrip("/") + expected_source_url = ( + f"https://raw.githubusercontent.com/{repository_slug}/{profile.source_commit}/{profile.source_path.lstrip('/')}" + ) + version_schema = runtime.get("properties", {}).get("version", {}) if isinstance(runtime, dict) else {} + manifest_profiles = _manifest().get("profiles", {}) + if ( + profile.name != expected_name + or not profile.source_repository.startswith(repository_prefix) + or not profile.source_commit + or profile.source_url != expected_source_url + or runtime.get("$schema") != profile.schema_dialect + or not isinstance(runtime.get("$id"), str) + or profile.resource_uri != f"urn:sidemantic:ossie:schema:{profile.document_kind}:{profile.version}" + or version_schema.get("const") != profile.version + or len(profile.dependencies) != len(set(profile.dependencies)) + or any(dependency == profile.name for dependency in profile.dependencies) + or any(dependency not in manifest_profiles for dependency in profile.dependencies) + ): + raise SchemaAssetIntegrityError( + "ossie.schema.provenance_integrity", + f"Manifest provenance is inconsistent for {profile.name}", + ) + + if profile.transformations: + if _apply_declared_transformations(profile, upstream) != runtime: + raise SchemaAssetIntegrityError( + "ossie.schema.transformation_integrity", + f"Runtime asset for {profile.name} does not match its declared transformations", + ) + elif runtime_bytes != upstream_bytes: + raise SchemaAssetIntegrityError( + "ossie.schema.transformation_integrity", + f"Runtime asset for {profile.name} differs from upstream without declared transformations", + ) + return runtime + + +@lru_cache(maxsize=1) +def available_schema_profiles() -> tuple[SchemaProfile, ...]: + """Return every vendored profile in deterministic manifest order.""" + + profiles = _manifest()["profiles"] + return tuple(_profile_from_record(name, record) for name, record in profiles.items()) + + +def get_schema_profile(name: str) -> SchemaProfile: + """Return a pinned profile or raise ``KeyError`` for an unknown name.""" + + for profile in available_schema_profiles(): + if profile.name == name: + return profile + raise KeyError(name) + + +def detect_schema_profile(document: Any) -> SchemaProfile | None: + """Identify an unambiguous upstream profile from document shape and version.""" + + if not isinstance(document, Mapping): + return None + + version = document.get("version") + if version == "0.1.1": + return get_schema_profile("logical-0.1.1") + if version != "0.2.0.dev0": + return None + + has_logical_root = "semantic_model" in document + has_ontology_root = "ontology" in document or "ontology_mappings" in document + if has_logical_root == has_ontology_root: + return None + if has_logical_root: + return get_schema_profile("logical-0.2.0.dev0") + return get_schema_profile("ontology-0.2.0.dev0") + + +def _json_pointer(path: Sequence[JsonPathPart]) -> str: + def escape(part: JsonPathPart) -> str: + return str(part).replace("~", "~0").replace("/", "~1") + + return "" if not path else "/" + "/".join(escape(part) for part in path) + + +@cache +def _load_schema(profile_name: str) -> Mapping[str, Any]: + profile = get_schema_profile(profile_name) + return _verify_bundle_integrity(profile) + + +def _load_validator_runtime() -> tuple[Any, ...]: + """Load optional validator dependencies without top-level imports.""" + + jsonschema = importlib.import_module("jsonschema") + referencing = importlib.import_module("referencing") + referencing_exceptions = importlib.import_module("referencing.exceptions") + return ( + jsonschema.Draft202012Validator, + jsonschema.exceptions.SchemaError, + referencing.Registry, + referencing.Resource, + ( + referencing_exceptions.Unresolvable, + referencing_exceptions.NoSuchResource, + referencing_exceptions.NoSuchAnchor, + referencing_exceptions.PointerToNowhere, + ), + ) + + +def _diagnostic( + *, + code: str, + message: str, + profile: SchemaProfile | None, + ossie_profile: OssieProfile | None = None, + instance_path: Sequence[JsonPathPart] = (), +) -> OssieDiagnostic: + schema: OssieSchemaProvenance | None = None + if profile is not None: + if ossie_profile is None: + ossie_profile = resolve_ossie_profile(profile.version, OssieConsumerProfile.OSSIE_CORE) + schema = OssieSchemaProvenance( + schema_id=profile.resource_uri, + version=profile.version, + commit=profile.source_commit, + sha256=profile.runtime_sha256, + ) + return OssieDiagnostic( + code=code, + severity=OssieDiagnosticSeverity.ERROR, + message=message, + json_pointer=_json_pointer(instance_path), + profile=ossie_profile, + schema=schema, + ) + + +def _result_with_diagnostic( + diagnostic: OssieDiagnostic, + profile: SchemaProfile | None, + failure_stage: DiagnosticStage, +) -> SchemaValidationResult: + return SchemaValidationResult( + valid=False, + profile=profile.name if profile else None, + schema_commit=profile.source_commit if profile else None, + diagnostics=(diagnostic,), + failure_stage=failure_stage, + ) + + +def _validation_code(keyword: str | None) -> str: + normalized = { + "additionalProperties": "additional_properties", + "maxItems": "max_items", + "minItems": "min_items", + "maxLength": "max_length", + "minLength": "min_length", + "oneOf": "one_of", + "anyOf": "any_of", + "allOf": "all_of", + }.get(keyword, keyword) + return f"ossie.schema.{normalized or 'invalid'}" + + +def _build_validator(profile: SchemaProfile, runtime: tuple[Any, ...]): + validator_class, schema_error, registry_class, resource_class, _ = runtime + registry = registry_class() + + for dependency_name in profile.dependencies: + dependency = get_schema_profile(dependency_name) + dependency_schema = _load_schema(dependency.name) + validator_class.check_schema(dependency_schema) + registry = registry.with_resource( + dependency.resource_uri, + resource_class.from_contents(dependency_schema), + ) + + schema = _load_schema(profile.name) + validator_class.check_schema(schema) + try: + return validator_class(schema, registry=registry) + except TypeError as exc: # pragma: no cover - guards unsupported old jsonschema + raise schema_error("Installed jsonschema does not support offline registry validation") from exc + + +def validate_ossie_schema( + document: Any, + *, + profile: str | SchemaProfile | OssieProfile | None = None, + consumer_profile: OssieConsumerProfile | str | None = None, +) -> SchemaValidationResult: + """Validate a parsed Ossie document against one pinned, offline schema. + + Passing ``profile`` is recommended for malformed or migration inputs. Auto + detection intentionally succeeds only when version and root shape identify + exactly one upstream document family. + """ + + resolved_profile: SchemaProfile | None + contract_profile: OssieProfile | None = profile if isinstance(profile, OssieProfile) else None + try: + explicit_consumer = OssieConsumerProfile(consumer_profile) if consumer_profile is not None else None + except ValueError: + diagnostic = _diagnostic( + code="ossie.schema.consumer_profile_unknown", + message=f"Unknown Apache Ossie consumer profile: {consumer_profile}", + profile=None, + ) + return _result_with_diagnostic(diagnostic, None, "profile") + + document_version = document.get("version") if isinstance(document, Mapping) else None + if contract_profile is not None: + if explicit_consumer is not None and explicit_consumer is not contract_profile.consumer_profile: + diagnostic = _diagnostic( + code="ossie.schema.profile_context_mismatch", + message="Explicit consumer profile does not match the supplied Ossie profile", + profile=None, + ossie_profile=contract_profile, + ) + return _result_with_diagnostic(diagnostic, None, "profile") + if document_version != contract_profile.schema_version: + diagnostic = _diagnostic( + code="ossie.schema.profile_context_mismatch", + message=( + f"Document version {document_version!r} does not match explicit profile " + f"{contract_profile.identifier}" + ), + profile=None, + ossie_profile=contract_profile, + instance_path=("version",), + ) + return _result_with_diagnostic(diagnostic, None, "profile") + try: + if contract_profile.is_compatibility_alias: + resolved_profile = get_schema_profile(f"logical-{contract_profile.validation_schema_version}") + else: + resolved_profile = detect_schema_profile(document) or get_schema_profile( + f"logical-{contract_profile.validation_schema_version}" + ) + except KeyError: + resolved_profile = None + elif isinstance(profile, SchemaProfile): + resolved_profile = profile + elif isinstance(profile, str): + try: + resolved_profile = get_schema_profile(profile) + except KeyError: + diagnostic = _diagnostic( + code="ossie.schema.profile_unknown", + message=f"Unknown Apache Ossie schema profile: {profile}", + profile=None, + ) + return _result_with_diagnostic(diagnostic, None, "profile") + else: + resolved_profile = detect_schema_profile(document) + + if contract_profile is None and isinstance(document_version, str): + if document_version == "0.1.0": + if explicit_consumer is not OssieConsumerProfile.DBT_1_12: + code = ( + "ossie.schema.profile_context_required" + if explicit_consumer is None + else "ossie.schema.profile_context_mismatch" + ) + diagnostic = _diagnostic( + code=code, + message=( + "Ossie version 0.1.0 is only a dbt-1.12 compatibility alias; " + "pass that consumer/profile context explicitly" + ), + profile=None, + instance_path=("version",), + ) + return _result_with_diagnostic(diagnostic, None, "profile") + contract_profile = resolve_ossie_profile(document_version, explicit_consumer) + resolved_profile = get_schema_profile("logical-0.1.1") + elif explicit_consumer is not None: + try: + contract_profile = resolve_ossie_profile(document_version, explicit_consumer) + except ValueError as exc: + diagnostic = _diagnostic( + code="ossie.schema.profile_context_mismatch", + message=str(exc), + profile=resolved_profile, + instance_path=("version",), + ) + return _result_with_diagnostic(diagnostic, resolved_profile, "profile") + + if resolved_profile is None: + diagnostic = _diagnostic( + code="ossie.schema.profile_undetected", + message=( + "Document version and root shape do not identify exactly one " + "pinned Apache Ossie schema profile; pass profile explicitly." + ), + profile=None, + ) + return _result_with_diagnostic(diagnostic, None, "profile") + + try: + _load_schema(resolved_profile.name) + for dependency_name in resolved_profile.dependencies: + _load_schema(dependency_name) + except SchemaAssetIntegrityError as exc: + diagnostic = _diagnostic( + code=exc.code, + message=str(exc), + profile=resolved_profile, + ossie_profile=contract_profile, + ) + return _result_with_diagnostic(diagnostic, resolved_profile, "integrity") + + try: + runtime = _load_validator_runtime() + except ImportError: + diagnostic = _diagnostic( + code="ossie.validator.unavailable", + message=_INSTALL_GUIDANCE, + profile=resolved_profile, + ) + return _result_with_diagnostic(diagnostic, resolved_profile, "availability") + + _, schema_error, _, _, reference_errors = runtime + try: + validator = _build_validator(resolved_profile, runtime) + validation_document = document + if contract_profile is not None and contract_profile.is_compatibility_alias: + # dbt 1.12 emits logical 0.1.0 although its accepted shape is the + # pinned upstream 0.1.1 schema. Normalize only the validation copy; + # callers retain and serialize the original 0.1.0 declaration. + validation_document = dict(document) + validation_document["version"] = contract_profile.validation_schema_version + errors = sorted( + validator.iter_errors(validation_document), + key=lambda error: ( + _json_pointer(tuple(error.absolute_path)), + _json_pointer(tuple(error.absolute_schema_path)), + str(error.validator), + error.message, + ), + ) + except SchemaAssetIntegrityError as exc: + diagnostic = _diagnostic( + code=exc.code, + message=str(exc), + profile=resolved_profile, + ossie_profile=contract_profile, + ) + return _result_with_diagnostic(diagnostic, resolved_profile, "integrity") + except (schema_error, *reference_errors) as exc: + diagnostic = _diagnostic( + code="ossie.schema.resource_error", + message=f"Pinned schema bundle could not be resolved offline: {exc}", + profile=resolved_profile, + ossie_profile=contract_profile, + ) + return _result_with_diagnostic(diagnostic, resolved_profile, "integrity") + + diagnostics = tuple( + _diagnostic( + code=_validation_code(str(error.validator)), + message=error.message, + profile=resolved_profile, + ossie_profile=contract_profile, + instance_path=tuple(error.absolute_path), + ) + for error in errors + ) + return SchemaValidationResult( + valid=not diagnostics, + profile=resolved_profile.name, + schema_commit=resolved_profile.source_commit, + diagnostics=sort_diagnostics(diagnostics), + failure_stage="schema" if diagnostics else None, + ) diff --git a/tests/interchange/ossie/test_diagnostics.py b/tests/interchange/ossie/test_diagnostics.py new file mode 100644 index 00000000..ec9cacaf --- /dev/null +++ b/tests/interchange/ossie/test_diagnostics.py @@ -0,0 +1,95 @@ +from dataclasses import FrozenInstanceError + +import pytest + +from sidemantic.interchange.ossie import ( + OSSIE_CORE_0_2_0_DEV0, + OssieDiagnostic, + OssieDiagnosticSeverity, + OssieSchemaProvenance, + OssieSourceLocation, + sort_diagnostics, +) + + +def test_diagnostic_preserves_structured_context_and_is_immutable(): + diagnostic = OssieDiagnostic( + severity="error", + code="ossie.schema.required", + message="'datasets' is required", + json_pointer="/semantic_model/0", + source=OssieSourceLocation("models/commerce.yaml", line=7, column=3), + scope="commerce", + profile=OSSIE_CORE_0_2_0_DEV0, + schema=OssieSchemaProvenance( + schema_id="https://ossie.apache.org/schema/core", + version="0.2.0.dev0", + commit="88e0011", + sha256="abc123", + ), + ) + + assert diagnostic.severity is OssieDiagnosticSeverity.ERROR + assert diagnostic.source.line == 7 + assert diagnostic.profile.identifier == "ossie-core:0.2.0.dev0" + assert diagnostic.schema.commit == "88e0011" + + with pytest.raises(FrozenInstanceError): + diagnostic.message = "changed" + + +def test_diagnostics_sort_deterministically_by_source_path_and_identity(): + source = OssieSourceLocation("models/commerce.yaml", line=4, column=2) + diagnostics = [ + OssieDiagnostic( + severity=OssieDiagnosticSeverity.WARNING, + code="ossie.semantic.reference", + message="unknown dataset", + json_pointer="/semantic_model/0/relationships/0", + source=source, + ), + OssieDiagnostic( + severity=OssieDiagnosticSeverity.ERROR, + code="ossie.schema.type", + message="expected an array", + json_pointer="/semantic_model/0", + source=source, + ), + OssieDiagnostic( + severity=OssieDiagnosticSeverity.INFO, + code="ossie.profile.alias", + message="using compatibility alias", + source=OssieSourceLocation("models/compat.json", line=1, column=1), + ), + ] + + expected_codes = ["ossie.schema.type", "ossie.semantic.reference", "ossie.profile.alias"] + assert [item.code for item in sort_diagnostics(diagnostics)] == expected_codes + assert [item.code for item in sort_diagnostics(reversed(diagnostics))] == expected_codes + + +@pytest.mark.parametrize( + "kwargs", + [ + {"identifier": "source", "line": 0}, + {"identifier": "source", "column": 1}, + {"identifier": "source", "end_column": 1}, + ], +) +def test_source_locations_require_one_based_consistent_coordinates(kwargs): + with pytest.raises(ValueError): + OssieSourceLocation(**kwargs) + + +def test_diagnostic_validates_stable_code_message_and_json_pointer(): + with pytest.raises(ValueError, match="contain no whitespace"): + OssieDiagnostic(severity="error", code="not stable", message="invalid") + with pytest.raises(ValueError, match="message must not be empty"): + OssieDiagnostic(severity="error", code="ossie.invalid", message="") + with pytest.raises(ValueError, match="start with"): + OssieDiagnostic( + severity="error", + code="ossie.invalid", + message="invalid", + json_pointer="semantic_model/0", + ) diff --git a/tests/interchange/ossie/test_documents.py b/tests/interchange/ossie/test_documents.py new file mode 100644 index 00000000..71b4bec4 --- /dev/null +++ b/tests/interchange/ossie/test_documents.py @@ -0,0 +1,132 @@ +from dataclasses import FrozenInstanceError + +import pytest + +from sidemantic.interchange.ossie import ( + FrozenJSONObject, + OssieDocumentSource, + OssieLogicalDocument, + OssieOntologyDocument, + OssieSerialization, + UnsupportedOssieDocument, +) + + +def test_logical_document_deeply_preserves_canonical_data_and_presence(): + parsed = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "commerce", + "datasets": [ + { + "name": "orders", + "fields": [ + { + "name": "ordered_at", + "dimension": {"is_time": False}, + "x-field-extension": {"enabled": True}, + } + ], + } + ], + } + ], + "x-root-extension": {"owner": "analytics"}, + } + original_bytes = b"# retained verbatim when requested\nversion: 0.2.0.dev0\n" + document = OssieLogicalDocument( + canonical_data=parsed, + serialization=OssieSerialization.YAML, + source=OssieDocumentSource(identifier="models/commerce.ossie.yaml", original_bytes=original_bytes), + ) + + parsed["semantic_model"][0]["name"] = "mutated" + parsed["x-root-extension"]["owner"] = "mutated" + + assert document.version == "0.2.0.dev0" + assert document.semantic_models[0]["name"] == "commerce" + assert document.unknown_data.to_dict() == {"x-root-extension": {"owner": "analytics"}} + assert document.is_field_present("/semantic_model/0/datasets/0/fields/0/dimension/is_time") + assert not document.is_field_present("/semantic_model/0/datasets/0/fields/0/dimension/granularity") + assert document.source is not None + assert document.source.original_bytes == original_bytes + assert len(document.source.sha256) == 64 + + +def test_document_data_is_deeply_immutable_but_can_be_thawed_for_serialization(): + document = OssieLogicalDocument( + canonical_data={"version": "0.1.1", "semantic_model": [{"name": "sales"}]}, + serialization="json", + ) + + assert isinstance(document.canonical_data, FrozenJSONObject) + with pytest.raises(TypeError): + document.canonical_data["version"] = "changed" + with pytest.raises(FrozenInstanceError): + document.serialization = OssieSerialization.YAML + + thawed = document.to_parsed_data() + thawed["semantic_model"][0]["name"] = "changed" + assert document.semantic_models[0]["name"] == "sales" + + +def test_frozen_json_object_constructor_also_freezes_nested_values(): + source = {"nested": [1, 2]} + frozen = FrozenJSONObject((("extension", source),)) + + source["nested"].append(3) + + assert frozen.to_dict() == {"extension": {"nested": [1, 2]}} + + +def test_ontology_document_keeps_ontology_and_mapping_families_distinct(): + document = OssieOntologyDocument( + canonical_data={ + "version": "0.2.0.dev0", + "name": "commerce-ontology", + "description": None, + "ontology": [{"concept": "Order", "type": "EntityType"}], + "ontology_mappings": [ + { + "name": "commerce-mapping", + "semantic_model": {"name": "commerce", "datasets": []}, + "concept_mappings": [], + } + ], + "x-ontology-extension": "preserved", + }, + serialization=OssieSerialization.JSON, + ) + + assert document.ontology[0]["concept"] == "Order" + assert document.ontology_mappings[0]["semantic_model"]["name"] == "commerce" + assert document.is_field_present("/description") + assert document.unknown_data.to_dict() == {"x-ontology-extension": "preserved"} + + +def test_unsupported_document_can_preserve_non_object_parsed_data(): + document = UnsupportedOssieDocument( + canonical_data=["not", {"yet": "classifiable"}], + serialization=OssieSerialization.YAML, + reason="root is not an object", + ) + + assert document.to_parsed_data() == ["not", {"yet": "classifiable"}] + assert document.version is None + assert document.reason == "root is not an object" + + +@pytest.mark.parametrize("invalid", [{"value": float("nan")}, {"value": object()}]) +def test_documents_reject_values_outside_the_json_compatible_data_model(invalid): + with pytest.raises((TypeError, ValueError)): + UnsupportedOssieDocument(canonical_data=invalid, serialization=OssieSerialization.JSON) + + +def test_source_bytes_are_optional_and_must_be_bytes(): + source = OssieDocumentSource(identifier="memory:logical") + assert source.original_bytes is None + assert source.sha256 is None + + with pytest.raises(TypeError, match="original_bytes must be bytes"): + OssieDocumentSource(original_bytes=bytearray(b"mutable")) diff --git a/tests/interchange/ossie/test_expression_validation.py b/tests/interchange/ossie/test_expression_validation.py new file mode 100644 index 00000000..592d4a83 --- /dev/null +++ b/tests/interchange/ossie/test_expression_validation.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from sidemantic.interchange.ossie.expression_validation import scalar_sql_expression_error + + +@pytest.mark.parametrize( + "expression", + [ + "SELECT amount FROM orders", + "amount IN (SELECT amount FROM refunds)", + "WITH cte_values AS (SELECT 1) SELECT * FROM cte_values", + "SELECT amount FROM orders UNION SELECT amount FROM refunds", + "CREATE TABLE unsafe (id INT)", + "DROP TABLE unsafe", + "INSERT INTO unsafe VALUES (1)", + "UPDATE unsafe SET id = 2", + "DELETE FROM unsafe", + ], +) +def test_prohibited_query_and_statement_corpus_is_rejected(expression: str) -> None: + error = scalar_sql_expression_error(expression, sqlglot_dialect=None) + + assert error is not None + assert "cannot contain" in error + + +@pytest.mark.parametrize( + "expression", + [ + "amount + tax", + "status IN ('paid', 'refunded')", + "CASE WHEN amount > 0 THEN amount ELSE 0 END", + "COUNT(DISTINCT customer_id)", + "SUM(amount) OVER (PARTITION BY region ORDER BY occurred_at)", + "COALESCE(discount, 0)", + "CAST(occurred_at AS DATE)", + ], +) +def test_required_scalar_constructs_cross_the_structural_gate(expression: str) -> None: + assert scalar_sql_expression_error(expression, sqlglot_dialect=None) is None + + +def test_multiple_statements_are_rejected() -> None: + assert scalar_sql_expression_error("amount; DROP TABLE unsafe", sqlglot_dialect=None) == ( + "expression must contain exactly one SQL expression" + ) diff --git a/tests/interchange/ossie/test_identifier.py b/tests/interchange/ossie/test_identifier.py new file mode 100644 index 00000000..4f4348d9 --- /dev/null +++ b/tests/interchange/ossie/test_identifier.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from sidemantic.interchange.ossie.identifier import ( + OSSIE_IDENTIFIER_MAX_LENGTH, + identifier_length, + identifier_within_limit, + normalize_identifier, +) + + +def test_regular_and_quoted_identifier_normalization() -> None: + assert normalize_identifier("customer_id") == "CUSTOMER_ID" + assert normalize_identifier("Customer_Id") == "CUSTOMER_ID" + assert normalize_identifier('"CUSTOMER_ID"') == "CUSTOMER_ID" + assert normalize_identifier('"customer_id"') == "customer_id" + assert normalize_identifier('"a""b"') == 'a"b' + + +def test_identifier_limit_counts_the_decoded_identifier_body() -> None: + assert identifier_length('"a""b"') == 3 + assert identifier_within_limit("x" * OSSIE_IDENTIFIER_MAX_LENGTH) + assert not identifier_within_limit("x" * (OSSIE_IDENTIFIER_MAX_LENGTH + 1)) diff --git a/tests/interchange/ossie/test_lowering.py b/tests/interchange/ossie/test_lowering.py new file mode 100644 index 00000000..f5323803 --- /dev/null +++ b/tests/interchange/ossie/test_lowering.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +import json + +import pytest + +from sidemantic.core.semantic_layer import SemanticLayer +from sidemantic.interchange.ossie import ( + OssieImportPolicy, + OssieParseOptions, + lower_ossie_document, + parse_ossie_document, +) + + +def _parse(text: str, *, policy: OssieImportPolicy = OssieImportPolicy.STRICT): + return parse_ossie_document( + text.encode(), + source_identifier="model.ossie.yaml", + options=OssieParseOptions(import_policy=policy, validate_schema=True), + ) + + +def test_lowers_multiple_scopes_without_flattening_duplicate_model_names() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: finance + datasets: + - name: orders + source: finance.orders + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + - name: marketing + datasets: + - name: orders + source: marketing.orders + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} +""" + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.catalog.scope_ids == ("finance", "marketing") + assert lowered.catalog["finance"].graph.get_model("orders").table == "finance.orders" + assert lowered.catalog["marketing"].graph.get_model("orders").table == "marketing.orders" + + +def test_classifies_query_source_and_preserves_unknown_primary_key() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: finance + datasets: + - name: orders + source: WITH recent AS (SELECT * FROM raw.orders) SELECT * FROM recent + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} +""" + ) + + model = lower_ossie_document(parsed, target_dialect="duckdb").catalog["finance"].graph.get_model("orders") + + assert model.table is None + assert model.sql.startswith("WITH recent") + assert model.primary_key is None + assert model.default_time_dimension is None + + +@pytest.mark.parametrize( + ("source", "expected_kind"), + [ + ("analytics.orders", "table"), + ('"analytics"."orders"', "table"), + ("SELECT * FROM raw.orders", "query"), + ("(SELECT * FROM raw.orders)", "query"), + ("WITH recent AS (SELECT * FROM raw.orders) SELECT * FROM recent", "query"), + ], +) +def test_source_kind_matrix_is_classified_without_rewriting(source: str, expected_kind: str) -> None: + parsed = _parse( + f"""version: 0.2.0.dev0 +semantic_model: + - name: scope + datasets: + - name: orders + source: '{source}' +""" + ) + + model = lower_ossie_document(parsed, target_dialect="duckdb").catalog["scope"].graph.get_model("orders") + + assert model.metadata["ossie_source_kind"] == expected_kind + assert (model.table or model.sql) == source + + +def test_query_source_is_executable_as_a_derived_table() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: scope + datasets: + - name: rows + source: SELECT 1 AS id UNION ALL SELECT 2 AS id + fields: + - name: id + datatype: Integer + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} +""" + ) + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + layer = SemanticLayer.from_catalog(lowered.catalog, auto_register=False) + + assert layer.query(dimensions=["rows.id"]).fetchall() == [(1,), (2,)] + + +def test_preserves_datatype_declared_time_and_derived_time_role_without_day_granularity() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: finance + datasets: + - name: events + source: analytics.events + fields: + - name: occurred_at + datatype: DateTime + expression: {dialects: [{dialect: ANSI_SQL, expression: occurred_at}]} + - name: loaded_at + datatype: DateTime + dimension: {is_time: false} + expression: {dialects: [{dialect: ANSI_SQL, expression: loaded_at}]} + - name: fiscal_year + datatype: Integer + dimension: {is_time: true} + expression: {dialects: [{dialect: ANSI_SQL, expression: fiscal_year}]} +""" + ) + + model = lower_ossie_document(parsed, target_dialect="duckdb").catalog["finance"].graph.get_model("events") + occurred = model.get_dimension("occurred_at") + loaded = model.get_dimension("loaded_at") + fiscal = model.get_dimension("fiscal_year") + + assert (occurred.type, occurred.logical_data_type, occurred.declared_is_time, occurred.granularity) == ( + "time", + "DateTime", + None, + None, + ) + assert (loaded.type, loaded.declared_is_time) == ("categorical", False) + assert (fiscal.type, fiscal.logical_data_type, fiscal.declared_is_time) == ("time", "Integer", True) + + +def test_selects_exact_bigquery_expression_before_ansi_fallback() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: warehouse + datasets: + - name: events + source: analytics.events + fields: + - name: occurred_on + expression: + dialects: + - {dialect: ANSI_SQL, expression: CAST(occurred_at AS DATE)} + - {dialect: BIGQUERY, expression: DATE(occurred_at)} +""" + ) + + graph = lower_ossie_document(parsed, target_dialect="bigquery").catalog["warehouse"].graph + assert graph.get_model("events").get_dimension("occurred_on").sql == "DATE(occurred_at)" + + +def test_lowers_named_relationship_without_fabricating_keys() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + primary_key: [id] + fields: + - {name: id, expression: {dialects: [{dialect: ANSI_SQL, expression: id}]}} + - {name: customer_id, expression: {dialects: [{dialect: ANSI_SQL, expression: customer_id}]}} + - name: customers + source: analytics.customers + primary_key: [id] + fields: + - {name: id, expression: {dialects: [{dialect: ANSI_SQL, expression: id}]}} + relationships: + - name: order_customer + from: orders + to: customers + from_columns: [customer_id] + to_columns: [id] +""" + ) + + graph = lower_ossie_document(parsed, target_dialect="duckdb").catalog["commerce"].graph + relationship = graph.get_model("orders").relationships[0] + path = graph.find_relationship_path("orders", "customers") + + assert relationship.edge_id == "order_customer" + assert relationship.foreign_key == "customer_id" + assert relationship.primary_key == "id" + assert path[0].edge_id == "order_customer" + + +def test_case_varied_regular_references_bind_to_canonical_runtime_names() -> None: + source = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "Commerce", + "datasets": [ + { + "name": "Orders", + "source": "analytics.orders", + "primary_key": ["iD"], + "fields": [ + {"name": "Id", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "id"}]}}, + { + "name": "Customer_Id", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "customer_id"}]}, + }, + ], + }, + { + "name": "Customers", + "source": "analytics.customers", + "primary_key": ["id"], + "unique_keys": [["ID"]], + "fields": [ + {"name": "ID", "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "id"}]}} + ], + }, + ], + "relationships": [ + { + "name": "Order_Customer", + "from": "ORDERS", + "to": "customers", + "from_columns": ["CUSTOMER_ID"], + "to_columns": ["Id"], + } + ], + } + ], + } + original = json.loads(json.dumps(source)) + parsed = parse_ossie_document( + json.dumps(source).encode(), + source_identifier="normalized.ossie.json", + options=OssieParseOptions(import_policy=OssieImportPolicy.STRICT, validate_schema=True), + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + graph = lowered.catalog["Commerce"].graph + orders = graph.get_model("Orders") + relationship = orders.relationships[0] + + assert lowered.valid + assert orders.primary_key == "Id" + assert graph.get_model("Customers").primary_key == "ID" + assert graph.get_model("Customers").unique_keys == [["ID"]] + assert relationship.name == "Customers" + assert relationship.foreign_key == "Customer_Id" + assert relationship.primary_key == "ID" + assert parsed.document.to_parsed_data() == original + + +def test_quoted_lowercase_reference_does_not_bind_to_regular_lowercase_declaration() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: + - {name: customer_id, expression: {dialects: [{dialect: ANSI_SQL, expression: customer_id}]}} + - name: customers + source: analytics.customers + primary_key: [id] + fields: + - {name: id, expression: {dialects: [{dialect: ANSI_SQL, expression: id}]}} + relationships: + - {name: bad_case, from: orders, to: '"customers"', from_columns: [customer_id], to_columns: ['"id"']} +""", + policy=OssieImportPolicy.PERMISSIVE, + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.executable + assert lowered.catalog["commerce"].graph.get_model("orders").relationships == [] + assert any( + diagnostic.code == "ossie.semantic.relationship.to_dataset_unknown" for diagnostic in lowered.diagnostics + ) + + +@pytest.mark.parametrize("policy", [OssieImportPolicy.STRICT, OssieImportPolicy.PERMISSIVE]) +def test_overlong_identifier_is_never_lowered(policy: OssieImportPolicy) -> None: + too_long = "x" * 129 + parsed = parse_ossie_document( + json.dumps( + { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "commerce", + "datasets": [{"name": too_long, "source": "analytics.rows", "fields": []}], + } + ], + } + ).encode(), + options=OssieParseOptions(import_policy=policy, validate_schema=True), + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert any(diagnostic.code == "ossie.semantic.identifier.length_exceeded" for diagnostic in lowered.diagnostics) + if policy is OssieImportPolicy.STRICT: + assert not lowered.executable + else: + assert lowered.catalog["commerce"].graph.models == {} + + +def test_strict_validation_errors_block_all_executable_scopes() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: [] + relationships: + - {name: unsafe, from: orders, to: missing, from_columns: [], to_columns: []} +""" + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert not lowered.executable + assert len(lowered.catalog) == 0 + assert any(diagnostic.code.startswith("ossie.semantic.relationship") for diagnostic in lowered.diagnostics) + + +def test_lowering_runs_schema_validation_when_parser_did_not_request_it() -> None: + parsed = parse_ossie_document( + b'{"version":"0.2.0.dev0","semantic_model":{}}', + options=OssieParseOptions(import_policy=OssieImportPolicy.STRICT, validate_schema=False), + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert not lowered.executable + assert lowered.schema_validation is not None + assert any(diagnostic.code == "ossie.schema.type" for diagnostic in lowered.diagnostics) + + +def test_permissive_policy_preserves_invalid_relationship_but_excludes_its_edge() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: + - {name: id, expression: {dialects: [{dialect: ANSI_SQL, expression: id}]}} + relationships: + - {name: unsafe, from: orders, to: missing, from_columns: [], to_columns: []} +""", + policy=OssieImportPolicy.PERMISSIVE, + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.executable + assert lowered.catalog["commerce"].graph.get_model("orders").relationships == [] + assert any(diagnostic.code == "ossie.lowering.relationship_unsafe" for diagnostic in lowered.diagnostics) + + +def test_permissive_policy_never_passes_malformed_unique_keys_to_runtime_models() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + unique_keys: [[id], [42]] + fields: + - {name: id, expression: {dialects: [{dialect: ANSI_SQL, expression: id}]}} +""", + policy=OssieImportPolicy.PERMISSIVE, + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.executable + scope = lowered.catalog["commerce"] + assert scope.graph.get_model("orders").unique_keys == [["id"]] + assert not scope.valid + assert any(diagnostic.code.startswith("ossie.schema") for diagnostic in scope.diagnostics) + assert any(diagnostic.code.startswith("ossie.schema") for diagnostic in lowered.diagnostics) + + +def test_permissive_policy_diagnoses_and_drops_unresolvable_declared_keys() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + primary_key: [missing] + unique_keys: [[id, ID]] + fields: + - {name: id, expression: {dialects: [{dialect: ANSI_SQL, expression: id}]}} +""", + policy=OssieImportPolicy.PERMISSIVE, + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + model = lowered.catalog["commerce"].graph.get_model("orders") + + assert lowered.executable + assert not lowered.valid + assert model.primary_key is None + assert model.unique_keys is None + assert {diagnostic.code for diagnostic in lowered.diagnostics} >= { + "ossie.semantic.dataset.key_field_unknown", + "ossie.semantic.dataset.key_column_duplicate", + } + + +def test_ontology_is_preserved_but_not_projected_as_an_executable_graph() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +name: business +ontology: [] +""" + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.document is parsed.document + assert not lowered.executable + assert len(lowered.catalog) == 0 + + +def test_target_dialect_is_required_for_executable_lowering() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: empty + datasets: [] +""" + ) + + lowered = lower_ossie_document(parsed) + + assert not lowered.executable + assert lowered.diagnostics[-1].code == "ossie.lowering.target_dialect_required" + + +def test_source_dialect_partitions_compilation_identity() -> None: + source = b"""version: 0.2.0.dev0 +semantic_model: + - name: scope + datasets: + - name: t + source: a-b +""" + default = parse_ossie_document( + source, + options=OssieParseOptions(import_policy=OssieImportPolicy.PERMISSIVE, validate_schema=True), + ) + bigquery = parse_ossie_document( + source, + options=OssieParseOptions( + import_policy=OssieImportPolicy.PERMISSIVE, + validate_schema=True, + source_dialect="bigquery", + ), + ) + + default_scope = lower_ossie_document(default, target_dialect="duckdb").catalog["scope"] + bigquery_scope = lower_ossie_document(bigquery, target_dialect="duckdb").catalog["scope"] + + assert default_scope.content_id == bigquery_scope.content_id + assert default_scope.compilation_id != bigquery_scope.compilation_id + assert default_scope.cache_key != bigquery_scope.cache_key + + +def test_strict_lowering_rejects_malformed_selected_sql_expression() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: + - name: broken + expression: {dialects: [{dialect: ANSI_SQL, expression: "not ("}]} +""" + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert not lowered.executable + assert not lowered.valid + assert any(diagnostic.code == "ossie.lowering.expression_invalid" for diagnostic in lowered.diagnostics) + + +def test_permissive_lowering_rejects_multiple_statements_but_keeps_safe_scope() -> None: + parsed = _parse( + """version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders + fields: + - name: unsafe + expression: {dialects: [{dialect: ANSI_SQL, expression: "id; DROP TABLE orders"}]} + - name: safe + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} +""", + policy=OssieImportPolicy.PERMISSIVE, + ) + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.executable + model = lowered.catalog["commerce"].graph.get_model("orders") + assert [dimension.name for dimension in model.dimensions] == ["safe"] + assert any(diagnostic.code == "ossie.lowering.expression_invalid" for diagnostic in lowered.diagnostics) diff --git a/tests/interchange/ossie/test_ossie_parser.py b/tests/interchange/ossie/test_ossie_parser.py new file mode 100644 index 00000000..49e19308 --- /dev/null +++ b/tests/interchange/ossie/test_ossie_parser.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from sidemantic.interchange.ossie import ( + OSSIE_CORE_0_1_1, + OSSIE_CORE_0_2_0_DEV0, + OssieConsumerProfile, + OssieImportPolicy, + OssieLogicalDocument, + OssieOntologyDocument, + OssieParseOptions, + OssiePreservationPolicy, + OssieSerialization, + UnsupportedOssieDocument, + parse_ossie_document, +) + +LOGICAL_YAML = b"""version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: [] +""" + +LOGICAL_JSON = b'{"version":"0.1.1","semantic_model":[{"name":"commerce","datasets":[]}]}' + + +def diagnostic_codes(result) -> list[str]: + return [diagnostic.code for diagnostic in result.diagnostics] + + +@pytest.mark.parametrize( + ("source", "identifier", "expected_serialization", "expected_profile"), + [ + (LOGICAL_YAML, "commerce.ossie.yaml", OssieSerialization.YAML, OSSIE_CORE_0_2_0_DEV0), + (LOGICAL_JSON, "commerce.ossie.json", OssieSerialization.JSON, OSSIE_CORE_0_1_1), + (LOGICAL_YAML, "memory:commerce", OssieSerialization.YAML, OSSIE_CORE_0_2_0_DEV0), + (LOGICAL_JSON, "memory:commerce", OssieSerialization.JSON, OSSIE_CORE_0_1_1), + ], +) +def test_parses_exact_bytes_and_infers_serialization_without_selecting_version( + source, + identifier, + expected_serialization, + expected_profile, +): + result = parse_ossie_document(source, source_identifier=identifier) + + assert isinstance(result.document, OssieLogicalDocument) + assert result.document.serialization is expected_serialization + assert result.document.version == expected_profile.schema_version + assert result.profile is expected_profile + assert result.options is not None + assert result.options.serialization is expected_serialization + assert result.document.source.identifier == identifier + assert result.document.source.media_type == f"application/{expected_serialization.value}" + assert result.document.source.original_bytes is None + assert result.valid + + +def test_explicit_serialization_wins_without_overriding_version_or_consumer_profile(): + result = parse_ossie_document( + LOGICAL_YAML, + source_identifier="misleading.json", + options=OssieParseOptions( + serialization="yaml", + consumer_profile=OssieConsumerProfile.OSSIE_CORE, + ), + ) + + assert result.document.serialization is OssieSerialization.YAML + assert result.profile is OSSIE_CORE_0_2_0_DEV0 + + +def test_source_bytes_are_retained_only_under_source_bytes_policy(): + retained = parse_ossie_document( + LOGICAL_YAML, + options=OssieParseOptions(preservation_policy=OssiePreservationPolicy.SOURCE_BYTES), + ) + canonical_only = parse_ossie_document(LOGICAL_YAML) + + assert retained.document.source.original_bytes == LOGICAL_YAML + assert retained.document.source.sha256 is not None + assert canonical_only.document.source.original_bytes is None + assert canonical_only.document.source.identifier == "" + assert canonical_only.document.source.media_type == "application/yaml" + + +@pytest.mark.parametrize( + ("root", "expected_type"), + [ + (b'{"version":"0.2.0.dev0","semantic_model":[]}', OssieLogicalDocument), + (b'{"version":"0.2.0.dev0","ontology":[]}', OssieOntologyDocument), + (b'{"version":"0.2.0.dev0","ontology_mappings":[]}', OssieOntologyDocument), + ], +) +def test_classifies_supported_document_families(root, expected_type): + result = parse_ossie_document(root) + + assert isinstance(result.document, expected_type) + assert "ossie.document.family_missing" not in diagnostic_codes(result) + assert "ossie.document.family_mixed" not in diagnostic_codes(result) + + +@pytest.mark.parametrize( + ("root", "expected_code", "reason_fragment"), + [ + ( + b'{"version":"0.2.0.dev0","semantic_model":[],"ontology":[]}', + "ossie.document.family_mixed", + "mixes logical", + ), + (b'{"version":"0.2.0.dev0","name":"unknown"}', "ossie.document.family_missing", "none of"), + (b'["version", "0.2.0.dev0"]', "ossie.document.root_type", "root must be an object"), + ], +) +def test_mixed_missing_and_non_object_roots_are_focused_unsupported_documents(root, expected_code, reason_fragment): + result = parse_ossie_document(root) + + assert isinstance(result.document, UnsupportedOssieDocument) + assert reason_fragment in result.document.reason + assert expected_code in diagnostic_codes(result) + assert not result.valid + assert result.blocks_lowering + + +def test_version_is_read_from_document_and_unsupported_profile_is_diagnostic(): + result = parse_ossie_document(b'{"version":"9.9","semantic_model":[]}') + + assert isinstance(result.document, OssieLogicalDocument) + assert result.document.version == "9.9" + assert result.options is None + assert result.profile is None + assert diagnostic_codes(result) == ["ossie.profile.unsupported"] + + +@pytest.mark.parametrize( + ("source", "expected_code"), + [ + (b'{"semantic_model":[]}', "ossie.profile.version_missing"), + (b'{"version":2,"semantic_model":[]}', "ossie.profile.version_type"), + ], +) +def test_missing_or_non_string_versions_are_diagnostics(source, expected_code): + result = parse_ossie_document(source) + + assert result.options is None + assert result.profile is None + assert diagnostic_codes(result) == [expected_code] + + +@pytest.mark.parametrize( + ("source", "identifier", "expected_line", "expected_column"), + [ + (b'{"version":"0.2.0.dev0",', "broken.json", 1, 25), + (b"version: [\nsemantic_model: []\n", "broken.yaml", 3, 1), + ], +) +def test_syntax_errors_are_unsupported_results_not_parser_tracebacks( + source, + identifier, + expected_line, + expected_column, +): + result = parse_ossie_document(source, source_identifier=identifier) + + assert isinstance(result.document, UnsupportedOssieDocument) + assert diagnostic_codes(result) == ["ossie.parse.syntax"] + diagnostic = result.diagnostics[0] + assert diagnostic.source.identifier == identifier + assert diagnostic.source.line == expected_line + assert diagnostic.source.column == expected_column + + +@pytest.mark.parametrize( + ("source", "identifier", "expected_line"), + [ + ( + b"version: 0.2.0.dev0\nsemantic_model: []\nsemantic_model: []\n", + "duplicate.yaml", + 3, + ), + ( + b'{"version":"0.2.0.dev0","semantic_model":[],"semantic_model":[]}', + "duplicate.json", + None, + ), + ], +) +def test_duplicate_keys_never_silently_overwrite(source, identifier, expected_line): + result = parse_ossie_document(source, source_identifier=identifier) + + assert isinstance(result.document, UnsupportedOssieDocument) + assert diagnostic_codes(result) == ["ossie.parse.duplicate_key"] + assert result.diagnostics[0].source.line == expected_line + + +def test_non_json_yaml_values_are_rejected_with_a_structured_diagnostic(): + result = parse_ossie_document( + b"version: 0.2.0.dev0\nsemantic_model: []\nloaded_at: 2026-08-23\n", + source_identifier="typed.yaml", + ) + + assert isinstance(result.document, UnsupportedOssieDocument) + assert diagnostic_codes(result) == ["ossie.parse.non_json_value"] + + +def test_optional_schema_validation_is_returned_and_strict_errors_are_visible(): + result = parse_ossie_document( + b'{"version":"0.2.0.dev0","semantic_model":{}}', + options=OssieParseOptions(validate_schema=True, import_policy=OssieImportPolicy.STRICT), + ) + + assert result.schema_validation is not None + assert not result.schema_validation.valid + assert result.schema_validation.failure_stage == "schema" + assert "ossie.schema.type" in diagnostic_codes(result) + assert result.blocks_lowering + + +def test_permissive_policy_preserves_errors_without_claiming_validity_or_blocking(): + result = parse_ossie_document( + b'{"version":"0.2.0.dev0","semantic_model":{}}', + options=OssieParseOptions(validate_schema=True, import_policy=OssieImportPolicy.PERMISSIVE), + ) + + assert not result.valid + assert not result.blocks_lowering + assert result.schema_validation is not None + + +def test_parse_result_and_options_are_immutable(): + result = parse_ossie_document(LOGICAL_YAML) + + with pytest.raises(FrozenInstanceError): + result.document = UnsupportedOssieDocument( + canonical_data=None, + serialization=OssieSerialization.YAML, + ) + with pytest.raises(FrozenInstanceError): + result.parse_options.validate_schema = True + + +def test_dbt_compatibility_alias_comes_from_consumer_options_not_json_extension(): + result = parse_ossie_document( + b'{"version":"0.1.0","semantic_model":[]}', + source_identifier="alias.yaml", + options=OssieParseOptions( + serialization=OssieSerialization.JSON, + consumer_profile=OssieConsumerProfile.DBT_1_12, + ), + ) + + assert result.profile is not None + assert result.profile.is_compatibility_alias + assert result.profile.consumer_profile is OssieConsumerProfile.DBT_1_12 + + +def test_dbt_compatibility_alias_validates_with_explicit_consumer_context(): + result = parse_ossie_document( + b'{"version":"0.1.0","semantic_model":[]}', + options=OssieParseOptions( + serialization=OssieSerialization.JSON, + consumer_profile=OssieConsumerProfile.DBT_1_12, + validate_schema=True, + ), + ) + + assert result.valid + assert result.schema_validation is not None + assert result.schema_validation.valid + assert result.schema_validation.profile == "logical-0.1.1" + + +def test_exact_bytes_are_required(): + with pytest.raises(TypeError, match="exact immutable bytes"): + parse_ossie_document(bytearray(LOGICAL_YAML)) + + +def test_deeply_nested_json_returns_a_limit_diagnostic_instead_of_recursion_error(): + source = b"[" * 10_000 + b"]" * 10_000 + + result = parse_ossie_document( + source, + source_identifier="deep.json", + options=OssieParseOptions(serialization=OssieSerialization.JSON), + ) + + assert isinstance(result.document, UnsupportedOssieDocument) + assert diagnostic_codes(result) == ["ossie.parse.limit"] + + +def test_parser_enforces_a_bounded_input_budget(): + result = parse_ossie_document(b" " * (16 * 1024 * 1024 + 1), source_identifier="large.yaml") + + assert isinstance(result.document, UnsupportedOssieDocument) + assert diagnostic_codes(result) == ["ossie.parse.limit"] diff --git a/tests/interchange/ossie/test_profiles.py b/tests/interchange/ossie/test_profiles.py new file mode 100644 index 00000000..ebde523b --- /dev/null +++ b/tests/interchange/ossie/test_profiles.py @@ -0,0 +1,92 @@ +from dataclasses import FrozenInstanceError + +import pytest + +from sidemantic.interchange.ossie import ( + DBT_1_12_0_1_0_ALIAS, + OSSIE_CORE_0_1_1, + OssieConsumerProfile, + OssieImportPolicy, + OssieOptions, + OssiePreservationPolicy, + OssieProfileError, + OssieSerialization, + resolve_ossie_profile, +) + + +@pytest.mark.parametrize("schema_version", ["0.1.1", "0.2.0.dev0"]) +@pytest.mark.parametrize("serialization", [OssieSerialization.YAML, OssieSerialization.JSON]) +def test_core_schema_version_and_serialization_are_independent(schema_version, serialization): + options = OssieOptions(schema_version=schema_version, serialization=serialization) + + assert options.schema_version == schema_version + assert options.serialization is serialization + assert options.profile.consumer_profile is OssieConsumerProfile.OSSIE_CORE + + +def test_0_1_0_is_only_a_dbt_1_12_compatibility_alias(): + with pytest.raises(OssieProfileError, match="only a dbt-1.12 compatibility alias"): + resolve_ossie_profile("0.1.0", OssieConsumerProfile.OSSIE_CORE) + + options = OssieOptions( + schema_version="0.1.0", + serialization=OssieSerialization.JSON, + consumer_profile=OssieConsumerProfile.DBT_1_12, + ) + + assert options.profile is DBT_1_12_0_1_0_ALIAS + assert options.profile.is_compatibility_alias + assert options.profile.compatibility_alias_for == "0.1.1" + assert options.profile.validation_schema_version == "0.1.1" + assert options.profile.upstream_schema_version is None + + +def test_ordinary_profile_validation_versions_are_unchanged(): + assert OSSIE_CORE_0_1_1.validation_schema_version == "0.1.1" + + +def test_dbt_1_12_rejects_draft_0_2_profile(): + with pytest.raises(OssieProfileError, match="Unsupported schema version"): + OssieOptions( + schema_version="0.2.0.dev0", + serialization=OssieSerialization.JSON, + consumer_profile=OssieConsumerProfile.DBT_1_12, + ) + + +def test_invalid_enum_options_use_the_central_profile_error(): + with pytest.raises(OssieProfileError, match="Invalid Ossie option"): + OssieOptions(schema_version="0.1.1", serialization="toml") + + +def test_options_keep_policy_dialects_and_preservation_independent(): + options = OssieOptions( + schema_version="0.1.1", + serialization="yaml", + consumer_profile="ossie-core", + import_policy="permissive", + source_dialect="ANSI_SQL", + target_dialect="postgres", + preservation_policy="source-bytes", + ) + + assert options.profile is OSSIE_CORE_0_1_1 + assert options.serialization is OssieSerialization.YAML + assert options.import_policy is OssieImportPolicy.PERMISSIVE + assert options.source_dialect == "ANSI_SQL" + assert options.target_dialect == "postgres" + assert options.preservation_policy is OssiePreservationPolicy.SOURCE_BYTES + + with pytest.raises(FrozenInstanceError): + options.target_dialect = "bigquery" + + +@pytest.mark.parametrize( + ("field", "value"), + [("schema_version", " 0.1.1"), ("source_dialect", ""), ("target_dialect", " ")], +) +def test_options_reject_ambiguous_empty_or_whitespace_values(field, value): + values = {"schema_version": "0.1.1", "serialization": OssieSerialization.JSON, field: value} + with pytest.raises(OssieProfileError): + OssieOptions(**values) diff --git a/tests/interchange/ossie/test_schema_conformance_corpus.py b/tests/interchange/ossie/test_schema_conformance_corpus.py new file mode 100644 index 00000000..5f41b94e --- /dev/null +++ b/tests/interchange/ossie/test_schema_conformance_corpus.py @@ -0,0 +1,564 @@ +"""Deterministic instance mutations for the vendored Apache Ossie schemas. + +The corpus deliberately tests instance-observable constraints rather than +repeating every occurrence of a schema keyword. ``properties``, ``items``, +and ``$defs`` are applicators: their behavior is observed through the child +constraints below. ``$ref`` is also an applicator and cannot produce a +standalone instance diagnostic; the ontology cases exercise both an external +logical-schema reference and the ``oneOf`` reached through that reference. +Schema annotations such as ``description``, ``title``, ``examples``, ``$id``, +and ``$schema`` have no instance failure mode and are intentionally not +mutated. +""" + +from __future__ import annotations + +import json +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from sidemantic.interchange.ossie import ( + OssieImportPolicy, + OssieParseOptions, + OssieSerialization, + lower_ossie_document, + parse_ossie_document, + validation, +) + +TEST_ROOT = Path(__file__).parents[2] +FIXTURE_ROOT = TEST_ROOT / "ossie-fixtures" / "cases" +SCHEMA_ROOT = Path(validation.__file__).parent / "schemas" + + +@dataclass(frozen=True, slots=True) +class ProfileAsset: + name: str + schema_path: str + fixture_path: str + fixture_serialization: OssieSerialization + + +@dataclass(frozen=True, slots=True) +class MutationCase: + profile: str + name: str + keyword: str + pointer: str + path: tuple[str | int, ...] + value: Any = None + remove: bool = False + type_value_class: str | None = None + + @property + def expected_code(self) -> str: + return { + "required": "ossie.schema.required", + "enum": "ossie.schema.enum", + "const": "ossie.schema.const", + "minItems": "ossie.schema.min_items", + "type": "ossie.schema.type", + "additionalProperties": "ossie.schema.additional_properties", + "oneOf": "ossie.schema.one_of", + # A $ref delegates the observable failure to the referenced + # schema keyword. This case proves the ontology -> logical ref. + "ref": "ossie.schema.type", + }[self.keyword] + + +PROFILE_ASSETS = ( + ProfileAsset( + name="logical-0.1.1", + schema_path="logical/0.1.1/schema.json", + fixture_path="logical-0.1.1-valid/document.json", + fixture_serialization=OssieSerialization.JSON, + ), + ProfileAsset( + name="logical-0.2.0.dev0", + schema_path="logical/0.2.0.dev0/schema.json", + fixture_path="logical-0.2.0.dev0-valid/document.json", + fixture_serialization=OssieSerialization.JSON, + ), + ProfileAsset( + name="ontology-0.2.0.dev0", + schema_path="ontology/0.2.0.dev0/schema.json", + fixture_path="ontology-0.2.0.dev0-valid/document.yaml", + fixture_serialization=OssieSerialization.YAML, + ), +) + + +def _case( + profile: str, + name: str, + keyword: str, + pointer: str, + *path: str | int, + value: Any = None, + remove: bool = False, + type_value_class: str | None = None, +) -> MutationCase: + return MutationCase( + profile=profile, + name=name, + keyword=keyword, + pointer=pointer, + path=path, + value=value, + remove=remove, + type_value_class=type_value_class, + ) + + +MUTATION_CASES = ( + # OSI 0.1.1: enum is represented by the Dialect definition. + _case( + "logical-0.1.1", + "required-dataset-name", + "required", + "/semantic_model/0/datasets/0", + "semantic_model", + 0, + "datasets", + 0, + "name", + remove=True, + ), + _case( + "logical-0.1.1", + "enum-dialect", + "enum", + "/dialects/0", + "dialects", + 0, + value="NOT_A_DIALECT", + ), + _case("logical-0.1.1", "const-version", "const", "/version", "version", value="9.9.9"), + _case( + "logical-0.1.1", + "min-items-datasets", + "minItems", + "/semantic_model/0/datasets", + "semantic_model", + 0, + "datasets", + value=[], + ), + _case( + "logical-0.1.1", + "type-primitive-name", + "type", + "/semantic_model/0/name", + "semantic_model", + 0, + "name", + value=42, + type_value_class="primitive", + ), + _case( + "logical-0.1.1", + "type-object-semantic-model", + "type", + "/semantic_model", + "semantic_model", + value={}, + type_value_class="object", + ), + _case( + "logical-0.1.1", + "type-array-name", + "type", + "/semantic_model/0/name", + "semantic_model", + 0, + "name", + value=[], + type_value_class="array", + ), + _case( + "logical-0.1.1", + "additional-properties-semantic-model", + "additionalProperties", + "/semantic_model/0", + "semantic_model", + 0, + "unexpected", + value=True, + ), + _case( + "logical-0.1.1", + "one-of-ai-context", + "oneOf", + "/semantic_model/0/ai_context", + "semantic_model", + 0, + "ai_context", + value=42, + ), + # Apache Ossie 0.2: enum is represented by the logical DataType. + _case( + "logical-0.2.0.dev0", + "required-dataset-name", + "required", + "/semantic_model/0/datasets/0", + "semantic_model", + 0, + "datasets", + 0, + "name", + remove=True, + ), + _case( + "logical-0.2.0.dev0", + "enum-datatype", + "enum", + "/semantic_model/0/datasets/0/fields/0/datatype", + "semantic_model", + 0, + "datasets", + 0, + "fields", + 0, + "datatype", + value="NOT_A_DATATYPE", + ), + _case("logical-0.2.0.dev0", "const-version", "const", "/version", "version", value="9.9.9"), + _case( + "logical-0.2.0.dev0", + "min-items-datasets", + "minItems", + "/semantic_model/0/datasets", + "semantic_model", + 0, + "datasets", + value=[], + ), + _case( + "logical-0.2.0.dev0", + "type-primitive-name", + "type", + "/semantic_model/0/name", + "semantic_model", + 0, + "name", + value=42, + type_value_class="primitive", + ), + _case( + "logical-0.2.0.dev0", + "type-object-semantic-model", + "type", + "/semantic_model", + "semantic_model", + value={}, + type_value_class="object", + ), + _case( + "logical-0.2.0.dev0", + "type-array-name", + "type", + "/semantic_model/0/name", + "semantic_model", + 0, + "name", + value=[], + type_value_class="array", + ), + _case( + "logical-0.2.0.dev0", + "additional-properties-semantic-model", + "additionalProperties", + "/semantic_model/0", + "semantic_model", + 0, + "unexpected", + value=True, + ), + _case( + "logical-0.2.0.dev0", + "one-of-ai-context", + "oneOf", + "/semantic_model/0/ai_context", + "semantic_model", + 0, + "ai_context", + value=42, + ), + # Apache Ossie ontology 0.2: component type and multiplicity are enums. + _case( + "ontology-0.2.0.dev0", + "required-component-type", + "required", + "/ontology/0", + "ontology", + 0, + "type", + remove=True, + ), + _case( + "ontology-0.2.0.dev0", + "enum-component-type", + "enum", + "/ontology/0/type", + "ontology", + 0, + "type", + value="NOT_A_CONCEPT_TYPE", + ), + _case("ontology-0.2.0.dev0", "const-version", "const", "/version", "version", value="9.9.9"), + _case("ontology-0.2.0.dev0", "min-items-ontology", "minItems", "/ontology", "ontology", value=[]), + _case( + "ontology-0.2.0.dev0", + "type-primitive-name", + "type", + "/name", + "name", + value=42, + type_value_class="primitive", + ), + _case( + "ontology-0.2.0.dev0", + "type-object-ontology", + "type", + "/ontology", + "ontology", + value={}, + type_value_class="object", + ), + _case( + "ontology-0.2.0.dev0", + "type-array-name", + "type", + "/name", + "name", + value=[], + type_value_class="array", + ), + _case( + "ontology-0.2.0.dev0", + "additional-properties-component", + "additionalProperties", + "/ontology/0", + "ontology", + 0, + "unexpected", + value=True, + ), + _case( + "ontology-0.2.0.dev0", + "one-of-external-ai-context-ref", + "oneOf", + "/ai_context", + "ai_context", + value=42, + ), + _case( + "ontology-0.2.0.dev0", + "external-logical-semantic-model-ref", + "ref", + "/ontology_mappings/0/semantic_model", + "ontology_mappings", + 0, + "semantic_model", + value=[], + ), +) + +EXPECTED_CASE_KEYWORDS = { + "logical-0.1.1": {"required", "enum", "const", "minItems", "type", "additionalProperties", "oneOf"}, + "logical-0.2.0.dev0": {"required", "enum", "const", "minItems", "type", "additionalProperties", "oneOf"}, + "ontology-0.2.0.dev0": { + "required", + "enum", + "const", + "minItems", + "type", + "additionalProperties", + "oneOf", + "ref", + }, +} + + +def _asset(name: str) -> ProfileAsset: + return next(asset for asset in PROFILE_ASSETS if asset.name == name) + + +def _load_fixture(asset: ProfileAsset) -> dict[str, Any]: + source = (FIXTURE_ROOT / asset.fixture_path).read_text(encoding="utf-8") + value = json.loads(source) if asset.fixture_serialization is OssieSerialization.JSON else yaml.safe_load(source) + assert isinstance(value, dict) + return value + + +def _representative_document(profile: str) -> dict[str, Any]: + """Return a valid fixture-derived document with ref/enum targets present.""" + + document = _load_fixture(_asset(profile)) + if profile == "logical-0.1.1": + document["dialects"] = ["ANSI_SQL"] + + if profile.startswith("logical-"): + dataset = document["semantic_model"][0]["datasets"][0] + dataset["fields"] = [ + { + "name": "id", + "expression": { + "dialects": [{"dialect": "ANSI_SQL", "expression": "id"}], + }, + } + ] + if profile == "logical-0.2.0.dev0": + dataset["fields"][0]["datatype"] = "String" + return document + + +def _load_schema(asset: ProfileAsset) -> dict[str, Any]: + schema = json.loads((SCHEMA_ROOT / asset.schema_path).read_text(encoding="utf-8")) + assert isinstance(schema, dict) + return schema + + +def _schema_keywords(value: Any) -> set[str]: + keywords = {"type", "const", "enum", "minItems", "required", "additionalProperties", "oneOf", "$ref"} + if isinstance(value, dict): + return {key for key, child in value.items() if key in keywords} | { + keyword for child in value.values() for keyword in _schema_keywords(child) + } + if isinstance(value, list): + return {keyword for child in value for keyword in _schema_keywords(child)} + return set() + + +def _apply_mutation(document: dict[str, Any], case: MutationCase) -> None: + parent: Any = document + for part in case.path[:-1]: + parent = parent[part] + leaf = case.path[-1] + if case.remove: + del parent[leaf] + else: + parent[leaf] = deepcopy(case.value) + + +def _diagnostic_contract(result: validation.SchemaValidationResult) -> list[tuple[str, str]]: + return [(diagnostic.code, diagnostic.json_pointer) for diagnostic in result.diagnostics] + + +@pytest.mark.parametrize("asset", PROFILE_ASSETS, ids=lambda asset: asset.name) +def test_pinned_valid_fixtures_pass(asset: ProfileAsset) -> None: + """Every corpus starts from the existing valid fixture for its asset.""" + + result = validation.validate_ossie_schema(_load_fixture(asset), profile=asset.name) + + assert result.valid + assert result.profile == asset.name + + +def test_pinned_schema_assets_expose_the_corpus_keyword_classes() -> None: + expected = {"type", "const", "required", "additionalProperties", "minItems"} + for asset in PROFILE_ASSETS: + schema = _load_schema(asset) + assert schema["type"] == "object" + assert schema["properties"]["version"]["const"] in {"0.1.1", "0.2.0.dev0"} + assert expected <= _schema_keywords(schema) + + assert {"enum", "oneOf"} <= _schema_keywords(_load_schema(_asset("logical-0.1.1"))) + assert {"enum", "oneOf"} <= _schema_keywords(_load_schema(_asset("logical-0.2.0.dev0"))) + # The ontology's AIContext ref reaches the logical schema's oneOf, while + # the ontology asset itself declares the external $ref rather than a + # duplicate oneOf. + assert {"enum", "$ref"} <= _schema_keywords(_load_schema(_asset("ontology-0.2.0.dev0"))) + + +def test_mutation_matrix_covers_each_observable_constraint_for_each_profile() -> None: + for profile, expected_keywords in EXPECTED_CASE_KEYWORDS.items(): + profile_cases = [case for case in MUTATION_CASES if case.profile == profile] + assert {case.keyword for case in profile_cases} == expected_keywords + assert {case.type_value_class for case in profile_cases if case.keyword == "type"} == { + "primitive", + "object", + "array", + } + + +@pytest.mark.parametrize("case", MUTATION_CASES, ids=lambda case: f"{case.profile}-{case.name}") +def test_schema_conformance_mutation_fails_at_stable_code_and_pointer(case: MutationCase) -> None: + baseline = _representative_document(case.profile) + assert validation.validate_ossie_schema(baseline, profile=case.profile).valid + + mutated = deepcopy(baseline) + _apply_mutation(mutated, case) + result = validation.validate_ossie_schema(mutated, profile=case.profile) + + assert not result.valid + assert _diagnostic_contract(result) == [(case.expected_code, case.pointer)] + + +def test_strict_parser_and_lowering_block_a_schema_mutation() -> None: + document = _representative_document("logical-0.2.0.dev0") + case = next( + case + for case in MUTATION_CASES + if case.profile == "logical-0.2.0.dev0" and case.name == "additional-properties-semantic-model" + ) + _apply_mutation(document, case) + + parsed = parse_ossie_document( + json.dumps(document, separators=(",", ":")).encode(), + source_identifier="schema-conformance.json", + options=OssieParseOptions( + serialization=OssieSerialization.JSON, + import_policy=OssieImportPolicy.STRICT, + validate_schema=True, + ), + ) + assert not parsed.valid + assert parsed.blocks_lowering + assert _diagnostic_contract(parsed.schema_validation) == [(case.expected_code, case.pointer)] + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + assert not lowered.executable + assert lowered.catalog.scope_ids == () + assert _diagnostic_contract(lowered.schema_validation) == [(case.expected_code, case.pointer)] + + +def test_permissive_parser_and_lowering_retain_a_schema_diagnostic() -> None: + document = _representative_document("logical-0.2.0.dev0") + case = next( + case + for case in MUTATION_CASES + if case.profile == "logical-0.2.0.dev0" and case.name == "additional-properties-semantic-model" + ) + _apply_mutation(document, case) + + parsed = parse_ossie_document( + json.dumps(document, separators=(",", ":")).encode(), + source_identifier="schema-conformance.json", + options=OssieParseOptions( + serialization=OssieSerialization.JSON, + import_policy=OssieImportPolicy.PERMISSIVE, + validate_schema=True, + ), + ) + assert not parsed.valid + assert not parsed.blocks_lowering + assert _diagnostic_contract(parsed.schema_validation) == [(case.expected_code, case.pointer)] + + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + assert lowered.executable + assert lowered.catalog.scope_ids == ("commerce",) + assert not lowered.valid + assert _diagnostic_contract(lowered.schema_validation) == [(case.expected_code, case.pointer)] + scope = lowered.catalog["commerce"] + assert not scope.valid + assert [ + (diagnostic.code, diagnostic.json_pointer) + for diagnostic in scope.diagnostics + if diagnostic.code == case.expected_code + ] == [(case.expected_code, case.pointer)] diff --git a/tests/interchange/ossie/test_schema_validation.py b/tests/interchange/ossie/test_schema_validation.py new file mode 100644 index 00000000..7ecaf9eb --- /dev/null +++ b/tests/interchange/ossie/test_schema_validation.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import socket +import subprocess +import sys +import urllib.request +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from sidemantic.interchange.ossie import validation +from sidemantic.interchange.ossie.profiles import DBT_1_12_0_1_0_ALIAS, OssieConsumerProfile + +TEST_ROOT = Path(__file__).parents[2] +FIXTURE_ROOT = TEST_ROOT / "ossie-fixtures" +SCHEMA_ROOT = Path(validation.__file__).parent / "schemas" + + +def _load_manifest() -> dict[str, Any]: + return yaml.safe_load((FIXTURE_ROOT / "manifest.yaml").read_text()) + + +def _load_case(case: dict[str, Any]) -> Any: + source = (FIXTURE_ROOT / case["input"]).read_text() + if case["serialization"] == "json": + return json.loads(source) + return yaml.safe_load(source) + + +def _diagnostic_contract(result: validation.SchemaValidationResult) -> list[dict[str, Any]]: + return [ + { + "code": diagnostic.code, + "instance_path": diagnostic.json_pointer, + } + for diagnostic in result.diagnostics + ] + + +@pytest.mark.parametrize("case", _load_manifest()["cases"], ids=lambda case: case["id"]) +def test_manifest_schema_case(case: dict[str, Any]) -> None: + result = validation.validate_ossie_schema( + _load_case(case), + profile=case["profile"], + ) + + assert result.valid is case["expected"]["valid"] + assert result.profile == case["expected"].get("profile", case["profile"]) + assert _diagnostic_contract(result) == case["expected"]["diagnostics"] + + +def test_yaml_and_json_are_equivalent_within_each_version() -> None: + manifest = _load_manifest() + cases = {case["id"]: case for case in manifest["cases"]} + + for case_ids in manifest["equivalence_groups"].values(): + documents = [_load_case(cases[case_id]) for case_id in case_ids] + assert documents[1:] == documents[:-1] + + +def test_schema_profiles_have_exact_pins_and_integrity() -> None: + expected = { + "logical-0.1.1": ( + "faf581054dcf7964d5fe0ceae7d6f415c8ce32a5", + "c1e9adec39562786aa78809665fba568797b15f4c53a0847d9cbcf2dead1bc94", + "c1e9adec39562786aa78809665fba568797b15f4c53a0847d9cbcf2dead1bc94", + ), + "logical-0.2.0.dev0": ( + "88e0011148283302c9a04cd0287e00e0b9d87354", + "8ce9f82aa92080265f9ae119e31cda5bef062f489674d3c467245c2d4c5ff264", + "8ce9f82aa92080265f9ae119e31cda5bef062f489674d3c467245c2d4c5ff264", + ), + "ontology-0.2.0.dev0": ( + "88e0011148283302c9a04cd0287e00e0b9d87354", + "555820756a7d30bc6986ce1b57feaa9937ec4ddd3288878b8af0e3361d824a41", + "c0ce26ff658aff52307f01bdc564061d194c1987e930d61ff498e63456b9b41d", + ), + } + + profiles = {profile.name: profile for profile in validation.available_schema_profiles()} + assert set(profiles) == set(expected) + + for name, (commit, runtime_checksum, upstream_checksum) in expected.items(): + profile = profiles[name] + assert profile.source_commit == commit + assert commit in profile.source_url + assert profile.runtime_sha256 == runtime_checksum + assert profile.upstream_sha256 == upstream_checksum + schema_bytes = (SCHEMA_ROOT / profile.runtime_path).read_bytes() + upstream_bytes = (SCHEMA_ROOT / profile.upstream_path).read_bytes() + assert hashlib.sha256(schema_bytes).hexdigest() == runtime_checksum + assert hashlib.sha256(upstream_bytes).hexdigest() == upstream_checksum + + +def test_ontology_runtime_schema_uses_only_pinned_local_refs() -> None: + logical = json.loads((SCHEMA_ROOT / "logical/0.2.0.dev0/schema.json").read_text()) + upstream = json.loads((SCHEMA_ROOT / "ontology/0.2.0.dev0/upstream.json").read_text()) + runtime = json.loads((SCHEMA_ROOT / "ontology/0.2.0.dev0/schema.json").read_text()) + + def refs(value: Any) -> list[str]: + if isinstance(value, dict): + return [ + *([value["$ref"]] if "$ref" in value else []), + *(ref for child in value.values() for ref in refs(child)), + ] + if isinstance(value, list): + return [ref for child in value for ref in refs(child)] + return [] + + assert upstream["$id"] == logical["$id"] + assert runtime["$id"] == "urn:sidemantic:ossie:schema:ontology:0.2.0.dev0" + assert runtime["$id"] != logical["$id"] + assert any("/apache/ossie/main/" in ref for ref in refs(upstream)) + assert not any(ref.startswith("http") for ref in refs(runtime)) + assert refs(runtime).count("urn:sidemantic:ossie:schema:logical:0.2.0.dev0#/$defs/AIContext") == 1 + assert refs(runtime).count("urn:sidemantic:ossie:schema:logical:0.2.0.dev0#/$defs/SemanticModel") == 1 + + +def test_ontology_validation_cannot_open_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_network(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError("schema validation attempted network access") + + monkeypatch.setattr(urllib.request, "urlopen", fail_network) + monkeypatch.setattr(socket, "create_connection", fail_network) + + ontology_case = next(case for case in _load_manifest()["cases"] if case["id"] == "ontology-0.2.0.dev0-offline-refs") + result = validation.validate_ossie_schema( + _load_case(ontology_case), + profile=ontology_case["profile"], + ) + assert result.valid + + +def test_optional_validator_is_imported_lazily() -> None: + completed = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; " + "import sidemantic.interchange.ossie.validation; " + "assert 'jsonschema' not in sys.modules; " + "assert 'referencing' not in sys.modules" + ), + ], + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stderr + + +def test_missing_optional_validator_fails_explicitly( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def unavailable() -> tuple[Any, ...]: + raise ModuleNotFoundError("No module named 'jsonschema'") + + monkeypatch.setattr(validation, "_load_validator_runtime", unavailable) + document = _load_case(_load_manifest()["cases"][0]) + + result = validation.validate_ossie_schema(document, profile="logical-0.1.1") + + assert not result.valid + assert result.diagnostics[0].code == "ossie.validator.unavailable" + assert result.failure_stage == "availability" + assert "sidemantic[ossie]" in result.diagnostics[0].message + + +def test_auto_detection_requires_an_unambiguous_upstream_profile() -> None: + valid_document = _load_case(_load_manifest()["cases"][0]) + assert validation.validate_ossie_schema(valid_document).valid + + result = validation.validate_ossie_schema({"version": "0.2.0.dev0", "semantic_model": [], "ontology": []}) + assert not result.valid + assert result.diagnostics[0].code == "ossie.schema.profile_undetected" + assert result.failure_stage == "profile" + + +def test_validation_result_is_serialization_friendly() -> None: + invalid_case = next(case for case in _load_manifest()["cases"] if case["id"] == "logical-0.1.0-not-upstream-valid") + result = validation.validate_ossie_schema( + _load_case(invalid_case), + profile=invalid_case["profile"], + ) + + serialized = json.loads(json.dumps(result.to_dict())) + assert serialized["stage"] == "schema" + assert serialized["valid"] is False + assert serialized["diagnostics"][0]["json_pointer"] == "/version" + + +def test_dbt_0_1_0_alias_requires_context_and_validates_as_pinned_0_1_1() -> None: + document = json.loads((FIXTURE_ROOT / "cases/logical-0.1.1-valid/document.json").read_text()) + document["version"] = "0.1.0" + + without_context = validation.validate_ossie_schema(document) + wrong_consumer = validation.validate_ossie_schema( + document, + consumer_profile=OssieConsumerProfile.OSSIE_CORE, + ) + by_consumer = validation.validate_ossie_schema( + document, + consumer_profile=OssieConsumerProfile.DBT_1_12, + ) + by_profile = validation.validate_ossie_schema(document, profile=DBT_1_12_0_1_0_ALIAS) + + assert without_context.diagnostics[0].code == "ossie.schema.profile_context_required" + assert wrong_consumer.diagnostics[0].code == "ossie.schema.profile_context_mismatch" + assert by_consumer.valid + assert by_profile.valid + assert by_profile.profile == "logical-0.1.1" + assert document["version"] == "0.1.0" + + +@pytest.mark.parametrize("consumer", [OssieConsumerProfile.OSSIE_CORE, OssieConsumerProfile.DBT_1_12]) +def test_ordinary_0_1_1_profiles_remain_valid(consumer: OssieConsumerProfile) -> None: + document = json.loads((FIXTURE_ROOT / "cases/logical-0.1.1-valid/document.json").read_text()) + + result = validation.validate_ossie_schema(document, consumer_profile=consumer) + + assert result.valid + assert result.profile == "logical-0.1.1" + + +@pytest.fixture +def temporary_schema_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + schema_root = tmp_path / "schemas" + shutil.copytree(SCHEMA_ROOT, schema_root) + monkeypatch.setattr(validation, "_schema_root", lambda: schema_root) + validation._manifest.cache_clear() + validation.available_schema_profiles.cache_clear() + validation._load_schema.cache_clear() + yield schema_root + validation._manifest.cache_clear() + validation.available_schema_profiles.cache_clear() + validation._load_schema.cache_clear() + + +@pytest.mark.parametrize( + ("asset_path", "expected_code"), + [ + ("ontology/0.2.0.dev0/schema.json", "ossie.schema.runtime_asset_integrity"), + ("ontology/0.2.0.dev0/upstream.json", "ossie.schema.upstream_asset_integrity"), + ], +) +def test_tampered_bundle_assets_fail_with_structured_diagnostics( + temporary_schema_root: Path, + asset_path: str, + expected_code: str, +) -> None: + target = temporary_schema_root / asset_path + target.write_bytes(target.read_bytes() + b"\n") + document = _load_case( + next(case for case in _load_manifest()["cases"] if case["id"] == "ontology-0.2.0.dev0-offline-refs") + ) + + result = validation.validate_ossie_schema(document, profile="ontology-0.2.0.dev0") + + assert not result.valid + assert result.failure_stage == "integrity" + assert result.diagnostics[0].code == expected_code + + +def test_declared_transformations_must_reproduce_runtime_asset(temporary_schema_root: Path) -> None: + manifest_path = temporary_schema_root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["profiles"]["ontology-0.2.0.dev0"]["transformations"][1]["to"] = "urn:tampered" + manifest_path.write_text(json.dumps(manifest)) + validation._manifest.cache_clear() + validation.available_schema_profiles.cache_clear() + document = _load_case( + next(case for case in _load_manifest()["cases"] if case["id"] == "ontology-0.2.0.dev0-offline-refs") + ) + + result = validation.validate_ossie_schema(document, profile="ontology-0.2.0.dev0") + + assert result.diagnostics[0].code == "ossie.schema.transformation_integrity" diff --git a/tests/interchange/ossie/test_semantic_validation.py b/tests/interchange/ossie/test_semantic_validation.py new file mode 100644 index 00000000..5e974bc3 --- /dev/null +++ b/tests/interchange/ossie/test_semantic_validation.py @@ -0,0 +1,754 @@ +from __future__ import annotations + +import json +from dataclasses import FrozenInstanceError + +import pytest + +from sidemantic.interchange.ossie import ( + OssieDocumentSource, + OssieLogicalDocument, + OssieOntologyDocument, + SemanticValidationResult, + validate_ossie_semantics, +) +from sidemantic.interchange.ossie.profiles import DBT_1_12_0_1_0_ALIAS +from sidemantic.interchange.ossie.validation import validate_ossie_schema + + +def _expression( + text: str = "id", + dialect: str = "ANSI_SQL", +) -> dict[str, object]: + return { + "dialects": [ + { + "dialect": dialect, + "expression": text, + } + ] + } + + +def _dataset( + name: str, + *, + field_names: tuple[str, ...] = ("id",), + primary_key: tuple[str, ...] | None = ("id",), + unique_keys: tuple[tuple[str, ...], ...] = (), +) -> dict[str, object]: + dataset: dict[str, object] = { + "name": name, + "source": name, + "fields": [ + { + "name": field_name, + "expression": _expression(field_name), + } + for field_name in field_names + ], + } + if primary_key is not None: + dataset["primary_key"] = list(primary_key) + if unique_keys: + dataset["unique_keys"] = [list(key) for key in unique_keys] + return dataset + + +def _logical_document(*semantic_models: dict[str, object]) -> dict[str, object]: + return { + "version": "0.2.0.dev0", + "semantic_model": list(semantic_models), + } + + +def _semantic_model( + name: str, + *, + datasets: list[dict[str, object]] | None = None, + relationships: list[dict[str, object]] | None = None, + metrics: list[dict[str, object]] | None = None, +) -> dict[str, object]: + model: dict[str, object] = { + "name": name, + "datasets": datasets or [_dataset("orders")], + } + if relationships is not None: + model["relationships"] = relationships + if metrics is not None: + model["metrics"] = metrics + return model + + +def _contracts(result: SemanticValidationResult) -> list[tuple[str, str, str | None]]: + return [(diagnostic.code, diagnostic.json_pointer, diagnostic.scope) for diagnostic in result.diagnostics] + + +@pytest.mark.parametrize( + ("version", "schema_profile"), + [ + ("0.1.1", "logical-0.1.1"), + ("0.2.0.dev0", "logical-0.2.0.dev0"), + ], +) +def test_schema_valid_document_passes_the_semantic_stage( + version: str, + schema_profile: str, +) -> None: + document = _logical_document( + _semantic_model( + "commerce", + datasets=[ + _dataset("orders", field_names=("id", "customer_id")), + _dataset("customers"), + ], + relationships=[ + { + "name": "order_customer", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["id"], + } + ], + metrics=[{"name": "order_count", "expression": _expression("COUNT(*)")}], + ) + ) + document["version"] = version + + schema_result = validate_ossie_schema(document, profile=schema_profile) + semantic_result = validate_ossie_semantics(document) + + assert schema_result.valid + assert semantic_result.valid + assert semantic_result.stage == "semantic" + assert semantic_result.failure_stage is None + assert semantic_result.document_kind == "logical" + assert semantic_result.checked_scopes == ("commerce",) + assert semantic_result.diagnostics == () + + +def test_duplicate_names_are_checked_in_their_own_namespaces() -> None: + orders = _dataset("orders", field_names=("id",)) + orders["fields"].append({"name": "id", "expression": _expression("other_id")}) + model = _semantic_model( + "commerce", + datasets=[orders, _dataset("orders")], + relationships=[ + { + "name": "same_edge", + "from": "orders", + "to": "orders", + "from_columns": ["id"], + "to_columns": ["id"], + }, + { + "name": "same_edge", + "from": "orders", + "to": "orders", + "from_columns": ["id"], + "to_columns": ["id"], + }, + ], + metrics=[ + {"name": "revenue", "expression": _expression()}, + {"name": "revenue", "expression": _expression()}, + ], + ) + result = validate_ossie_semantics(_logical_document(model, model.copy())) + + assert not result.valid + assert result.failure_stage == "semantic" + assert {code for code, _, _ in _contracts(result)} >= { + "ossie.semantic.semantic_model.name_duplicate", + "ossie.semantic.dataset.name_duplicate", + "ossie.semantic.field.name_duplicate", + "ossie.semantic.metric.name_duplicate", + "ossie.semantic.relationship.name_duplicate", + } + assert ( + "ossie.semantic.semantic_model.name_duplicate", + "/semantic_model/1/name", + None, + ) in _contracts(result) + assert ( + "ossie.semantic.field.name_duplicate", + "/semantic_model/0/datasets/0/fields/1/name", + "commerce@0", + ) in _contracts(result) + + +def test_duplicate_dataset_names_are_allowed_across_scopes() -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model("commerce", datasets=[_dataset("orders")]), + _semantic_model("operations", datasets=[_dataset("orders")]), + ) + ) + + assert result.valid + assert result.checked_scopes == ("commerce", "operations") + + +def test_names_are_independent_across_dataset_metric_relationship_and_field_namespaces() -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[ + _dataset("orders", field_names=("id", "customers")), + _dataset("customers", field_names=("id",)), + ], + metrics=[{"name": "customers", "expression": _expression("COUNT(*)")}], + relationships=[ + { + "name": "customers", + "from": "orders", + "to": "customers", + "from_columns": ["customers"], + "to_columns": ["id"], + } + ], + ) + ) + ) + + assert result.valid + + +def test_relationship_references_do_not_cross_scope_boundaries() -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[_dataset("orders", field_names=("id", "customer_id"))], + relationships=[ + { + "name": "order_customer", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["id"], + } + ], + ), + _semantic_model("crm", datasets=[_dataset("customers")]), + ) + ) + + assert _contracts(result) == [ + ( + "ossie.semantic.relationship.to_dataset_unknown", + "/semantic_model/0/relationships/0/to", + "commerce", + ) + ] + + +def test_regular_identifier_duplicates_are_detected_after_case_normalization() -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[_dataset("orders"), _dataset("Orders")], + metrics=[ + {"name": "revenue", "expression": _expression("1")}, + {"name": "Revenue", "expression": _expression("2")}, + ], + ) + ) + ) + + assert _contracts(result) == [ + ( + "ossie.semantic.dataset.name_duplicate", + "/semantic_model/0/datasets/1/name", + "commerce", + ), + ( + "ossie.semantic.metric.name_duplicate", + "/semantic_model/0/metrics/1/name", + "commerce", + ), + ] + + +def test_quoted_identifiers_use_exact_case_when_resolving_relationships() -> None: + model = _semantic_model( + "commerce", + datasets=[ + _dataset("orders", field_names=("id", "customer_id")), + _dataset("customers", field_names=("id",)), + ], + relationships=[ + { + "name": "exact", + "from": '"ORDERS"', + "to": '"CUSTOMERS"', + "from_columns": ['"CUSTOMER_ID"'], + "to_columns": ['"ID"'], + }, + { + "name": "case_mismatch", + "from": "orders", + "to": '"customers"', + "from_columns": ["customer_id"], + "to_columns": ['"id"'], + }, + ], + ) + + result = validate_ossie_semantics(_logical_document(model)) + + assert _contracts(result) == [ + ( + "ossie.semantic.relationship.to_dataset_unknown", + "/semantic_model/0/relationships/1/to", + "commerce", + ) + ] + + +def test_identifier_over_128_characters_has_structured_refusal() -> None: + too_long = "x" * 129 + result = validate_ossie_semantics(_logical_document(_semantic_model("commerce", datasets=[_dataset(too_long)]))) + + assert _contracts(result) == [ + ( + "ossie.semantic.identifier.length_exceeded", + "/semantic_model/0/datasets/0/name", + "commerce", + ) + ] + assert "129 characters" in result.diagnostics[0].message + + +@pytest.mark.parametrize( + ("relationship", "expected"), + [ + ( + { + "name": "missing_to_keys", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + }, + [ + ( + "ossie.semantic.relationship.keys_incomplete", + "/semantic_model/0/relationships/0", + "commerce", + ) + ], + ), + ( + { + "name": "empty_keys", + "from": "orders", + "to": "customers", + "from_columns": [], + "to_columns": [], + }, + [ + ( + "ossie.semantic.relationship.keys_empty", + "/semantic_model/0/relationships/0/from_columns", + "commerce", + ), + ( + "ossie.semantic.relationship.keys_empty", + "/semantic_model/0/relationships/0/to_columns", + "commerce", + ), + ], + ), + ( + { + "name": "different_arity", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id", "region_id"], + "to_columns": ["id"], + }, + [ + ( + "ossie.semantic.relationship.key_arity_mismatch", + "/semantic_model/0/relationships/0", + "commerce", + ) + ], + ), + ], +) +def test_relationship_key_arrays_are_executable_together( + relationship: dict[str, object], + expected: list[tuple[str, str, str | None]], +) -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[ + _dataset("orders", field_names=("id", "customer_id", "region_id")), + _dataset("customers"), + ], + relationships=[relationship], + ) + ) + ) + + assert _contracts(result) == expected + + +def test_relationship_keys_must_resolve_to_fields_and_a_unique_target_tuple() -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[ + _dataset("orders", field_names=("id", "customer_id")), + _dataset( + "customers", + field_names=("id", "email"), + primary_key=("id",), + ), + ], + relationships=[ + { + "name": "unknown_fields", + "from": "orders", + "to": "customers", + "from_columns": ["missing_customer_id"], + "to_columns": ["missing_id"], + }, + { + "name": "non_unique_target", + "from": "orders", + "to": "customers", + "from_columns": ["customer_id"], + "to_columns": ["email"], + }, + ], + ) + ) + ) + + assert _contracts(result) == [ + ( + "ossie.semantic.relationship.from_key_field_unknown", + "/semantic_model/0/relationships/0/from_columns/0", + "commerce", + ), + ( + "ossie.semantic.relationship.to_key_field_unknown", + "/semantic_model/0/relationships/0/to_columns/0", + "commerce", + ), + ( + "ossie.semantic.relationship.target_key_not_unique", + "/semantic_model/0/relationships/1/to_columns", + "commerce", + ), + ] + + +def test_declared_dataset_keys_resolve_and_are_unique_after_normalization() -> None: + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[ + _dataset( + "orders", + field_names=("id", "region_id"), + primary_key=["ID", "missing"], + unique_keys=[["id", "ID"], ["region_id"], ["REGION_ID"]], + ) + ], + ) + ) + ) + + assert _contracts(result) == [ + ( + "ossie.semantic.dataset.key_field_unknown", + "/semantic_model/0/datasets/0/primary_key/1", + "commerce", + ), + ( + "ossie.semantic.dataset.key_column_duplicate", + "/semantic_model/0/datasets/0/unique_keys/0/1", + "commerce", + ), + ( + "ossie.semantic.dataset.key_duplicate", + "/semantic_model/0/datasets/0/unique_keys/2", + "commerce", + ), + ] + + +def test_primary_and_unique_composite_target_keys_are_safe() -> None: + customers = _dataset( + "customers", + field_names=("id", "tenant_id", "external_id"), + primary_key=("id",), + unique_keys=(("tenant_id", "external_id"),), + ) + orders = _dataset( + "orders", + field_names=("id", "tenant_id", "customer_external_id"), + ) + result = validate_ossie_semantics( + _logical_document( + _semantic_model( + "commerce", + datasets=[orders, customers], + relationships=[ + { + "name": "order_customer", + "from": "orders", + "to": "customers", + "from_columns": ["tenant_id", "customer_external_id"], + "to_columns": ["tenant_id", "external_id"], + } + ], + ) + ) + ) + + assert result.valid + + +def test_expression_variants_are_non_empty_and_unique_per_expression() -> None: + fields = [ + { + "name": "empty_variants", + "expression": {"dialects": []}, + }, + { + "name": "bad_variants", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": " "}, + {"dialect": "ansi_sql", "expression": "id"}, + {"dialect": " ", "expression": "id"}, + ] + }, + }, + ] + dataset = _dataset("orders", primary_key=None) + dataset["fields"] = fields + + result = validate_ossie_semantics(_logical_document(_semantic_model("commerce", datasets=[dataset]))) + + assert _contracts(result) == [ + ( + "ossie.semantic.expression.dialects_empty", + "/semantic_model/0/datasets/0/fields/0/expression/dialects", + "commerce", + ), + ( + "ossie.semantic.expression.text_empty", + "/semantic_model/0/datasets/0/fields/1/expression/dialects/0/expression", + "commerce", + ), + ( + "ossie.semantic.expression.dialect_duplicate", + "/semantic_model/0/datasets/0/fields/1/expression/dialects/1/dialect", + "commerce", + ), + ( + "ossie.semantic.expression.dialect_empty", + "/semantic_model/0/datasets/0/fields/1/expression/dialects/2/dialect", + "commerce", + ), + ] + + +def test_logical_document_input_carries_source_and_profile_into_diagnostics() -> None: + document = OssieLogicalDocument( + canonical_data=_logical_document( + _semantic_model( + "commerce", + datasets=[_dataset("orders")], + relationships=[ + { + "name": "missing_customer", + "from": "orders", + "to": "customers", + "from_columns": ["id"], + "to_columns": ["id"], + } + ], + ) + ), + serialization="yaml", + source=OssieDocumentSource(identifier="models/commerce.ossie.yaml"), + ) + + result = validate_ossie_semantics(document) + diagnostic = result.diagnostics[0] + + assert diagnostic.source is not None + assert diagnostic.source.identifier == "models/commerce.ossie.yaml" + assert diagnostic.profile is not None + assert diagnostic.profile.identifier == "ossie-core:0.2.0.dev0" + + +def test_explicit_consumer_profile_carries_into_semantic_diagnostics() -> None: + document = _logical_document( + _semantic_model( + "commerce", + relationships=[ + { + "name": "missing_customer", + "from": "orders", + "to": "customers", + "from_columns": ["id"], + "to_columns": ["id"], + } + ], + ) + ) + document["version"] = "0.1.0" + + result = validate_ossie_semantics(document, profile=DBT_1_12_0_1_0_ALIAS) + + assert result.diagnostics[0].profile is DBT_1_12_0_1_0_ALIAS + + +def test_ontology_validation_checks_explicit_mapping_references_only() -> None: + ontology = OssieOntologyDocument( + canonical_data={ + "version": "0.2.0.dev0", + "name": "commerce-ontology", + "ontology": [ + { + "concept": "Order", + "type": "EntityType", + } + ], + "ontology_mappings": [ + { + "name": "commerce-map", + "semantic_model": _semantic_model("commerce"), + "concept_mappings": [ + { + "concept": "MissingConcept", + "object_mappings": [{"concept": "Order"}], + "link_mappings": [ + { + "object_mapping": {"concept": "AlsoMissing"}, + "children": [{"object_mapping": {"concept": "Order"}}], + } + ], + } + ], + } + ], + }, + serialization="json", + ) + + result = validate_ossie_semantics(ontology) + + assert result.document_kind == "ontology" + assert result.checked_scopes == ("commerce",) + assert _contracts(result) == [ + ( + "ossie.semantic.ontology.concept_unknown", + "/ontology_mappings/0/concept_mappings/0/concept", + None, + ), + ( + "ossie.semantic.ontology.concept_unknown", + "/ontology_mappings/0/concept_mappings/0/link_mappings/0/object_mapping/concept", + None, + ), + ] + assert all("reason" not in diagnostic.message.lower() for diagnostic in result.diagnostics) + + +def test_ontology_embedded_models_are_isolated_logical_scopes() -> None: + ontology = { + "version": "0.2.0.dev0", + "name": "commerce-ontology", + "ontology": [{"concept": "Order", "type": "EntityType"}], + "ontology_mappings": [ + { + "semantic_model": _semantic_model( + "orders-scope", + datasets=[_dataset("orders")], + relationships=[ + { + "name": "cross_scope", + "from": "orders", + "to": "customers", + "from_columns": ["id"], + "to_columns": ["id"], + } + ], + ), + "concept_mappings": [], + }, + { + "semantic_model": _semantic_model( + "customers-scope", + datasets=[_dataset("customers")], + ), + "concept_mappings": [], + }, + ], + } + + result = validate_ossie_semantics(ontology) + + assert result.checked_scopes == ("orders-scope", "customers-scope") + assert _contracts(result) == [ + ( + "ossie.semantic.relationship.to_dataset_unknown", + "/ontology_mappings/0/semantic_model/relationships/0/to", + "orders-scope", + ) + ] + + +def test_result_is_deterministic_immutable_and_serialization_friendly() -> None: + document = _logical_document( + _semantic_model( + "commerce", + datasets=[_dataset("orders")], + relationships=[ + { + "name": "broken", + "from": "missing", + "to": "also_missing", + "from_columns": [], + "to_columns": [], + } + ], + ) + ) + + first = validate_ossie_semantics(document) + second = validate_ossie_semantics(document) + + assert first == second + assert json.loads(json.dumps(first.to_dict()))["stage"] == "semantic" + with pytest.raises(FrozenInstanceError): + first.valid = True + + +def test_unsupported_mapping_has_no_semantic_claims() -> None: + result = validate_ossie_semantics({"version": "0.2.0.dev0"}) + + assert result.valid + assert result.document_kind == "unsupported" + assert result.checked_scopes == () + assert result.diagnostics == () + + +def test_non_mapping_input_is_rejected() -> None: + with pytest.raises(TypeError, match="parsed mapping"): + validate_ossie_semantics(["not", "a", "mapping"]) diff --git a/tests/interchange/ossie/test_serialization.py b/tests/interchange/ossie/test_serialization.py new file mode 100644 index 00000000..4f746f51 --- /dev/null +++ b/tests/interchange/ossie/test_serialization.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import json +from dataclasses import FrozenInstanceError + +import pytest +import yaml + +from sidemantic.interchange.ossie import ( + DBT_1_12_0_1_0_ALIAS, + OssieConsumerProfile, + OssieDocumentSource, + OssieLogicalDocument, + OssieOntologyDocument, + OssieParseOptions, + OssiePreservationPolicy, + OssieSerialization, + OssieSerializationError, + UnsupportedOssieDocument, + parse_ossie_document, + serialize_ossie_document, +) + +LOGICAL_DATA = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "commerce", + "datasets": [{"name": "orders", "source": "analytics.orders"}], + } + ], +} + +ONTOLOGY_DATA = { + "version": "0.2.0.dev0", + "name": "commerce-ontology", + "ai_context": {"instructions": "Use café concepts consistently."}, + "ontology": [{"concept": "Customer", "type": "EntityType"}], + "ontology_mappings": [ + { + "semantic_model": { + "name": "commerce", + "datasets": [{"name": "customers", "source": "analytics.customers"}], + }, + "concept_mappings": [], + } + ], +} + + +def _logical_document(*, source: OssieDocumentSource | None = None) -> OssieLogicalDocument: + return OssieLogicalDocument( + canonical_data=LOGICAL_DATA, + serialization=OssieSerialization.YAML, + source=source, + ) + + +def _diagnostic_codes(error: OssieSerializationError) -> list[str]: + return [diagnostic.code for diagnostic in error.diagnostics] + + +def test_explicit_exact_source_mode_reuses_equivalent_original_bytes() -> None: + original = b"# exact comment\nversion: 0.2.0.dev0\nsemantic_model:\n- datasets:\n - source: analytics.orders\n name: orders\n name: commerce\n" + parsed = parse_ossie_document( + original, + source_identifier="commerce.ossie.yaml", + options=OssieParseOptions(preservation_policy=OssiePreservationPolicy.SOURCE_BYTES), + ) + + result = serialize_ossie_document(parsed.document, "yaml", exact_source=True) + + assert result.data == original + assert result.exact_source_reused + assert result.serialization is OssieSerialization.YAML + assert result.diagnostics == () + with pytest.raises(FrozenInstanceError): + result.data = b"changed" + + +def test_cross_serialization_is_canonical_and_does_not_reuse_source() -> None: + source = OssieDocumentSource(original_bytes=b"version: 0.2.0.dev0\nsemantic_model: []\n") + result = serialize_ossie_document(_logical_document(source=source), OssieSerialization.JSON, exact_source=True) + + assert json.loads(result.data) == LOGICAL_DATA + assert result.data.endswith(b"\n") + assert not result.exact_source_reused + + +def test_canonical_json_and_yaml_are_deterministic_unicode_safe_and_tag_free() -> None: + document = OssieOntologyDocument(canonical_data=ONTOLOGY_DATA, serialization="json") + + json_results = [serialize_ossie_document(document, "json").data for _ in range(2)] + yaml_results = [serialize_ossie_document(document, "yaml").data for _ in range(2)] + + assert json_results[0] == json_results[1] + assert yaml_results[0] == yaml_results[1] + assert json_results[0].endswith(b"\n") + assert yaml_results[0].endswith(b"\n") + assert "café" in json_results[0].decode() + assert "café" in yaml_results[0].decode() + assert b"!!python" not in yaml_results[0] + assert json.loads(json_results[0]) == ONTOLOGY_DATA + assert yaml.safe_load(yaml_results[0]) == ONTOLOGY_DATA + + +def test_schema_invalid_document_is_refused_with_structured_diagnostics() -> None: + document = OssieLogicalDocument( + canonical_data={"version": "0.2.0.dev0", "semantic_model": {}}, + serialization="json", + ) + + with pytest.raises(OssieSerializationError) as raised: + serialize_ossie_document(document, "json") + + assert "ossie.schema.type" in _diagnostic_codes(raised.value) + assert raised.value.diagnostics[0].json_pointer == "/semantic_model" + assert raised.value.diagnostics[0].schema is not None + + +@pytest.mark.parametrize( + ("document", "expected_code"), + [ + ( + OssieLogicalDocument( + canonical_data={"version": "9.9", "semantic_model": []}, + serialization="json", + ), + "ossie.schema.profile_unknown", + ), + ( + UnsupportedOssieDocument( + canonical_data={"version": "0.2.0.dev0", "name": "unknown"}, + serialization="json", + reason="document family is unknown", + ), + "ossie.serialization.unsupported_document", + ), + ], +) +def test_unsupported_versions_and_documents_are_refused(document, expected_code) -> None: + with pytest.raises(OssieSerializationError) as raised: + serialize_ossie_document(document, "yaml") + + assert _diagnostic_codes(raised.value) == [expected_code] + + +def test_stale_retained_source_is_not_blindly_reused() -> None: + stale_source = b"version: 0.2.0.dev0\nsemantic_model: []\n" + document = _logical_document(source=OssieDocumentSource(original_bytes=stale_source)) + + result = serialize_ossie_document(document, "yaml", exact_source=True) + + assert not result.exact_source_reused + assert result.data != stale_source + assert yaml.safe_load(result.data) == LOGICAL_DATA + assert [diagnostic.code for diagnostic in result.diagnostics] == ["ossie.serialization.exact_source_mismatch"] + + +def test_dbt_alias_exact_and_canonical_json_retain_0_1_0() -> None: + original = b'{"version":"0.1.0","semantic_model":[]}\n' + document = OssieLogicalDocument( + canonical_data=json.loads(original), + serialization="json", + source=OssieDocumentSource(identifier="dbt.json", original_bytes=original), + ) + + exact = serialize_ossie_document(document, "json", exact_source=True, profile=DBT_1_12_0_1_0_ALIAS) + canonical = serialize_ossie_document( + document, + "json", + consumer_profile=OssieConsumerProfile.DBT_1_12, + ) + + assert exact.data == original + assert exact.exact_source_reused + assert json.loads(canonical.data)["version"] == "0.1.0" + assert b'"version": "0.1.0"' in canonical.data + + +def test_dbt_alias_serialization_rejects_missing_or_wrong_context() -> None: + document = OssieLogicalDocument( + canonical_data={"version": "0.1.0", "semantic_model": []}, + serialization="json", + ) + + with pytest.raises(OssieSerializationError) as missing: + serialize_ossie_document(document, "json") + with pytest.raises(OssieSerializationError) as wrong: + serialize_ossie_document( + document, + "json", + profile=DBT_1_12_0_1_0_ALIAS, + consumer_profile=OssieConsumerProfile.OSSIE_CORE, + ) + + assert _diagnostic_codes(missing.value) == ["ossie.schema.profile_context_required"] + assert _diagnostic_codes(wrong.value) == ["ossie.schema.profile_context_mismatch"] diff --git a/tests/interchange/ossie/test_synthesis.py b/tests/interchange/ossie/test_synthesis.py new file mode 100644 index 00000000..61a4767e --- /dev/null +++ b/tests/interchange/ossie/test_synthesis.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import pytest + +from sidemantic.core.dimension import Dimension +from sidemantic.core.metric import Metric +from sidemantic.core.model import Model +from sidemantic.core.relationship import Relationship +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.interchange.ossie import ( + OssieConsumerProfile, + OssieSynthesisError, + require_synthesized_document, + synthesize_ossie_document, +) + + +def _graph() -> SemanticGraph: + graph = SemanticGraph() + orders = Model( + name="orders", + table="analytics.orders", + primary_key="id", + dimensions=[ + Dimension(name="id", type="numeric", sql="id", logical_data_type="Integer"), + Dimension(name="customer_id", type="numeric", sql="customer_id", logical_data_type="Integer"), + Dimension( + name="loaded_at", + type="categorical", + sql="loaded_at", + logical_data_type="DateTime", + declared_is_time=False, + ), + ], + metrics=[Metric(name="revenue", agg="sum", sql="amount", logical_data_type="Decimal")], + ) + customers = Model( + name="customers", + table="analytics.customers", + primary_key="id", + dimensions=[Dimension(name="id", type="numeric", sql="id", logical_data_type="Integer")], + ) + orders.relationships.append( + Relationship( + name="customers", + edge_id="order_customer", + type="many_to_one", + foreign_key="customer_id", + primary_key="id", + ) + ) + graph.add_model(orders) + graph.add_model(customers) + return graph + + +def test_synthesis_requires_explicit_scope_and_expression_dialect() -> None: + graph = _graph() + + with pytest.raises(ValueError, match="scope_name"): + synthesize_ossie_document(graph, scope_name="", expression_dialect="BIGQUERY") + with pytest.raises(ValueError, match="supported"): + synthesize_ossie_document(graph, scope_name="commerce", expression_dialect="duckdb") + + +def test_synthesis_preserves_types_time_false_keys_and_edge_identity() -> None: + result = synthesize_ossie_document(_graph(), scope_name="commerce", expression_dialect="BIGQUERY") + + assert result.valid + data = result.document.to_parsed_data() + semantic_model = data["semantic_model"][0] + orders = semantic_model["datasets"][0] + loaded_at = orders["fields"][2] + relationship = semantic_model["relationships"][0] + metric = semantic_model["metrics"][0] + + assert orders["primary_key"] == ["id"] + assert loaded_at["datatype"] == "DateTime" + assert loaded_at["dimension"] == {"is_time": False} + assert relationship["name"] == "order_customer" + assert relationship["to_columns"] == ["id"] + assert metric["datatype"] == "Decimal" + assert metric["expression"]["dialects"] == [{"dialect": "BIGQUERY", "expression": "SUM(orders.amount)"}] + + +def test_dbt_alias_synthesis_requires_and_preserves_explicit_consumer_context() -> None: + graph = SemanticGraph() + graph.add_model(Model(name="orders", table="analytics.orders")) + rejected = synthesize_ossie_document( + graph, + scope_name="commerce", + expression_dialect="ANSI_SQL", + schema_version="0.1.0", + ) + accepted = synthesize_ossie_document( + graph, + scope_name="commerce", + expression_dialect="ANSI_SQL", + schema_version="0.1.0", + consumer_profile=OssieConsumerProfile.DBT_1_12, + ) + + assert not rejected.valid + assert [diagnostic.code for diagnostic in rejected.diagnostics] == ["ossie.synthesis.profile_unsupported"] + assert accepted.valid + assert accepted.document.version == "0.1.0" + + +def test_synthesis_refuses_to_invent_relationship_identity_or_keys() -> None: + graph = _graph() + graph.models["orders"].relationships = [ + Relationship(name="customers", type="many_to_one", foreign_key="customer_id", primary_key=None) + ] + + result = synthesize_ossie_document(graph, scope_name="commerce", expression_dialect="ANSI_SQL") + + assert not result.valid + assert result.document is None + assert {diagnostic.code for diagnostic in result.diagnostics} == {"ossie.synthesis.relationship_identity_missing"} + with pytest.raises(OssieSynthesisError): + require_synthesized_document(result) + + +def test_synthesis_refuses_models_without_sources_instead_of_emitting_invalid_output() -> None: + graph = SemanticGraph() + graph.add_model(Model(name="orders")) + + result = synthesize_ossie_document(graph, scope_name="commerce", expression_dialect="SNOWFLAKE") + + assert not result.valid + assert result.document is None + assert any(diagnostic.code == "ossie.synthesis.source_missing" for diagnostic in result.diagnostics) + assert any(diagnostic.code.startswith("ossie.schema") for diagnostic in result.diagnostics) + + +def test_synthesis_refuses_invalid_or_multiple_statement_expressions() -> None: + graph = SemanticGraph() + graph.add_model( + Model( + name="orders", + table="analytics.orders", + dimensions=[Dimension(name="unsafe", type="categorical", sql="id; DROP TABLE orders")], + ) + ) + + result = synthesize_ossie_document(graph, scope_name="commerce", expression_dialect="ANSI_SQL") + + assert not result.valid + assert result.document is None + assert any(diagnostic.code == "ossie.synthesis.expression_invalid" for diagnostic in result.diagnostics) + + +def test_synthesis_closes_over_declared_key_and_relationship_field_semantics() -> None: + graph = _graph() + graph.models["orders"].primary_key = "missing_id" + graph.models["orders"].relationships[0].foreign_key = "missing_customer_id" + + result = synthesize_ossie_document(graph, scope_name="commerce", expression_dialect="ANSI_SQL") + + assert not result.valid + assert result.document is None + assert {diagnostic.code for diagnostic in result.diagnostics} >= { + "ossie.semantic.dataset.key_field_unknown", + "ossie.semantic.relationship.from_key_field_unknown", + } + + +def test_synthesis_closes_over_relationship_endpoint_semantics() -> None: + graph = _graph() + graph.models["orders"].relationships[0].name = "missing_customers" + + result = synthesize_ossie_document(graph, scope_name="commerce", expression_dialect="ANSI_SQL") + + assert not result.valid + assert result.document is None + assert any( + diagnostic.code + in { + "ossie.synthesis.relationship_keys_unusable", + "ossie.semantic.relationship.to_dataset_unknown", + } + for diagnostic in result.diagnostics + ) diff --git a/tests/interchange/ossie/test_temporal_conformance.py b/tests/interchange/ossie/test_temporal_conformance.py new file mode 100644 index 00000000..f4ad760e --- /dev/null +++ b/tests/interchange/ossie/test_temporal_conformance.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import json + +from sidemantic.interchange.ossie import ( + OssieParseOptions, + lower_ossie_document, + parse_ossie_document, + serialize_ossie_document, + synthesize_ossie_document, +) + +DATA_TYPES = ("String", "Integer", "Decimal", "Float", "Boolean", "Date", "Time", "DateTime", "DateTimeTz", "Opaque") +TEMPORAL_TYPES = {"Date", "Time", "DateTime", "DateTimeTz"} +DECLARATIONS = (("omitted", None), ("true", True), ("false", False)) + + +def test_complete_datatype_and_time_role_matrix_survives_lowering_and_synthesis() -> None: + fields: list[dict[str, object]] = [] + expected: dict[str, tuple[str, str, bool | None]] = {} + for data_type in DATA_TYPES: + for declaration_name, declared_is_time in DECLARATIONS: + name = f"{data_type.lower()}_{declaration_name}" + field: dict[str, object] = { + "name": name, + "datatype": data_type, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": name}]}, + } + if declared_is_time is not None: + field["dimension"] = {"is_time": declared_is_time} + effective_is_time = declared_is_time if declared_is_time is not None else data_type in TEMPORAL_TYPES + if effective_is_time: + runtime_type = "time" + elif data_type == "Boolean": + runtime_type = "boolean" + elif data_type in {"Integer", "Decimal", "Float"}: + runtime_type = "numeric" + else: + runtime_type = "categorical" + expected[name] = (runtime_type, data_type, declared_is_time) + fields.append(field) + + source = { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "types", + "datasets": [{"name": "values", "source": "analytics.type_values", "fields": fields}], + } + ], + } + parsed = parse_ossie_document( + json.dumps(source).encode(), + options=OssieParseOptions(validate_schema=True), + ) + lowered = lower_ossie_document(parsed, target_dialect="duckdb") + + assert lowered.valid + model = lowered.catalog["types"].graph.get_model("values") + for dimension in model.dimensions: + assert (dimension.type, dimension.logical_data_type, dimension.declared_is_time) == expected[dimension.name] + + synthesized = synthesize_ossie_document( + lowered.catalog["types"].graph, + scope_name="types", + expression_dialect="ANSI_SQL", + ) + + assert synthesized.valid + synthesized_fields = { + field["name"]: field + for field in synthesized.document.to_parsed_data()["semantic_model"][0]["datasets"][0]["fields"] + } + for name, (_, data_type, declared_is_time) in expected.items(): + assert synthesized_fields[name]["datatype"] == data_type + if declared_is_time is None: + assert "dimension" not in synthesized_fields[name] + else: + assert synthesized_fields[name]["dimension"] == {"is_time": declared_is_time} + + canonical = serialize_ossie_document(synthesized.document, "json") + reparsed = parse_ossie_document(canonical.data, options=OssieParseOptions(validate_schema=True)) + assert reparsed.valid + assert reparsed.document.to_parsed_data() == synthesized.document.to_parsed_data() diff --git a/tests/interchange/ossie/test_upstream_validator_gate.py b/tests/interchange/ossie/test_upstream_validator_gate.py new file mode 100644 index 00000000..e2f10214 --- /dev/null +++ b/tests/interchange/ossie/test_upstream_validator_gate.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from sidemantic.core.dimension import Dimension +from sidemantic.core.metric import Metric +from sidemantic.core.model import Model +from sidemantic.core.semantic_graph import SemanticGraph +from sidemantic.interchange.ossie import ( + OssieSerialization, + require_synthesized_document, + serialize_ossie_document, + synthesize_ossie_document, +) + +_REPOSITORY_ROOT = Path(__file__).parents[3] +_UPSTREAM_ROOT = _REPOSITORY_ROOT / "tests" / "ossie-fixtures" / "upstream" +_VALIDATOR = _UPSTREAM_ROOT / "validation" / "validate.py" +_SCHEMAS = _UPSTREAM_ROOT / "schemas" / "logical" + + +def _graph(*, include_datatypes: bool) -> SemanticGraph: + graph = SemanticGraph() + graph.add_model( + Model( + name="orders", + table="analytics.orders", + primary_key="id", + dimensions=[ + Dimension( + name="id", + type="numeric", + sql="id", + logical_data_type="Integer" if include_datatypes else None, + ), + Dimension( + name="order_date", + type="time", + sql="order_date", + logical_data_type="Date" if include_datatypes else None, + ), + ], + metrics=[ + Metric( + name="revenue", + agg="sum", + sql="amount", + logical_data_type="Decimal" if include_datatypes else None, + ) + ], + ) + ) + return graph + + +def _run_pinned_validator(output_path: Path, *, schema_version: str) -> subprocess.CompletedProcess[str]: + schema_path = _SCHEMAS / schema_version / "schema.json" + environment = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"} + return subprocess.run( + [sys.executable, str(_VALIDATOR), str(output_path), "--schema", str(schema_path)], + cwd=_UPSTREAM_ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize( + ("schema_version", "serialization"), + [("0.1.1", OssieSerialization.JSON), ("0.2.0.dev0", OssieSerialization.YAML)], +) +def test_canonical_sidemantic_export_passes_exact_pinned_apache_validator( + tmp_path: Path, + schema_version: str, + serialization: OssieSerialization, +) -> None: + synthesis = synthesize_ossie_document( + _graph(include_datatypes=schema_version == "0.2.0.dev0"), + scope_name="commerce", + expression_dialect="ANSI_SQL", + schema_version=schema_version, + serialization=serialization, + ) + document = require_synthesized_document(synthesis) + output = tmp_path / f"canonical-{schema_version}.{serialization.value}" + serialized = serialize_ossie_document(document, serialization) + output.write_bytes(serialized.data) + + completed = _run_pinned_validator(output, schema_version=schema_version) + + assert completed.returncode == 0, completed.stdout + completed.stderr + assert f"Validation PASSED: {output.name}" in completed.stdout + + +def test_pinned_apache_validator_rejects_deliberately_invalid_input() -> None: + invalid = _UPSTREAM_ROOT / "cases" / "invalid-semantic-model.yaml" + + completed = _run_pinned_validator(invalid, schema_version="0.2.0.dev0") + + assert completed.returncode != 0 + combined = completed.stdout + completed.stderr + assert "Validation FAILED" in combined + assert "semantic_model" in combined + + +def test_gate_uses_local_validator_and_schema_paths_only(tmp_path: Path) -> None: + output = tmp_path / "empty.json" + output.write_text(json.dumps({"version": "0.1.1", "semantic_model": []}) + "\n") + + completed = _run_pinned_validator(output, schema_version="0.1.1") + + assert completed.returncode == 0, completed.stdout + completed.stderr + assert str(_VALIDATOR) not in completed.stdout + assert yaml.safe_load(output.read_text())["version"] == "0.1.1" diff --git a/tests/ossie-fixtures/cases/logical-0.1.0-invalid/document.json b/tests/ossie-fixtures/cases/logical-0.1.0-invalid/document.json new file mode 100644 index 00000000..6b490635 --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.1.0-invalid/document.json @@ -0,0 +1,14 @@ +{ + "version": "0.1.0", + "semantic_model": [ + { + "name": "commerce", + "datasets": [ + { + "name": "orders", + "source": "analytics.orders" + } + ] + } + ] +} diff --git a/tests/ossie-fixtures/cases/logical-0.1.1-valid/document.json b/tests/ossie-fixtures/cases/logical-0.1.1-valid/document.json new file mode 100644 index 00000000..0c2c9f4d --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.1.1-valid/document.json @@ -0,0 +1,14 @@ +{ + "version": "0.1.1", + "semantic_model": [ + { + "name": "commerce", + "datasets": [ + { + "name": "orders", + "source": "analytics.orders" + } + ] + } + ] +} diff --git a/tests/ossie-fixtures/cases/logical-0.1.1-valid/document.yaml b/tests/ossie-fixtures/cases/logical-0.1.1-valid/document.yaml new file mode 100644 index 00000000..a60c66be --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.1.1-valid/document.yaml @@ -0,0 +1,6 @@ +version: 0.1.1 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders diff --git a/tests/ossie-fixtures/cases/logical-0.2-additional-root-property/document.yaml b/tests/ossie-fixtures/cases/logical-0.2-additional-root-property/document.yaml new file mode 100644 index 00000000..6eb2827f --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.2-additional-root-property/document.yaml @@ -0,0 +1,7 @@ +version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders +unsupported: true diff --git a/tests/ossie-fixtures/cases/logical-0.2-dataset-missing-name/document.json b/tests/ossie-fixtures/cases/logical-0.2-dataset-missing-name/document.json new file mode 100644 index 00000000..5f128ae1 --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.2-dataset-missing-name/document.json @@ -0,0 +1,13 @@ +{ + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "commerce", + "datasets": [ + { + "source": "analytics.orders" + } + ] + } + ] +} diff --git a/tests/ossie-fixtures/cases/logical-0.2-semantic-model-wrong-type/document.yaml b/tests/ossie-fixtures/cases/logical-0.2-semantic-model-wrong-type/document.yaml new file mode 100644 index 00000000..e2b071ca --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.2-semantic-model-wrong-type/document.yaml @@ -0,0 +1,4 @@ +version: 0.2.0.dev0 +semantic_model: + name: commerce + datasets: [] diff --git a/tests/ossie-fixtures/cases/logical-0.2.0.dev0-valid/document.json b/tests/ossie-fixtures/cases/logical-0.2.0.dev0-valid/document.json new file mode 100644 index 00000000..c1c10aa8 --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.2.0.dev0-valid/document.json @@ -0,0 +1,14 @@ +{ + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "commerce", + "datasets": [ + { + "name": "orders", + "source": "analytics.orders" + } + ] + } + ] +} diff --git a/tests/ossie-fixtures/cases/logical-0.2.0.dev0-valid/document.yaml b/tests/ossie-fixtures/cases/logical-0.2.0.dev0-valid/document.yaml new file mode 100644 index 00000000..08ee5722 --- /dev/null +++ b/tests/ossie-fixtures/cases/logical-0.2.0.dev0-valid/document.yaml @@ -0,0 +1,6 @@ +version: 0.2.0.dev0 +semantic_model: + - name: commerce + datasets: + - name: orders + source: analytics.orders diff --git a/tests/ossie-fixtures/cases/ontology-0.2.0.dev0-valid/document.yaml b/tests/ossie-fixtures/cases/ontology-0.2.0.dev0-valid/document.yaml new file mode 100644 index 00000000..126695c4 --- /dev/null +++ b/tests/ossie-fixtures/cases/ontology-0.2.0.dev0-valid/document.yaml @@ -0,0 +1,14 @@ +version: 0.2.0.dev0 +name: commerce-ontology +ai_context: + instructions: Use customer concepts consistently. +ontology: + - concept: Customer + type: EntityType +ontology_mappings: + - semantic_model: + name: commerce + datasets: + - name: customers + source: analytics.customers + concept_mappings: [] diff --git a/tests/ossie-fixtures/manifest.yaml b/tests/ossie-fixtures/manifest.yaml new file mode 100644 index 00000000..8a26b476 --- /dev/null +++ b/tests/ossie-fixtures/manifest.yaml @@ -0,0 +1,91 @@ +manifest_version: 1 + +equivalence_groups: + logical-0.1.1: + - logical-0.1.1-yaml + - logical-0.1.1-json + logical-0.2.0.dev0: + - logical-0.2.0.dev0-yaml + - logical-0.2.0.dev0-json + +cases: + - id: logical-0.1.1-yaml + input: cases/logical-0.1.1-valid/document.yaml + serialization: yaml + profile: logical-0.1.1 + expected: + valid: true + diagnostics: [] + + - id: logical-0.1.1-json + input: cases/logical-0.1.1-valid/document.json + serialization: json + profile: logical-0.1.1 + expected: + valid: true + diagnostics: [] + + - id: logical-0.2.0.dev0-yaml + input: cases/logical-0.2.0.dev0-valid/document.yaml + serialization: yaml + profile: logical-0.2.0.dev0 + expected: + valid: true + diagnostics: [] + + - id: logical-0.2.0.dev0-json + input: cases/logical-0.2.0.dev0-valid/document.json + serialization: json + profile: logical-0.2.0.dev0 + expected: + valid: true + diagnostics: [] + + - id: logical-0.1.0-not-upstream-valid + input: cases/logical-0.1.0-invalid/document.json + serialization: json + profile: logical-0.1.1 + expected: + valid: false + profile: null + diagnostics: + - code: ossie.schema.profile_context_required + instance_path: /version + + - id: logical-0.2-semantic-model-wrong-type + input: cases/logical-0.2-semantic-model-wrong-type/document.yaml + serialization: yaml + profile: logical-0.2.0.dev0 + expected: + valid: false + diagnostics: + - code: ossie.schema.type + instance_path: /semantic_model + + - id: logical-0.2-dataset-missing-name + input: cases/logical-0.2-dataset-missing-name/document.json + serialization: json + profile: logical-0.2.0.dev0 + expected: + valid: false + diagnostics: + - code: ossie.schema.required + instance_path: /semantic_model/0/datasets/0 + + - id: logical-0.2-additional-root-property + input: cases/logical-0.2-additional-root-property/document.yaml + serialization: yaml + profile: logical-0.2.0.dev0 + expected: + valid: false + diagnostics: + - code: ossie.schema.additional_properties + instance_path: "" + + - id: ontology-0.2.0.dev0-offline-refs + input: cases/ontology-0.2.0.dev0-valid/document.yaml + serialization: yaml + profile: ontology-0.2.0.dev0 + expected: + valid: true + diagnostics: [] diff --git a/tests/ossie-fixtures/upstream/README.md b/tests/ossie-fixtures/upstream/README.md new file mode 100644 index 00000000..880b264b --- /dev/null +++ b/tests/ossie-fixtures/upstream/README.md @@ -0,0 +1,28 @@ +# Offline Apache Ossie validator gate + +This directory is test-only and deliberately contains no network-dependent +test step. + +`validation/validate.py` is an unmodified vendored copy of the Apache Ossie +validator at commit +`88e0011148283302c9a04cd0287e00e0b9d87354`: + +https://github.com/apache/ossie/blob/88e0011148283302c9a04cd0287e00e0b9d87354/validation/validate.py + +Its Apache License 2.0 header is preserved in the vendored file. The +`0.2.0.dev0` schema is the matching Apache Ossie `core-spec/osi-schema.json` +from that same commit. The validator gate always passes that schema through +the CLI's explicit `--schema` option. + +The Apache snapshot contains the draft `0.2.0.dev0` core schema, but not the +released `0.1.1` schema. The `0.1.1` schema here is the released upstream OSI +schema from legacy repository commit +`2af09b20b8ff5641c3780a9940dc0c94249ae1b3` (blob +`30210d18eb47a1ccb67a036557bf9730842d8c93`), retained so the exact pinned +validator can also validate the supported released JSON profile. It is not +presented as a schema from the `88e00111` Apache commit. The dbt `0.1.0` +compatibility alias is intentionally not covered by this gate. + +The gate invokes the vendored script as a subprocess with only these local +validator, schema, and generated-output paths. It checks both successful +canonical Sidemantic exports and a deliberately invalid input. diff --git a/tests/ossie-fixtures/upstream/cases/invalid-semantic-model.yaml b/tests/ossie-fixtures/upstream/cases/invalid-semantic-model.yaml new file mode 100644 index 00000000..a138c9dd --- /dev/null +++ b/tests/ossie-fixtures/upstream/cases/invalid-semantic-model.yaml @@ -0,0 +1,2 @@ +version: 0.2.0.dev0 +semantic_model: {} diff --git a/tests/ossie-fixtures/upstream/schemas/logical/0.1.1/schema.json b/tests/ossie-fixtures/upstream/schemas/logical/0.1.1/schema.json new file mode 100644 index 00000000..30210d18 --- /dev/null +++ b/tests/ossie-fixtures/upstream/schemas/logical/0.1.1/schema.json @@ -0,0 +1,344 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", + "title": "OSI Core Metadata Specification", + "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.1.1", + "description": "OSI specification version" + }, + "dialects": { + "type": "array", + "description": "Supported expression language dialects (enumeration definition)", + "items": { + "$ref": "#/$defs/Dialect" + } + }, + "vendors": { + "type": "array", + "description": "Supported vendors for custom extensions (enumeration definition)", + "items": { + "$ref": "#/$defs/Vendor" + } + }, + "semantic_model": { + "type": "array", + "description": "Collection of semantic model definitions", + "items": { + "$ref": "#/$defs/SemanticModel" + } + } + }, + "required": ["version", "semantic_model"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS"], + "description": "Supported SQL and expression language dialects" + }, + "Vendor": { + "type": "string", + "enum": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS"], + "description": "Supported vendors for custom extensions" + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "CustomExtension": { + "type": "object", + "description": "Vendor-specific attributes for extensibility", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "string", + "description": "JSON string containing vendor-specific data" + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "required": ["dialects"], + "additionalProperties": false + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Indicates if this is a time-based dimension for temporal filtering" + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Primary key columns (single or composite)" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": "Array of unique key definitions (each can be single or composite)" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Foreign key columns in the 'from' dataset" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Primary/unique key columns in the 'to' dataset" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Quantifiable measures spanning datasets" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/tests/ossie-fixtures/upstream/schemas/logical/0.2.0.dev0/schema.json b/tests/ossie-fixtures/upstream/schemas/logical/0.2.0.dev0/schema.json new file mode 100644 index 00000000..f24e45f1 --- /dev/null +++ b/tests/ossie-fixtures/upstream/schemas/logical/0.2.0.dev0/schema.json @@ -0,0 +1,352 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/core-spec/osi-schema.json", + "title": "Apache Ossie Core Metadata Specification", + "description": "JSON Schema for validating Apache Ossie semantic model definitions", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.2.0.dev0", + "description": "Apache Ossie specification version" + }, + "semantic_model": { + "type": "array", + "description": "Collection of semantic model definitions", + "items": { + "$ref": "#/$defs/SemanticModel" + } + } + }, + "required": ["version", "semantic_model"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY"], + "description": "Supported SQL and expression language dialects" + }, + "Vendor": { + "type": "string", + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], + "description": "Vendor name for custom extensions. Any string value is accepted." + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "CustomExtension": { + "type": "object", + "description": "Vendor-specific attributes for extensibility", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "string", + "description": "JSON string containing vendor-specific data" + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "required": ["dialects"], + "additionalProperties": false + }, + "DataType": { + "type": "string", + "enum": [ + "String", + "Integer", + "Decimal", + "Float", + "Boolean", + "Date", + "Time", + "DateTime", + "DateTimeTz", + "Opaque" + ], + "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown; use `Opaque` plus `custom_extensions` for a known type outside the portable vocabulary." + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, or `DateTimeTz`, and `false` otherwise. Set `is_time: false` explicitly to opt a temporal-typed column (such as an audit timestamp) out of time-dimension treatment." + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Primary key columns (single or composite)" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + } + }, + "description": "Array of unique key definitions (each can be single or composite)" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + } + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Foreign key columns in the 'from' dataset" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Primary/unique key columns in the 'to' dataset" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Quantifiable measures spanning datasets" + }, + "custom_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/CustomExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/tests/ossie-fixtures/upstream/validation/validate.py b/tests/ossie-fixtures/upstream/validation/validate.py new file mode 100644 index 00000000..258d34f1 --- /dev/null +++ b/tests/ossie-fixtures/upstream/validation/validate.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "jsonschema>=4.26.0", +# "pyyaml>=6.0.3", +# "sqlglot>=30.12.0", +# ] +# /// + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Ossie Semantic Model Validator + +Validates Ossie YAML files against: +1. JSON Schema (structure, types, enums) +2. Unique names (datasets, fields, metrics, relationships) +3. Valid relationship references +4. SQL syntax (using sqlglot) + +Usage: + python validation/validate.py + python validation/validate.py --schema ontology/ontology.json + python validation/validate.py examples/tpcds_semantic_model.yaml +""" + +import json +import sys +from pathlib import Path + +try: + import yaml + from jsonschema import Draft202012Validator +except ImportError: + print("Missing dependencies. Install with:") + print(" pip install pyyaml jsonschema") + sys.exit(1) + +try: + import sqlglot + from sqlglot.errors import ParseError, TokenError + SQLGLOT_AVAILABLE = True +except ImportError: + SQLGLOT_AVAILABLE = False + +# Map Ossie dialects to sqlglot dialects +DIALECT_MAP = { + "ANSI_SQL": None, # sqlglot default + "SNOWFLAKE": "snowflake", + "DATABRICKS": "databricks", + "BIGQUERY": "bigquery", + "MDX": None, # Not supported by sqlglot, skip validation + "TABLEAU": None, # Not supported by sqlglot, skip validation + "MAQL": None, # Not supported by sqlglot, skip validation +} + +# Dialects that sqlglot cannot parse +SKIP_SQL_VALIDATION = {"MDX", "TABLEAU", "MAQL"} + + +def validate_schema(data: dict, schema: dict) -> list[str]: + """Validate against JSON Schema.""" + validator = Draft202012Validator(schema) + errors = [] + for error in validator.iter_errors(data): + path = " -> ".join(str(p) for p in error.absolute_path) if error.absolute_path else "(root)" + errors.append(f"[Schema] {path}: {error.message}") + return errors + + +def find_duplicates(items: list[str]) -> list[str]: + """Find duplicate items in a list.""" + seen = set() + duplicates = [] + for item in items: + if item in seen: + duplicates.append(item) + seen.add(item) + return duplicates + + +def validate_unique_names(data: dict) -> list[str]: + """Validate unique names for datasets, fields, metrics, relationships.""" + errors = [] + + for model in data.get("semantic_model", []): + model_name = model.get("name", "") + + # Check unique dataset names + dataset_names = [d.get("name") for d in model.get("datasets", []) if d.get("name")] + for dup in find_duplicates(dataset_names): + errors.append(f"[Unique] Duplicate dataset name '{dup}' in model '{model_name}'") + + # Check unique field names within each dataset + for dataset in model.get("datasets", []): + dataset_name = dataset.get("name", "") + field_names = [f.get("name") for f in dataset.get("fields", []) if f.get("name")] + for dup in find_duplicates(field_names): + errors.append(f"[Unique] Duplicate field name '{dup}' in dataset '{dataset_name}'") + + # Check unique metric names + metric_names = [m.get("name") for m in model.get("metrics", []) if m.get("name")] + for dup in find_duplicates(metric_names): + errors.append(f"[Unique] Duplicate metric name '{dup}' in model '{model_name}'") + + # Check unique relationship names + rel_names = [r.get("name") for r in model.get("relationships", []) if r.get("name")] + for dup in find_duplicates(rel_names): + errors.append(f"[Unique] Duplicate relationship name '{dup}' in model '{model_name}'") + + return errors + + +def validate_references(data: dict) -> list[str]: + """Validate that relationships reference existing datasets.""" + errors = [] + + for model in data.get("semantic_model", []): + model_name = model.get("name", "") + dataset_names = {d.get("name") for d in model.get("datasets", []) if d.get("name")} + + for rel in model.get("relationships", []): + rel_name = rel.get("name", "") + from_ds = rel.get("from") + to_ds = rel.get("to") + + if from_ds and from_ds not in dataset_names: + errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{from_ds}'") + if to_ds and to_ds not in dataset_names: + errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{to_ds}'") + + return errors + + +def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None: + """Validate a single SQL expression. Returns error message or None if valid.""" + if not SQLGLOT_AVAILABLE: + return None + + if dialect in SKIP_SQL_VALIDATION: + return None + + sqlglot_dialect = DIALECT_MAP.get(dialect) + + try: + # Try parsing as expression first (for field expressions like "column_name") + sqlglot.parse_one(expr, dialect=sqlglot_dialect) + return None + except (ParseError, TokenError): + pass + + try: + # Try wrapping in SELECT for simple column references + sqlglot.parse_one(f"SELECT {expr}", dialect=sqlglot_dialect) + return None + except (ParseError, TokenError) as e: + return f"[SQL] {context}: {str(e).split(chr(10))[0]}" + + +def validate_sql(data: dict) -> list[str]: + """Validate SQL expressions in fields and metrics.""" + # Only semantic model files contain SQL expressions to validate. + if not data.get("semantic_model"): + return [] + + if not SQLGLOT_AVAILABLE: + return ["[SQL] Warning: sqlglot not installed, skipping SQL validation. Install with: pip install sqlglot"] + + errors = [] + + for model in data.get("semantic_model", []): + model_name = model.get("name", "") + + # Validate field expressions + for dataset in model.get("datasets", []): + dataset_name = dataset.get("name", "") + for field in dataset.get("fields", []): + field_name = field.get("name", "") + expression = field.get("expression", {}) + for dialect_expr in expression.get("dialects", []): + dialect = dialect_expr.get("dialect", "ANSI_SQL") + expr = dialect_expr.get("expression", "") + if expr: + context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" + error = validate_sql_expression(expr, dialect, context) + if error: + errors.append(error) + + # Validate metric expressions + for metric in model.get("metrics", []): + metric_name = metric.get("name", "") + expression = metric.get("expression", {}) + for dialect_expr in expression.get("dialects", []): + dialect = dialect_expr.get("dialect", "ANSI_SQL") + expr = dialect_expr.get("expression", "") + if expr: + context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" + error = validate_sql_expression(expr, dialect, context) + if error: + errors.append(error) + + return errors + + +def main(): + if len(sys.argv) < 2: + print(__doc__) + sys.exit(1) + + args = sys.argv[1:] + yaml_path = Path(args[0]) + + schema_path = Path(__file__).parent.parent / "core-spec" / "osi-schema.json" + if len(args) > 1: + if len(args) == 3 and args[1] == "--schema": + schema_path = Path(args[2]) + else: + print("Usage: python validation/validate.py [--schema ]") + sys.exit(1) + + if not yaml_path.exists(): + print(f"Error: File not found: {yaml_path}") + sys.exit(1) + + if not schema_path.exists(): + print(f"Error: Schema not found: {schema_path}") + sys.exit(1) + + # Load files + with open(schema_path) as f: + schema = json.load(f) + + with open(yaml_path) as f: + try: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + print(f"Error: Invalid YAML: {e}") + sys.exit(1) + + # Run validations + errors = [] + errors.extend(validate_schema(data, schema)) + + # Run semantic-model-specific checks only for semantic model payloads. + if data.get("semantic_model"): + errors.extend(validate_unique_names(data)) + errors.extend(validate_references(data)) + errors.extend(validate_sql(data)) + + # Report results + if errors: + # Separate warnings from errors + warnings = [e for e in errors if "Warning:" in e] + actual_errors = [e for e in errors if "Warning:" not in e] + + for warning in warnings: + print(f" {warning}") + + if actual_errors: + print(f"\nValidation FAILED with {len(actual_errors)} error(s):\n") + for error in actual_errors: + print(f" {error}") + sys.exit(1) + else: + print(f"Validation PASSED: {yaml_path.name}") + sys.exit(0) + else: + print(f"Validation PASSED: {yaml_path.name}") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index 00bcf701..720958cb 100644 --- a/uv.lock +++ b/uv.lock @@ -12,6 +12,9 @@ resolution-markers = [ "python_full_version < '3.12'", ] +[options] +prerelease-mode = "if-necessary" + [[package]] name = "adbc-driver-manager" version = "1.9.0" @@ -3638,6 +3641,7 @@ dev = [ { name = "fastapi" }, { name = "httpx" }, { name = "inflect" }, + { name = "jsonschema" }, { name = "lkml" }, { name = "mypy" }, { name = "numpy" }, @@ -3660,6 +3664,7 @@ full = [ { name = "anywidget" }, { name = "fastapi" }, { name = "inflect" }, + { name = "jsonschema" }, { name = "lkml" }, { name = "lsprotocol" }, { name = "mcp", extra = ["cli"] }, @@ -3688,6 +3693,9 @@ mcp = [ metricflow = [ { name = "inflect" }, ] +ossie = [ + { name = "jsonschema" }, +] postgres = [ { name = "psycopg", extra = ["binary"] }, { name = "pyarrow" }, @@ -3761,6 +3769,8 @@ requires-dist = [ { name = "inflect", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "inflect", marker = "extra == 'metricflow'", specifier = ">=7.0.0" }, { name = "jinja2", specifier = ">=3.1.0" }, + { name = "jsonschema", marker = "extra == 'dev'", specifier = ">=4.23,<5" }, + { name = "jsonschema", marker = "extra == 'ossie'", specifier = ">=4.23,<5" }, { name = "lkml", marker = "extra == 'dev'", specifier = ">=1.3.7" }, { name = "lkml", marker = "extra == 'lookml'", specifier = ">=1.3.7" }, { name = "lsprotocol", marker = "extra == 'lsp'", specifier = ">=2025.0.0" }, @@ -3793,7 +3803,7 @@ requires-dist = [ { name = "riffq", marker = "extra == 'serve'", specifier = ">=0.1.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.0" }, { name = "sidemantic", extras = ["postgres", "bigquery", "snowflake", "clickhouse", "databricks", "spark", "adbc"], marker = "extra == 'all-databases'" }, - { name = "sidemantic", extras = ["workbench", "mcp", "apps", "charts", "lsp", "dax", "lookml", "malloy", "metricflow", "widget", "api"], marker = "extra == 'full'" }, + { name = "sidemantic", extras = ["workbench", "mcp", "apps", "charts", "lsp", "dax", "lookml", "malloy", "metricflow", "widget", "api", "ossie"], marker = "extra == 'full'" }, { name = "sidemantic-dax", marker = "extra == 'dax'", directory = "crates/dax-pyo3" }, { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=3.0.0" }, { name = "sqlglot", specifier = ">=30.1.0" }, @@ -3809,7 +3819,7 @@ requires-dist = [ { name = "vl-convert-python", marker = "extra == 'apps'", specifier = ">=1.0.0" }, { name = "vl-convert-python", marker = "extra == 'charts'", specifier = ">=1.0.0" }, ] -provides-extras = ["dev", "workbench", "mcp", "apps", "charts", "fast", "serve", "api", "postgres", "bigquery", "snowflake", "clickhouse", "databricks", "spark", "adbc", "lsp", "dax", "lookml", "malloy", "metricflow", "widget", "all-databases", "full"] +provides-extras = ["dev", "workbench", "mcp", "apps", "charts", "fast", "ossie", "serve", "api", "postgres", "bigquery", "snowflake", "clickhouse", "databricks", "spark", "adbc", "lsp", "dax", "lookml", "malloy", "metricflow", "widget", "all-databases", "full"] [package.metadata.requires-dev] dev = [