diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d496a39..0b53c6d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,7 @@ on: push: branches: # need this to be able to run the workflow until it has been merged into main - - yhong123/131-update_documentation + - yhong123/112-column_evaluators workflow_dispatch: env: PYTHON_VERSION: "3.12" diff --git a/datafaker/dialects.py b/datafaker/dialects.py index df09aff..9d92e7b 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -211,6 +211,96 @@ def compile_stddev_mssql(element: StdDev, compiler: Any, **kw: Any) -> str: return f"STDEV({e})" +class WordCount(ColumnElement[int]): # pylint: disable=too-many-ancestors + """Count whitespace-separated words in a text expression.""" + + expr: ColumnElement[str] + + _traverse_internals = [ + ("expr", InternalTraversal.dp_clauseelement), + ] + + def __init__(self, expr: ColumnElement[str]): + """Get a clause for the number of whitespace-separated words in ``expr``.""" + self.expr = expr + + __sa_operate__ = ColumnElement.operate + + +@compiles(WordCount) +def compile_word_count(element: WordCount, compiler: Any, **kw: Any) -> str: + """Exact word count via regexp_split_to_array (Postgres/DuckDB).""" + e = compiler.process(element.expr, **kw) + return f"array_length(regexp_split_to_array(trim({e}), '\\s+'), 1)" + + +@compiles(WordCount, "mssql") +def compile_word_count_mssql(element: WordCount, compiler: Any, **kw: Any) -> str: + """MSSQL equivalent: approximate word count via space-counting. + + No regex/array-split support, so approximate word count as (space + count in the trimmed text) + 1. This overcounts by one per run of 2+ + consecutive spaces relative to the exact regex-based count used on + Postgres/DuckDB, but there is no T-SQL primitive for a real + whitespace-run split. + """ + e = compiler.process(element.expr, **kw) + trimmed = f"TRIM({e})" + return ( + f"(CASE WHEN LEN({trimmed}) = 0 THEN 0 ELSE " + f"LEN({trimmed}) - LEN(REPLACE({trimmed}, ' ', '')) + 1 END)" + ) + + +class SentenceCount(ColumnElement[int]): # pylint: disable=too-many-ancestors + """Count '.'/'!'/'?'-delimited sentences in a text expression.""" + + expr: ColumnElement[str] + + _traverse_internals = [ + ("expr", InternalTraversal.dp_clauseelement), + ] + + def __init__(self, expr: ColumnElement[str]): + """Get a clause for the number of sentences in ``expr``.""" + self.expr = expr + + __sa_operate__ = ColumnElement.operate + + +@compiles(SentenceCount) +def compile_sentence_count(element: SentenceCount, compiler: Any, **kw: Any) -> str: + """Exact sentence count via regexp_split_to_array (Postgres/DuckDB).""" + e = compiler.process(element.expr, **kw) + return ( + f"array_length(regexp_split_to_array(trim({e}, ' .!?\\t\\n\\r'), '[.!?]+'), 1)" + ) + + +@compiles(SentenceCount, "mssql") +def compile_sentence_count_mssql( + element: SentenceCount, compiler: Any, **kw: Any +) -> str: + """MSSQL equivalent: approximate sentence count via delimiter-counting. + + No regex support, so approximate sentence count by normalizing '!'/'?' + to '.' and counting '.' characters, after trimming whitespace and + sentence-enders from both ends (so a leading/trailing sentence-ender + doesn't add a spurious segment, matching the Postgres/DuckDB + behaviour). This overcounts by one per run of 2+ consecutive delimiter + characters (e.g. "Wow!!") relative to the exact regex-based count, but + there is no T-SQL primitive for regex-run splitting. + """ + e = compiler.process(element.expr, **kw) + trim_chars = "' .!?' + CHAR(9) + CHAR(10) + CHAR(13)" + trimmed = f"TRIM({trim_chars} FROM {e})" + normalized = f"REPLACE(REPLACE({trimmed}, '!', '.'), '?', '.')" + return ( + f"(CASE WHEN LEN({trimmed}) = 0 THEN 0 ELSE " + f"LEN({normalized}) - LEN(REPLACE({normalized}, '.', '')) + 1 END)" + ) + + class IsNull(ColumnElement[bool]): # pylint: disable=too-many-ancestors """Represent IS NULL as an expression.""" diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py new file mode 100644 index 0000000..1f60538 --- /dev/null +++ b/datafaker/evaluators/column_evaluator.py @@ -0,0 +1,365 @@ +"""Evaluate how well a proposer's synthetic data matches a real column.""" + +import string +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import Column, Engine, UniqueConstraint, select +from sqlalchemy.types import Date, DateTime, Integer, Numeric, Time + +from datafaker.dialects import Random +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.evaluators.metrics import ( + DiversityMetric, + NoveltyMetric, + canonicalize_value, +) +from datafaker.evaluators.statistical_fidelity import StatisticalFidelity +from datafaker.proposers.base import Proposer, get_column_type + +__all__ = ["EvaluationProfile", "ColumnEvaluator", "ColumnStats", "ProposalEvaluation"] + + +@dataclass +class ColumnStats: + """Summary statistics of a column's (string-rendered) real values.""" + + row_count: int + unique_count: int + avg_length: float + space_ratio: float + digit_ratio: float + punctuation_ratio: float + + @property + def uniqueness(self): + """Fraction of rows whose value is distinct from the others.""" + return self.unique_count / max(self.row_count, 1) + + +@dataclass +class ProposalEvaluation: + """The scored result of evaluating one proposer against a column.""" + + proposer: Proposer + novelty: float + diversity: float + overall_score: float + pipeline_scores: dict[str, float] + copy_fraction: float = 0.0 + synthetic_uniqueness: float = 1.0 + + def __str__(self): + """Render a human-readable summary of the evaluation.""" + lines = [ + f"{self.proposer.name()} " + f" (error={self.overall_score:.6f})" + f" (novelty={self.novelty:.6f})" + f" (diversity={self.diversity:.6f})" + f" (copies={self.copy_fraction:.6f})" + ] + + for name, score in self.pipeline_scores.items(): + lines.append(f" {name:<15} {score:.6f}") + + return "\n".join(lines) + + __repr__ = __str__ + + +def analyse_column(values: list[Any]) -> ColumnStats: + """Compute length/space/digit/punctuation statistics over ``values``.""" + values = [str(v) for v in values if v is not None] + + total_chars = sum(len(v) for v in values) + + spaces = sum(v.count(" ") for v in values) + + digits = sum(c.isdigit() for v in values for c in v) + + punctuation = sum(c in string.punctuation for v in values for c in v) + + return ColumnStats( + row_count=len(values), + unique_count=len(set(values)), + avg_length=(total_chars / len(values) if values else 0), + space_ratio=(spaces / total_chars if total_chars else 0), + digit_ratio=(digits / total_chars if total_chars else 0), + punctuation_ratio=(punctuation / total_chars if total_chars else 0), + ) + + +# Below this fraction of distinct values, a column is treated as a small, +# repeated set of categories rather than a free-form or continuous quantity. +CATEGORICAL_UNIQUENESS_THRESHOLD = 0.2 + + +def choose_profile(stats: ColumnStats) -> EvaluationProfile: + """Pick a string column's evaluation profile from its statistics.""" + # mostly unique short values + if stats.avg_length < 30 and stats.uniqueness > 0.8: + return EvaluationProfile.SHORT_TEXT + + # repeated values + if stats.uniqueness < CATEGORICAL_UNIQUENESS_THRESHOLD: + return EvaluationProfile.CATEGORICAL + + # long text + if stats.avg_length > 50 or stats.space_ratio > 0.15: + return EvaluationProfile.FREE_TEXT + + return EvaluationProfile.SHORT_TEXT + + +def choose_numeric_profile(values) -> EvaluationProfile: + """Pick a numeric column's evaluation profile from its cardinality. + + Decide whether an Integer/Numeric column behaves like a small set of + repeated categories (a status code, rating, or flag) or like a + continuous / high-cardinality quantity (an age, salary, or identifier). + + Previously every Integer column was assumed to be categorical and every + Numeric column was assumed to be an identifier, regardless of how many + distinct values it actually had - which misclassifies something like an + integer age or salary column. This applies the same cardinality check + already used for string columns instead. + """ + non_null = [v for v in values if v is not None] + if not non_null: + return EvaluationProfile.IDENTIFIER + + uniqueness = len(set(non_null)) / len(non_null) + if uniqueness < CATEGORICAL_UNIQUENESS_THRESHOLD: + return EvaluationProfile.CATEGORICAL + + return EvaluationProfile.IDENTIFIER + + +def looks_like_email(values) -> bool: + """Heuristically decide whether a sample of values looks like emails.""" + nonempty = [str(v).strip() for v in values if v is not None and str(v).strip()] + if not nonempty: + return False + + email_like = 0 + for value in nonempty: + if "@" not in value: + continue + local_part, sep, domain = value.partition("@") + if not sep or not local_part or not domain: + continue + if "." in domain: + email_like += 1 + + return email_like / len(nonempty) > 0.5 + + +class ColumnEvaluator: # pylint: disable=too-many-instance-attributes + """Evaluate candidate proposers against one (or several merged) columns.""" + + def __init__(self): + """Initialize an unconfigured evaluator; call setup() before use.""" + self.engine = None + self.novelty_metric = None + self.diversity_metric = None + self.sample_size = 4000 + self.columns = [] + self.column = None + self.table = None + self.is_primary_key = False + self.is_unique_constrained = False + self.real_values = [] + self.column_is_numeric = False + self.real_uniqueness = 0.0 + self.statistical_fidelity = None + self.profile = None + + def setup(self, columns: list[Column], engine: Engine): + """Sample the real column(s) and pick the evaluation profile/pipelines.""" + self.engine = engine + self.novelty_metric = NoveltyMetric() + self.diversity_metric = DiversityMetric() + + self.columns = columns + self.column = columns[0] if len(columns) == 1 else None + self.table = columns[0].table if columns else None + # A composite key needs every constituent column marked primary_key; + # this reduces to the single column's own flag in the common case. + self.is_primary_key = bool(columns) and all(c.primary_key for c in columns) + # A real UNIQUE constraint/index needs the same uniqueness guarantee + # as a primary key, even though it isn't one - e.g. a UNIQUE email + # column. Distinct from is_primary_key so the two can be OR'd + # together (needs_uniqueness) without conflating "is the key" with + # "must be unique". + self.is_unique_constrained = self.is_primary_key or self._has_unique_constraint( + columns + ) + + with engine.connect() as conn: + if len(columns) == 1: + # Cap how many real rows we pull into memory so this scales to + # large tables, matching the synthetic sample_size above rather + # than materializing the entire column. Order randomly first: + # an unordered LIMIT returns whatever the DB naturally returns + # first (often insertion order), which can systematically miss + # real values for a column whose value correlates with row + # order (e.g. a "last_update" timestamp) - corrupting + # real_uniqueness/copy_fraction/novelty/diversity, all of + # which are computed from this sample. + rows = conn.execute( + select(columns[0]) + .select_from(columns[0].table) + .order_by(Random()) + .limit(self.sample_size) + ) + self.real_values = [row[0] for row in rows] + column_type = get_column_type(columns[0]) + else: + rows = conn.execute( + select(*columns) + .select_from(columns[0].table) + .order_by(Random()) + .limit(self.sample_size) + ) + self.real_values = [tuple(row) for row in rows] + column_type = None + + # Numeric columns (whether they end up profiled IDENTIFIER or, for a + # low-cardinality one like a foreign key, CATEGORICAL) have no shared + # real-world vocabulary that could explain a generator coincidentally + # matching real values - unlike a string column (e.g. common first + # names), where that overlap is expected and legitimate. Used to + # decide how broadly the resample-penalty in proposal_ranking.py + # applies. + self.column_is_numeric = isinstance(column_type, (Numeric, Integer)) + + # fraction of real values that are distinct - used to judge whether a + # generator that resamples from the real data verbatim (e.g. a + # ChoiceProposer) is doing the right thing (a small, shared + # vocabulary, where reproducing real values is expected/correct) or + # a privacy-losing shortcut (near-unique real values, where + # reproducing them verbatim just leaks real records). + non_null_real_values = [v for v in self.real_values if v is not None] + self.real_uniqueness = ( + len({str(v) for v in non_null_real_values}) / len(non_null_real_values) + if non_null_real_values + else 0.0 + ) + + # Evaluation dimensions: fidelity, novelty, diversity + if len(columns) == 1: + self.statistical_fidelity = StatisticalFidelity( + self.column, self.engine, sample_size=self.sample_size + ) + if isinstance(column_type, (Date, DateTime, Time)): + profile = EvaluationProfile.TEMPORAL + elif isinstance(column_type, (Numeric, Integer)): + profile = choose_numeric_profile(self.real_values) + else: + stats = analyse_column(self.real_values) + if looks_like_email(self.real_values): + profile = EvaluationProfile.EMAIL + else: + profile = choose_profile(stats) + self.profile = profile + self.statistical_fidelity.set_eval_pipelines(self.profile) + else: + self.statistical_fidelity = None + self.profile = EvaluationProfile.IDENTIFIER + + @staticmethod + def _has_unique_constraint(columns: list[Column]) -> bool: + """Whether exactly this column set is covered by a UNIQUE constraint/index.""" + if not columns: + return False + table = columns[0].table + column_set = set(columns) + for constraint in table.constraints: + if ( + isinstance(constraint, UniqueConstraint) + and set(constraint.columns) == column_set + ): + return True + for index in table.indexes: + if index.unique and set(index.columns) == column_set: + return True + return False + + def evaluate(self, proposer) -> ProposalEvaluation: + """Score one proposer's synthetic data against the sampled real data.""" + assert ( + self.novelty_metric is not None and self.diversity_metric is not None + ), "setup() must be called before evaluate()" + synthetic_samples = proposer.generate_data(self.sample_size) + + # calculate novelty + novelty_score = self.novelty_metric.compare( + self.real_values, + synthetic_samples, + ) + + # calculate diversity + diversity_score = self.diversity_metric.compare( + self.real_values, + synthetic_samples, + ) + + # calculate copy fraction: fraction of synthetic samples that exactly match a real value + try: + real_set = set( + str(canonicalize_value(v)) for v in self.real_values if v is not None + ) + if synthetic_samples: + copy_count = sum( + 1 + for s in synthetic_samples + if str(canonicalize_value(s)) in real_set + ) + copy_frac = copy_count / max(len(synthetic_samples), 1) + else: + copy_frac = 0.0 + except Exception: # pylint: disable=broad-exception-caught + # A proposer's own output is untrusted here - anything unhashable + # or unstringifiable should degrade to "no copies detected" + # rather than crash the whole propose/evaluate flow. + copy_frac = 0.0 + + # fraction of the synthetic sample's own values that are distinct from + # each other - a direct, real-data-independent measure of whether this + # proposer could satisfy a PRIMARY KEY/UNIQUE constraint at all. This + # is different from copy_fraction (which only checks overlap against + # the *real* values): a proposer resampling from a small vocabulary, + # or one like dist_gen.constant that emits the same value every time, + # duplicates against its own output regardless of whether that output + # happens to match any real value. + try: + synthetic_uniqueness = ( + len({str(s) for s in synthetic_samples}) / len(synthetic_samples) + if synthetic_samples + else 0.0 + ) + except Exception: # pylint: disable=broad-exception-caught + # Same rationale as the copy_frac catch above. + synthetic_uniqueness = 0.0 + + # calculate fidelity scores for each evaluation pipeline. Multi-column + # proposal evaluation does not map cleanly onto the single-column fidelity + # pipelines, so fall back to a neutral score instead of crashing. + if self.statistical_fidelity is None: + stat_fidelity_overall_score = 0.0 + stat_fidelity_pipeline_scores = {} + else: + ( + stat_fidelity_overall_score, + stat_fidelity_pipeline_scores, + ) = self.statistical_fidelity.calculate_scores(synthetic_samples) + + return ProposalEvaluation( + proposer=proposer, + novelty=novelty_score, + diversity=diversity_score, + overall_score=stat_fidelity_overall_score, + pipeline_scores=stat_fidelity_pipeline_scores, + copy_fraction=copy_frac, + synthetic_uniqueness=synthetic_uniqueness, + ) diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py new file mode 100644 index 0000000..d6afabe --- /dev/null +++ b/datafaker/evaluators/distribution_builders.py @@ -0,0 +1,287 @@ +"""Build real/synthetic value distributions for statistical fidelity scoring.""" + +import statistics +from abc import ABC, abstractmethod +from bisect import bisect_right +from collections import Counter +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import func, select + +from datafaker.dialects import Random +from datafaker.evaluators.feature_extractors import FeatureExtractor, IdentityExtractor + + +@dataclass +class Distribution: + """A discrete probability distribution over feature values.""" + + probabilities: dict[Any, float] + + @property + def vocabulary(self) -> set[Any]: + """The set of feature values this distribution assigns mass to.""" + return set(self.probabilities.keys()) + + def probability(self, key: Any) -> float: + """Return the probability mass assigned to ``key`` (0.0 if unseen).""" + return self.probabilities.get(key, 0.0) + + def as_vector(self, vocabulary: list[Any]) -> list[float]: + """Render this distribution as a probability vector over ``vocabulary``.""" + return [self.probability(v) for v in vocabulary] + + +class DistributionBuilder(ABC): + """Build a real or synthetic value's ``Distribution`` for one feature.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize the engine/table/column/extractor/sample_size shared by every builder.""" + self.engine = engine + self.table = table + self.column = column + self.extractor = extractor + self.sample_size = sample_size + + @abstractmethod + def build_from_table(self) -> Distribution: + """Build the distribution of this feature over the real column.""" + + @abstractmethod + def build_from_values(self, values) -> Distribution: + """Build the distribution of this feature over synthetic ``values``.""" + + +# pylint: disable=too-many-instance-attributes +class HistogramBuilder(DistributionBuilder): + """Bucket a continuous feature into a fixed-width histogram.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + engine, + table, + column, + bins=10, + extractor: FeatureExtractor = IdentityExtractor(), + sample_size=4000, + ): + """Initialize a histogram builder for one column feature.""" + super().__init__(engine, table, column, extractor, sample_size) + self.bins = bins + + # Learned from the real data + self.mean = None + self.stddev = None + self.bottom = None + self.width = None + self.edges = None + + def build_from_table(self) -> Distribution: + """Build the real column's histogram distribution.""" + feature_expr = self.extractor.expression(self.column) + + # + # Read a bounded, randomly-ordered sample of values for computing + # statistics (mean/stddev), rather than pulling the entire column + # into memory. Random ordering matters: an unordered LIMIT returns + # whatever the DB naturally returns first (often insertion order), + # which can systematically bias this sample for a column whose + # value correlates with row order (e.g. a "rental_date" column + # inserted roughly chronologically) - and since this is the "real" + # side of every fidelity comparison, that bias directly skews which + # generator looks like the better fit. + # + with self.engine.connect() as conn: + values = ( + conn.execute( + select(feature_expr) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + .scalars() + .all() + ) + + values = [float(v) for v in values if v is not None] + + if not values: + return Distribution({}) + + self.mean = statistics.mean(values) + self.stddev = statistics.stdev(values) if len(values) > 1 else 0.0 + + if self.stddev == 0: + self.edges = [ + float("-inf"), + self.mean - 2.0, + self.mean - 1.5, + self.mean - 1.0, + self.mean - 0.5, + self.mean, + self.mean + 0.5, + self.mean + 1.0, + self.mean + 1.5, + self.mean + 2.0, + float("inf"), + ] + bottom = self.mean - 2.0 + width = 0.5 + else: + self.edges = [ + float("-inf"), + self.mean - 2 * self.stddev, + self.mean - 1.5 * self.stddev, + self.mean - 1.0 * self.stddev, + self.mean - 0.5 * self.stddev, + self.mean, + self.mean + 0.5 * self.stddev, + self.mean + 1.0 * self.stddev, + self.mean + 1.5 * self.stddev, + self.mean + 2 * self.stddev, + float("inf"), + ] + bottom = self.mean - 2 * self.stddev + width = self.stddev / 2 + + if width == 0: + width = 1.0 + + # + # SQL histogram, computed exactly over the full table (this is a + # bounded, server-side aggregate query - it returns at most `bins` + # rows - so it doesn't need the sample_size cap that applies to the + # raw-value pulls above). + # + with self.engine.connect() as conn: + # Compute the bucket expression once, in an inner subquery, and + # group by the resulting materialized column in the outer query. + # MSSQL rejects both `GROUP BY ` (a SELECT-list alias + # referenced from GROUP BY) and `GROUP BY ` + # (its query planner does not recognize two separately-bound + # occurrences of the same parameterized expression as + # equivalent) - grouping by an actual column of a subquery is + # the one form every dialect (Postgres/DuckDB/MSSQL) accepts. + bucket_expr = func.floor((feature_expr - bottom) / width) + inner = ( + select(bucket_expr.label("bucket")).select_from(self.table).subquery() + ) + rows = conn.execute( + select( + inner.c.bucket, + func.count().label("count"), # pylint: disable=not-callable + ).group_by(inner.c.bucket) + ).all() + + # total must come from this same full-table query, not from the + # (possibly sampled) `values` above, so the probabilities sum to 1. + total = sum(count for bucket, count in rows if bucket is not None) + + if total == 0: + return Distribution({}) + + probs = {} + + for bucket, count in rows: + if bucket is None: + continue + + bucket = min( + self.bins - 1, + max(0, int(bucket) + 1), + ) + + probs[bucket] = probs.get(bucket, 0) + count / total + + return Distribution(probs) + + def build_from_values(self, values) -> Distribution: + """Build the synthetic sample's histogram distribution.""" + if self.edges is None: + # build_from_table() never learned edges - either it hasn't run + # yet, or the real column had no usable values. Either way there + # is nothing to bucket synthetic values against. + return Distribution({}) + + counts = Counter() + total = 0 + + for value in values: + for feature in self.extractor.extract(value): + feature = float(feature) + bucket = bisect_right(self.edges, feature) - 1 + bucket = max(0, min(bucket, len(self.edges) - 2)) + counts[bucket] += 1 + total += 1 + + if total == 0: + return Distribution({}) + + return Distribution({b: c / total for b, c in counts.items()}) + + +class CategoryBuilder(DistributionBuilder): + """Build a categorical distribution over a discrete feature's values.""" + + def build_from_table(self): + """Build the real column's categorical distribution.""" + counter = Counter() + + with self.engine.connect() as conn: + # Bounded, randomly-ordered sample rather than the entire column, + # so this scales to large tables (matches the synthetic sample + # size for a fair, like-for-like comparison) without an + # unordered LIMIT biasing the "real" distribution toward + # whatever the DB returns first (see HistogramBuilder above). + rows = conn.execute( + select(self.column) + .select_from(self.table) + .order_by(Random()) + .limit(self.sample_size) + ) + + for (value,) in rows: + if value is None: + continue + for feature in self.extractor.extract(value): + if feature is None: + continue + counter[feature] += 1 + + total = sum(counter.values()) + if total == 0: + return Distribution({}) + + return Distribution({k: v / total for k, v in counter.items()}) + + def build_from_values(self, values): + """Build the synthetic sample's categorical distribution.""" + counter = Counter() + + for value in values: + if value is None: + continue + for feature in self.extractor.extract(value): + if feature is None: + continue + counter[feature] += 1 + + total = sum(counter.values()) + if total == 0: + # No features were extracted from the synthetic values; represent this + # explicitly with a special missing token so metrics treat it as a + # disjoint distribution (maximally different) compared to a real + # distribution with concrete feature values. + return Distribution({"__MISSING__": 1.0}) + + return Distribution({k: v / total for k, v in counter.items()}) diff --git a/datafaker/evaluators/evaluation_profile.py b/datafaker/evaluators/evaluation_profile.py new file mode 100644 index 0000000..c7544eb --- /dev/null +++ b/datafaker/evaluators/evaluation_profile.py @@ -0,0 +1,20 @@ +"""The evaluation profile enum, kept in its own module to avoid a cycle. + +``column_evaluator`` and ``statistical_fidelity`` both need this type, and +each also needs something from the other module - keeping the enum here lets +both import it directly instead of one importing it transitively through the +other. +""" + +from enum import Enum, auto + + +class EvaluationProfile(Enum): + """The kind of value a column holds, used to pick its fidelity pipelines.""" + + SHORT_TEXT = auto() + CATEGORICAL = auto() + FREE_TEXT = auto() + EMAIL = auto() + IDENTIFIER = auto() + TEMPORAL = auto() diff --git a/datafaker/evaluators/feature_extractors.py b/datafaker/evaluators/feature_extractors.py new file mode 100644 index 0000000..5ddf542 --- /dev/null +++ b/datafaker/evaluators/feature_extractors.py @@ -0,0 +1,470 @@ +"""Extract a comparable feature from a column's real/synthetic values.""" + +import re +from abc import ABC, abstractmethod +from datetime import date, datetime +from typing import Any + +from sqlalchemy import String, func, literal + +from datafaker.dialects import SecondsDifference, SentenceCount, WordCount + +TOKEN = re.compile(r"\w+") + + +def _coerce_datetime(value): + """ + Best-effort coercion of a value into something with .year/.month/etc. + + A value that genuinely represents a real datetime can still arrive here + as a plain string rather than a datetime/date object - e.g. because the + SQL query that produced it (some proposers build queries from raw + column-name strings, losing type information) didn't preserve its type. + Without this, a temporal extractor would silently treat it as "not a + date" and drop it entirely, rather than raising a visible error - + corrupting the resulting distribution for one candidate while giving no + indication anything went wrong. + """ + if hasattr(value, "year"): + return value + if isinstance(value, str): + try: + return datetime.fromisoformat(value.strip()) + except ValueError: + return None + return None + + +class FeatureExtractor(ABC): + """Extract one comparable feature from a real or synthetic value.""" + + @abstractmethod + def extract(self, value: Any): + """ + Convert one cell into one or more feature values (synthetic/Python side). + + Returns: + iterable of feature values + """ + + @abstractmethod + def expression(self, column): + """Build the equivalent SQLAlchemy expression for the real/database side.""" + + +class IdentityExtractor(FeatureExtractor): + """Use the value itself as the feature.""" + + def extract(self, value): + """Yield the value unchanged.""" + if value is None: + return + yield value + + def expression(self, column): + """Return the column unchanged.""" + return column + + +class FirstLetterExtractor(FeatureExtractor): + """Extract a string value's first letter, lowercased.""" + + def extract(self, value): + """Yield the value's first letter, lowercased.""" + if value is None: + return + value = str(value).strip() + if value: + yield value[0].lower() + + def expression(self, column): + """Build the SQL expression for the column's first letter.""" + return func.lower(func.substr(column, 1, 1)) + + +class LastLetterExtractor(FeatureExtractor): + """Extract a string value's last letter, lowercased.""" + + def extract(self, value): + """Yield the value's last letter, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[-1] + + def expression(self, column): + """Build the SQL expression for the column's last letter.""" + return func.lower(func.substr(column, -1, 1)) + + +class LengthExtractor(FeatureExtractor): + """Extract a value's string length.""" + + def extract(self, value): + """Yield the value's string length.""" + if value is None: + return + + yield len(str(value)) + + def expression(self, column): + """Build the SQL expression for the column's string length.""" + return func.length(column) + + +class WordExtractor(FeatureExtractor): + """Extract a string value's individual word tokens.""" + + def extract(self, value): + """Yield each word token in the value, lowercased.""" + if value is None: + return + + yield from TOKEN.findall(str(value).lower()) + + def expression(self, column): + """Not implemented: word tokenization has no SQL equivalent here.""" + raise NotImplementedError("WordExtractor SQL expression is not implemented.") + + +class CharacterExtractor(FeatureExtractor): + """Extract a string value's individual characters.""" + + def extract(self, value): + """Yield each character in the value.""" + if value is None: + return + + yield from str(value) + + def expression(self, column): + """Not implemented: per-character extraction has no SQL equivalent here.""" + raise NotImplementedError( + "CharacterExtractor SQL expression is not implemented." + ) + + +class CharacterBigramExtractor(FeatureExtractor): + """Extract a string value's overlapping 2-character sequences.""" + + def extract(self, value): + """Yield each overlapping 2-character bigram, lowercased.""" + if value is None: + return + + value = str(value).lower() + + for i in range(len(value) - 1): + yield value[i : i + 2] + + def expression(self, column): + """Not implemented: bigram extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class CharacterTrigramExtractor(FeatureExtractor): + """Extract a string value's overlapping 3-character sequences.""" + + def extract(self, value): + """Yield each overlapping 3-character trigram, lowercased.""" + if value is None: + return + + value = str(value).lower() + + for i in range(len(value) - 2): + yield value[i : i + 3] + + def expression(self, column): + """Not implemented: trigram extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class PrefixExtractor(FeatureExtractor): + """Extract a string value's first ``length`` characters.""" + + def __init__(self, length=2): + """Initialize with the prefix length to extract.""" + self.length = length + + def extract(self, value): + """Yield the value's first ``length`` characters, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[: self.length] + + def expression(self, column): + """Not implemented: prefix extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class SuffixExtractor(FeatureExtractor): + """Extract a string value's last ``length`` characters.""" + + def __init__(self, length=2): + """Initialize with the suffix length to extract.""" + self.length = length + + def extract(self, value): + """Yield the value's last ``length`` characters, lowercased.""" + if value is None: + return + + value = str(value).strip().lower() + + if value: + yield value[-self.length :] + + def expression(self, column): + """Not implemented: suffix extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +VOWELS = set("aeiou") + + +class VowelConsonantPatternExtractor(FeatureExtractor): + """Extract a string value's vowel/consonant pattern (e.g. "CVCV").""" + + def extract(self, value): + """Yield the value's vowel/consonant pattern string.""" + if value is None: + return + + pattern = [] + + for ch in str(value).lower(): + if not ch.isalpha(): + continue + + pattern.append("V" if ch in VOWELS else "C") + + if pattern: + yield "".join(pattern) + + def expression(self, column): + """Not implemented: pattern extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class TimestampExtractor(FeatureExtractor): + """Extract a date/datetime as a single continuous days-since-epoch value. + + This captures year, month and day jointly as one quantity for a + HistogramBuilder, rather than scoring them as independent, + uncorrelated dimensions - a generator that draws month/day/hour + independently (uniformly within a calibrated year range) will never + reproduce real joint patterns (e.g. seasonal clustering) even if each + marginal component looks reasonable alone, and previously scored + accordingly poorly per-dimension despite being reasonably calibrated; + a single joint measure is a fairer, simpler fidelity signal. + """ + + _EPOCH = datetime(1970, 1, 1) + _EPOCH_DATE = date(1970, 1, 1) + + def extract(self, value): + """Yield the value's days-since-epoch, as a single continuous float.""" + value = _coerce_datetime(value) + if value is None: + return + if not isinstance(value, datetime): + value = datetime(value.year, value.month, value.day) + if getattr(value, "tzinfo", None) is not None: + value = value.replace(tzinfo=None) + yield (value - self._EPOCH).total_seconds() / 86400.0 + + def expression(self, column): + """Build the SQL expression for the column's days-since-epoch.""" + # func.extract("epoch", ...) has no MSSQL equivalent (DATEPART has no + # "epoch" field); SecondsDifference already solves exactly this via a + # dialect-specific DATEDIFF compilation on MSSQL. + epoch = literal(self._EPOCH_DATE, type_=column.type) + return SecondsDifference(column, epoch) / 86400.0 + + +class WeekdayExtractor(FeatureExtractor): + """Extract day-of-week: a cyclic pattern a continuous value can't capture. + + A continuous days-since-epoch value can't capture a cyclic pattern + (e.g. weekday/weekend clustering) on its own, since it isn't a function + of absolute date position. + """ + + def extract(self, value): + """Yield the value's day of the week (Monday=0).""" + value = _coerce_datetime(value) + if value is None: + return + yield value.weekday() + + def expression(self, column): + """Build the SQL expression for the column's day of the week.""" + return func.extract("dow", column) # pylint: disable=not-callable + + +class EmailLocalPartExtractor(FeatureExtractor): + """Extract the local part (before ``@``) of an email-like string.""" + + def extract(self, value): + """Yield the value's local part, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + local_part, _, _ = text.partition("@") + if local_part: + yield local_part.lower() + + def expression(self, column): + """Not implemented: local-part extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailDomainExtractor(FeatureExtractor): + """Extract the domain (after ``@``) of an email-like string.""" + + def extract(self, value): + """Yield the value's domain, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + _, _, domain = text.partition("@") + if domain: + yield domain.lower() + + def expression(self, column): + """Not implemented: domain extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailTopLevelDomainExtractor(FeatureExtractor): + """Extract the top-level domain of an email-like string.""" + + def extract(self, value): + """Yield the value's top-level domain, lowercased.""" + if value is None: + return + text = str(value).strip() + if not text or "@" not in text: + return + _, _, domain = text.partition("@") + if "." not in domain: + return + tld = domain.rsplit(".", 1)[-1].lower() + if tld: + yield tld + + def expression(self, column): + """Not implemented: TLD extraction has no SQL equivalent here.""" + raise NotImplementedError() + + +class EmailValidityExtractor(FeatureExtractor): + """Extract a 'valid'/'invalid' token for an email-like string. + + Uses the email-validator package when available, falling back to a + conservative regex. + """ + + # conservative regex fallback + _SIMPLE_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + def __init__(self): + """Try importing the robust validator; fall back to regex if unavailable.""" + try: + from email_validator import ( # type: ignore # pylint: disable=import-outside-toplevel,import-error + validate_email, + ) + + self._validator = validate_email + except ImportError: + self._validator = None + + def extract(self, value): + """Yield 'valid' or 'invalid' for the value's email syntax.""" + if value is None: + return + text = str(value).strip() + if not text: + return + valid = False + if self._validator is not None: + try: + # check_deliverability=False for syntactic check only + _ = self._validator(text, check_deliverability=False) + valid = True + except Exception: # pylint: disable=broad-exception-caught + # email_validator raises its own EmailNotValidError, but + # this is optional/best-effort - any failure here should + # just mean "invalid", not crash the whole evaluation. + valid = False + else: + # fallback to simple regex + valid = bool(self._SIMPLE_RE.match(text)) + + yield "valid" if valid else "invalid" + + def expression(self, column): + """Not implemented: validity checking has no SQL equivalent here.""" + raise NotImplementedError() + + +class WordCountExtractor(FeatureExtractor): + """Extract a string value's word count.""" + + def extract(self, value): + """Yield the value's word count.""" + if value is None: + return + text = str(value).strip() + if not text: + return + # count words using TOKEN + count = len(TOKEN.findall(text)) + yield count + + def expression(self, column): + """Build the SQL expression for the column's word count.""" + # Cast column to text to handle tsvector columns (e.g. fulltext) + # where regexp_split_to_array may not accept the tsvector type + # directly. WordCount compiles to an exact regexp-based count on + # Postgres/DuckDB and a string-function approximation on MSSQL + # (which has no regex support) - see dialects.py. + return WordCount(column.cast(String)) + + +class SentenceCountExtractor(FeatureExtractor): + """Extract a string value's sentence count.""" + + def extract(self, value): + """Yield the value's sentence count.""" + if value is None: + return + text = str(value).strip() + if not text: + return + # split on sentence enders + parts = re.split(r"[.!?]+", text) + # count non-empty segments containing a word + count = sum(1 for p in parts if TOKEN.search(p)) + yield count + + def expression(self, column): + """Build the SQL expression for the column's sentence count.""" + # Cast column to text to handle tsvector columns (e.g. fulltext) + # where regexp_split_to_array may not accept the tsvector type + # directly. SentenceCount compiles to an exact regexp-based count + # on Postgres/DuckDB and a string-function approximation on MSSQL + # (which has no regex support) - see dialects.py. + return SentenceCount(column.cast(String)) diff --git a/datafaker/evaluators/metrics.py b/datafaker/evaluators/metrics.py new file mode 100644 index 0000000..070889d --- /dev/null +++ b/datafaker/evaluators/metrics.py @@ -0,0 +1,178 @@ +"""Metrics for comparing a real column's data against a synthetic sample.""" + +import math +from abc import ABC, abstractmethod +from collections import Counter +from decimal import Decimal + +from datafaker.evaluators.distribution_builders import Distribution + + +def canonicalize_value(value): + """Normalize a value for real-vs-synthetic equality comparison. + + A column's real values may come back as decimal.Decimal (SQLAlchemy's + default for a Numeric column) or as a tuple (a merged/composite + column's row), while a proposer's synthetic values for the same data + are plain float/list. Comparing str(Decimal("123.40")) against + str(123.4), or str((1.2, 3.4)) against str([1.2, 3.4]), would never + match even when the underlying values are identical - normalize both + sides to the same numeric/sequence type first so an exact match survives. + """ + if isinstance(value, (list, tuple)): + return tuple(canonicalize_value(v) for v in value) + if isinstance(value, Decimal): + return float(value) + return value + + +class Metric(ABC): + """A comparison between a real and a synthetic distribution or sample.""" + + @abstractmethod + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score how closely ``synthetic`` matches ``real``.""" + + +def _canonicalize_distribution(dist: Distribution) -> Distribution: + """Normalize keys so None and mixed key types are comparable and sortable.""" + normalized = { + ("" if key is None else str(key)): value + for key, value in dist.probabilities.items() + } + return Distribution(normalized) + + +class MeanSquaredError(Metric): + """Mean squared error between two probability distributions.""" + + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score the mean squared error between ``real`` and ``synthetic``.""" + real = _canonicalize_distribution(real) + synthetic = _canonicalize_distribution(synthetic) + vocab = sorted(real.vocabulary | synthetic.vocabulary) + + if not vocab: + return 0.0 + + x = real.as_vector(vocab) + y = synthetic.as_vector(vocab) + + squared_error = sum((a - b) ** 2 for a, b in zip(x, y)) + + # x and y are probability vectors (each sums to 1), so their squared + # Euclidean distance is bounded in [0, 2] regardless of vocab size. + # Dividing by that fixed bound (rather than len(vocab)**2) keeps the + # score on a comparable scale to JensenShannon's [0, ln 2] range, so + # pipeline weights combining the two behave as intended. + score = squared_error / 2 + return score + + +class JensenShannon(Metric): + """Jensen-Shannon divergence between two probability distributions.""" + + EPS = 1e-12 + + def compare( + self, + real: Distribution, + synthetic: Distribution, + ) -> float: + """Score the Jensen-Shannon divergence between ``real`` and ``synthetic``.""" + real = _canonicalize_distribution(real) + synthetic = _canonicalize_distribution(synthetic) + vocab = sorted(real.vocabulary | synthetic.vocabulary) + + if not vocab: + return 0.0 + + p = real.as_vector(vocab) + q = synthetic.as_vector(vocab) + + m = [(a + b) / 2 for a, b in zip(p, q)] + + def kl(a, b): + total = 0 + for x, y in zip(a, b): + if x > 0: + total += x * math.log(x / max(y, self.EPS)) + return total + + return (kl(p, m) + kl(q, m)) / 2 + + +class NoveltyMetric(Metric): + """Fraction of synthetic values that don't already appear in the real data.""" + + def compare( + self, + real, + synthetic, + ): + """Score how much of ``synthetic`` is novel relative to ``real``.""" + real_set = { + str(canonicalize_value(v)).strip().lower() for v in real if v is not None + } + + generated = { + str(canonicalize_value(v)).strip().lower() + for v in synthetic + if v is not None + } + + if not generated: + return 0.0 + + overlap = generated & real_set + + return 1.0 - len(overlap) / len(generated) + + +class DiversityMetric(Metric): + """Compare the internal diversity of a real and a synthetic sample. + + Measures how closely the synthetic sample's internal diversity matches + the real sample's diversity, rather than rewarding diversity outright. + + Each sample is scored by its normalized Shannon entropy (entropy divided + by log(k), k = number of distinct values), which is 0 for a single + repeated value and 1 for a uniform distribution over all its distinct + values. The metric returns 1 minus the absolute difference between the + real and synthetic normalized entropies: 1.0 when the synthetic sample + is exactly as diverse (relative to its own value count) as the real + sample, dropping toward 0 the more it over- or under-diversifies + relative to the real data. This avoids favoring generators that spread + probability more uniformly than the real data actually does. + """ + + @staticmethod + def _normalized_entropy(values) -> float: + """Compute the normalized Shannon entropy of ``values``.""" + vals = [str(v).strip().lower() for v in values if v is not None] + if not vals: + return 0.0 + counts = Counter(vals) + k = len(counts) + if k <= 1: + return 0.0 + total = sum(counts.values()) + probs = [c / total for c in counts.values()] + ent = -sum(p * math.log(p) for p in probs if p > 0) + return ent / math.log(k) + + def compare(self, real, synthetic): + """Score how closely ``synthetic``'s diversity matches ``real``'s.""" + real_diversity = self._normalized_entropy(real) + synthetic_diversity = self._normalized_entropy(synthetic) + # clamp against floating-point noise (e.g. two near-1.0 entropies + # differing by a rounding error) so the result stays within [0, 1] + return max(0.0, 1.0 - abs(synthetic_diversity - real_diversity)) diff --git a/datafaker/evaluators/proposal_ranking.py b/datafaker/evaluators/proposal_ranking.py new file mode 100644 index 0000000..e9b06e1 --- /dev/null +++ b/datafaker/evaluators/proposal_ranking.py @@ -0,0 +1,549 @@ +"""Utilities for ranking proposal evaluations against multiple objectives.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Sequence + +from datafaker.evaluators.column_evaluator import ProposalEvaluation +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.proposers.choice import ChoiceProposer + +# Column-name keywords mapped to substrings of the generator's dotted name +# (e.g. "generic.person.first_name") that they hint at. Column naming +# conventions are a strong, cheap, complementary signal to the statistical +# fidelity/novelty/diversity evaluation: they catch cases the statistics +# genuinely can't discriminate (e.g. a column with a small realistic +# vocabulary where every candidate's exact-value fidelity ties) without +# having to rely on the generator library's own internal naming, which +# rarely matches real-world column names anyway. +# +# This is a soft boost, not a filter: a column name matching nothing here, +# or a generator matching no hint, is entirely unaffected - it never +# excludes a candidate the statistics would otherwise have picked. Keep this +# list small and grow it opportunistically as real columns turn up cases +# worth covering, rather than trying to anticipate every naming convention. +KEYWORD_GENERATOR_HINTS: list[tuple[list[str], list[str]]] = [ + ( + ["first_name", "firstname", "fname", "given_name", "forename"], + ["person.first_name"], + ), + ( + ["last_name", "lastname", "lname", "surname", "family_name"], + ["person.last_name"], + ), + (["full_name", "fullname"], ["person.full_name"]), + (["email"], ["person.email"]), + (["phone", "telephone", "mobile"], ["person.telephone", "address.calling_code"]), + (["country"], ["address.country", "address.country_code"]), + (["city"], ["address.city"]), + ( + ["street"], + ["address.street_name", "address.street_number", "address.street_suffix"], + ), + (["postcode", "postal_code", "zip"], ["address.postal_code"]), + (["gender", "sex"], ["person.gender"]), + (["occupation", "job_title"], ["person.occupation"]), + (["nationality"], ["person.nationality"]), + (["username", "login"], ["person.username"]), +] + +# Small enough not to override a clearly-better statistical result, large +# enough to break the near-ties this is meant to resolve. +KEYWORD_BOOST = 0.15 + +# Below this fraction of distinct values in a candidate's own 4000-value +# synthetic sample, it is producing enough duplicates against itself that it +# could not satisfy a real PRIMARY KEY/UNIQUE constraint once row counts grow +# past a handful - regardless of how well it otherwise fits the real +# distribution. Slack below 1.0 tolerates a rare coincidental collision from +# a genuinely-unique-generating proposer (e.g. a continuous float sampler) +# without treating it the same as a resampler or constant that duplicates +# constantly. +SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD = 0.999 + +# The weak-recommendation warning (below) is only worth showing when the +# winner's own combined score is actually weak - otherwise it's just noise +# on top of a perfectly good pick. +WEAK_WINNER_SCORE_THRESHOLD = 0.5 + +# Floor applied to the self-duplication penalty (see rank_proposals) for a +# column that doesn't actually need guaranteed-unique values (no PRIMARY +# KEY/UNIQUE constraint). Self-duplication still matters there - a +# generator whose synthetic sample is mostly repeats of a tiny fixed pool +# (e.g. a canned-quote generator) makes for an obviously repetitive column +# even without breaking a constraint - but it shouldn't be crushed as hard +# as it would be for a column where duplicates are a correctness bug. +SELF_DUPLICATION_PENALTY_FLOOR = 0.4 + + +def keyword_match( + column_name: str | None, proposer_name: str +) -> tuple[float, str | None]: + """Give a soft ranking boost when the column name hints at this generator's kind. + + Returns the boost plus the specific column keyword responsible (or None) + so the caller can show why. + + Uses substring matching (not exact tokenization), so this also catches + compound names like "customer_first_name". + """ + if not column_name: + return 0.0, None + name = column_name.lower() + generator_name = proposer_name.lower() + for column_keywords, generator_keywords in KEYWORD_GENERATOR_HINTS: + matched_keyword = next((kw for kw in column_keywords if kw in name), None) + if matched_keyword is not None and any( + gkw in generator_name for gkw in generator_keywords + ): + return KEYWORD_BOOST, matched_keyword + return 0.0, None + + +@dataclass +class ProposalRankingDisplay: + """Formatted text for a ranked proposal table.""" + + recommendation: str | None + profile_summary: str + rows: list[tuple[str, ...]] + fronts: list[int] + weak_recommendation_warning: str | None = None + no_uniqueness_guarantee: bool = False + + +# pylint: disable=too-many-instance-attributes +@dataclass +class ProposalRanking: + """A ranked set of proposal evaluations.""" + + recommended_index: int | None + recommended_reason: str | None + rows: list[tuple[str, ...]] + profile: EvaluationProfile | None + weights: tuple[float, float, float] + real_uniqueness: float = 0.0 + penalty_applies_to_all: bool = False + self_duplication_floored: bool = False + fronts: list[int] = field(default_factory=list) + scores: list[float] = field(default_factory=list) + weak_recommendation_warning: str | None = None + no_uniqueness_guarantee: bool = False + + +def normalize_list(vals: Sequence[float]) -> list[float]: + """Normalize values to the range [0, 1] for comparisons.""" + if not vals: + return [] + lo = min(vals) + hi = max(vals) + # Treat near-identical values as a tie rather than an exact one: e.g. + # when every candidate's synthetic sample shares no categories with the + # real data, their Jensen-Shannon divergence all sit at the same + # theoretical maximum (ln 2) but differ in the 10th+ decimal place from + # floating-point rounding. Min-max normalizing that noise would stretch + # it across the full [0, 1] range and manufacture a fake ranking. + if math.isclose(hi, lo, rel_tol=1e-9, abs_tol=1e-12): + return [0.5 for _ in vals] + return [(v - lo) / (hi - lo) for v in vals] + + +def _pareto_fronts(points: Sequence[tuple[float, float, float]]) -> list[list[int]]: + """Compute Pareto fronts for a list of 3D points.""" + remaining = set(range(len(points))) + fronts: list[list[int]] = [] + while remaining: + current_front: set[int] = set() + for i in remaining: + dominated = False + for j in remaining: + if i == j: + continue + if all( + points[j][d] >= points[i][d] for d in range(len(points[i])) + ) and any(points[j][d] > points[i][d] for d in range(len(points[i]))): + dominated = True + break + if not dominated: + current_front.add(i) + fronts.append(sorted(current_front)) + remaining -= current_front + return fronts + + +def _crowding_distances( + points: Sequence[tuple[float, float, float]], + fronts: Sequence[Sequence[int]], + objectives: tuple[Sequence[float], Sequence[float], Sequence[float]], +) -> list[float]: + """Compute NSGA-II style crowding distances.""" + crowding = [0.0 for _ in range(len(points))] + for front in fronts: + if len(front) <= 1: + for idx in front: + crowding[idx] = float("inf") + continue + for dim_vals in objectives: + vals = [(idx, dim_vals[idx]) for idx in front] + vals.sort(key=lambda item: item[1]) + lo = vals[0][1] + hi = vals[-1][1] + crowding[vals[0][0]] = float("inf") + crowding[vals[-1][0]] = float("inf") + if hi == lo: + continue + for k in range(1, len(vals) - 1): + prev_v = vals[k - 1][1] + next_v = vals[k + 1][1] + dist = (next_v - prev_v) / (hi - lo) + idx = vals[k][0] + if crowding[idx] != float("inf"): + crowding[idx] += dist + return crowding + + +def _penalty_for_candidate( + result: ProposalEvaluation, + real_uniqueness: float, + apply_to_every_proposer: bool, + needs_uniqueness: bool, +) -> float: + """Combine the resample and self-duplication penalties for one candidate. + + ChoiceProposer and its variants (dist_gen.choice/weighted_choice/ + zipf_choice) resample directly from the column's own observed values - + by construction, they can only ever emit values already in the real + data. That's the *correct* thing to do for a column with a small, + shared vocabulary (a status code, a gender) - reproducing real values + there is expected and unavoidable - but a privacy problem for a column + whose real values are meant to be unique per row (an email, a genuine + ID). Numeric columns get the same treatment regardless of proposer + (apply_to_every_proposer): a plain number has no shared human + vocabulary to explain a coincidental match, so a high copy_fraction + there means the real value range is dense, not a legitimate overlap. + + Separately, self_duplication_penalty catches a candidate duplicating + against its OWN output (synthetic_uniqueness) rather than against the + real data (copy_fraction, above) - e.g. dist_gen.constant on a column + whose fixed value happens not to appear in the sampled real data gets + copy_fraction=0 (no resample penalty) despite duplicating every row + against itself. It applies to every candidate, but its full strength + is only warranted when the column actually needs unique values (a + PRIMARY KEY/UNIQUE constraint); it's floored otherwise, since a merely + observed-unique column (e.g. free-text descriptions that happen to all + differ) doesn't have a correctness reason to punish a small, + repetitive-but-otherwise-plausible pool as hard as a real key would. + """ + resample_penalty = 1.0 + if isinstance(result.proposer, ChoiceProposer) or apply_to_every_proposer: + resample_penalty = 1.0 - real_uniqueness * result.copy_fraction + self_duplication_penalty = 1.0 - real_uniqueness * ( + 1.0 - result.synthetic_uniqueness + ) + if not needs_uniqueness: + self_duplication_penalty = max( + self_duplication_penalty, SELF_DUPLICATION_PENALTY_FLOOR + ) + return resample_penalty * self_duplication_penalty + + +# pylint: disable=too-many-arguments too-many-positional-arguments +# pylint: disable=too-many-locals too-many-statements +def rank_proposals( + results: Sequence[ProposalEvaluation], + profile: EvaluationProfile | None, + theme: Any | None = None, + column_name: str | None = None, + real_uniqueness: float = 0.0, + is_numeric_column: bool = False, + is_primary_key: bool = False, + is_unique_constrained: bool = False, +) -> ProposalRanking: + """Rank proposals by fidelity, novelty and diversity using Pareto fronts. + + The ranking logic is deliberately separated from the interactive shell so it can + be tested and reused independently of presentation concerns. + """ + if not results: + return ProposalRanking(None, None, [], profile, (0.5, 0.25, 0.25)) + + # prepare multi-objective ranking: normalize metrics so larger-is-better + # overall_score is lower-is-better (distance/error), so invert after normalization + scores = [result.overall_score for result in results] + novelties = [result.novelty for result in results] + diversities = [result.diversity for result in results] + + norm_scores = normalize_list(scores) + norm_nov = normalize_list(novelties) + norm_div = normalize_list(diversities) + + # fidelity (higher-is-better) is inverted normalized overall_score + fidelity_scores = [1.0 - v for v in norm_scores] + novelty_scores = norm_nov + diversity_scores = norm_div + + # build points for Pareto (higher-is-better in all dims) + points = list(zip(fidelity_scores, novelty_scores, diversity_scores)) + + # non-dominated sort into Pareto fronts (simple O(n^2) algorithm) + fronts = _pareto_fronts(points) + front_of = [0] * len(points) + for front_index, front in enumerate(fronts, start=1): + for idx in front: + front_of[idx] = front_index + + # compute crowding distances per front (NSGA-II style) to prefer diverse solutions + crowding = _crowding_distances( + points, fronts, (fidelity_scores, novelty_scores, diversity_scores) + ) + + # determine profile and weights for scoring + # + # SHORT_TEXT, EMAIL and TEMPORAL all draw from a bounded, structured + # real-world vocabulary (names, addresses, dates), the same way + # CATEGORICAL/IDENTIFIER data does. For that kind of column, novelty and + # fidelity are mechanically in tension for the *best* generators: a + # generator that accurately models the real distribution is more, not + # less, likely to occasionally reproduce a real value by chance. Giving + # novelty a higher weight than fidelity here (as before) let it override + # an even strongly-discriminating fidelity signal and penalize the + # generator that best matched the real data. FREE_TEXT is kept + # novelty-tolerant since open-ended text has no such bounded vocabulary - + # genuinely novel sentences there are a good sign, not a red flag. + profile_weights = { + EvaluationProfile.IDENTIFIER: (0.85, 0.05, 0.10), + EvaluationProfile.CATEGORICAL: (0.7, 0.15, 0.15), + EvaluationProfile.SHORT_TEXT: (0.7, 0.15, 0.15), + EvaluationProfile.EMAIL: (0.7, 0.15, 0.15), + EvaluationProfile.FREE_TEXT: (0.5, 0.3, 0.2), + EvaluationProfile.TEMPORAL: (0.7, 0.15, 0.15), + } + fid_w, nov_w, div_w = ( + profile_weights[profile] if profile in profile_weights else (0.5, 0.25, 0.25) + ) + + # compute profile-specific score (higher-is-better) + combined_scores = [fid_w * f + nov_w * n + div_w * d for (f, n, d) in points] + + # See _penalty_for_candidate for what resample_penalty and + # self_duplication_penalty actually guard against and why. + apply_to_every_proposer = is_numeric_column + needs_uniqueness = is_primary_key or is_unique_constrained + penalty_multipliers = [ + _penalty_for_candidate( + result, real_uniqueness, apply_to_every_proposer, needs_uniqueness + ) + for result in results + ] + for idx in range(len(results)): + combined_scores[idx] *= penalty_multipliers[idx] + + # soft boost for generators whose kind the column name itself hints at + # (see KEYWORD_GENERATOR_HINTS) - a cheap, complementary signal to the + # statistical evaluation above. Keep the matched keyword per candidate + # so the display can show *why* a boost was applied. + keyword_matches: list[str | None] = [] + for idx, result in enumerate(results): + boost, matched_keyword = keyword_match(column_name, result.proposer.name()) + combined_scores[idx] += boost + keyword_matches.append(matched_keyword) + + # combined_scores is a weighted sum of three [0, 1]-normalized dimensions + # with weights summing to 1.0, so it's bounded to [0, 1] on its own - but + # the keyword boost above is deliberately applied outside that weighted + # sum, and can otherwise push a score above 1. Clamp so "Score" stays a + # comparable, interpretable quantity. (The ChoiceProposer penalty above + # only ever shrinks a score toward 0, so it can't push below the range.) + combined_scores = [max(0.0, min(1.0, s)) for s in combined_scores] + + # A primary key/unique column needs values that are actually unique + # per-row, not just a good statistical fit. If every single candidate + # duplicates enough within its own synthetic sample (see + # synthetic_uniqueness) that it could never satisfy that constraint, + # there is no honest "Recommended: X" to give - naming a winner would + # imply a fitness that doesn't exist and invite a pick that later trips + # a real PRIMARY KEY/UNIQUE violation. This is independent of the + # resample penalty above (which only measures overlap with the *real* + # values): a proposer can have a low copy_fraction and still duplicate + # constantly against itself (e.g. dist_gen.constant). + no_uniqueness_guarantee = ( + is_primary_key + and bool(results) + and all( + result.synthetic_uniqueness < SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD + for result in results + ) + ) + + # pick recommended index: highest combined score across all candidates, + # tiebreaker by crowding distance. This used to restrict the pick to + # Pareto front 1, but the keyword boost and resample penalty above are + # applied after fronts are computed from the raw fidelity/novelty/ + # diversity values - so a technically-dominated candidate can end up + # with a genuinely higher combined_score than a front-1 one (e.g. a + # column-name match earning +0.15). Restricting to front 1 would + # silently override that, defeating the point of adding those signals. + # Front/crowding are still computed and shown per row for transparency + # (which candidates are non-dominated trade-off alternatives vs. + # objectively worse), just no longer used to gate the recommendation. + all_idxs = range(len(combined_scores)) + best_score = max(combined_scores[idx] for idx in all_idxs) + candidates = [idx for idx in all_idxs if combined_scores[idx] == best_score] + if len(candidates) == 1: + recommended_index = candidates[0] + recommended_reason = "best combined score" + else: + best_crowding = max(crowding[idx] for idx in candidates) + selected = [idx for idx in candidates if crowding[idx] == best_crowding] + recommended_index = selected[0] + recommended_reason = "tiebreak by crowding distance" + + # Flag the case where the candidates that actually fit the real data + # best were suppressed by the resample penalty, leaving something that + # was never a good fit to "win" by elimination rather than merit (e.g. + # every high-fidelity resampler on a unique ID column gets zeroed out, + # and dist_gen.constant is left to win despite fitting poorly itself). + # Deliberately narrow: only fires when a *specific* candidate was both + # heavily discounted (< 0.2x) and would otherwise have fit noticeably + # better (> 0.3 higher fidelity) - not just "the winner's score is low", + # which can also just mean a close call among decent options. Also + # requires the winner's own combined score to actually be weak: a + # suppressed resampler's fidelity for a unique-per-row column is + # inflated by the exact memorization the penalty exists to catch, so it + # being nominally higher than a genuinely good winner's (e.g. a keyword- + # matched generic.person.email with strong novelty/diversity) isn't a + # meaningful comparison and shouldn't cast doubt on that pick. Likewise, + # don't fire it when the winner itself is a candidate that guarantees + # fresh values (synthetic_uniqueness >= threshold, e.g. a sequence + # continuing past the observed max): a low combined score there is a + # structural artifact of the fidelity metric penalizing values that + # deliberately fall outside the observed real range, not a sign the + # recommendation is actually shaky - it's exactly the intended, + # principled pick for a column that needs guaranteed-unique values. + weak_recommendation_warning = None + if not no_uniqueness_guarantee and recommended_index is not None: + winner_fidelity = fidelity_scores[recommended_index] + winner_score = combined_scores[recommended_index] + winner_guarantees_uniqueness = ( + results[recommended_index].synthetic_uniqueness + >= SYNTHETIC_UNIQUENESS_GUARANTEE_THRESHOLD + ) + suppressed = [ + idx + for idx in range(len(results)) + if penalty_multipliers[idx] < 0.2 + and fidelity_scores[idx] > winner_fidelity + 0.3 + ] + if ( + suppressed + and winner_score < WEAK_WINNER_SCORE_THRESHOLD + and not winner_guarantees_uniqueness + ): + names = ", ".join(results[idx].proposer.name() for idx in suppressed) + weak_recommendation_warning = ( + f"Note: {names} fit the real data much better but were heavily " + "discounted by the resample penalty (see Penalty column) - reproducing " + "real values almost exactly isn't valid for a unique-per-row column. No " + "remaining candidate is a strong fit here; treat this recommendation as " + "a fallback, not a confident pick." + ) + + # prepare rows with Front and Score, mark Pareto front 1 with coloring. + # Crowding distance is still computed above (used as a tiebreak when + # scores are exactly equal) but isn't shown - it's an NSGA-II internal + # detail (how isolated a candidate is from its front-mates in objective + # space), not a quality signal a user picking a generator can reason + # about, and in practice it's "∞" for most rows anyway. + rows: list[tuple[str, ...]] = [] + for i, result in enumerate(results, start=1): + idx = i - 1 + front = front_of[idx] + cells = [ + str(i), + result.proposer.name(), + profile.name if profile is not None else "UNKNOWN", + str(front), + f"{combined_scores[idx]:.6f}", + keyword_matches[idx] or "", + f"{fidelity_scores[idx]:.6f}", + f"{novelty_scores[idx]:.6f}", + f"{diversity_scores[idx]:.6f}", + f"{real_uniqueness:.3f}", + f"{result.copy_fraction:.3f}", + f"{result.synthetic_uniqueness:.3f}", + f"{penalty_multipliers[idx]:.3f}", + ] + # color entire row for Pareto front 1 + if front == 1 and theme is not None: + cells = [f"{theme.function}{cell}{theme.reset}" for cell in cells] + # keep original types compatible with print_table (it will cast to list) + rows.append(tuple(cells)) + + return ProposalRanking( + recommended_index=recommended_index, + recommended_reason=recommended_reason, + rows=rows, + profile=profile, + weights=(fid_w, nov_w, div_w), + real_uniqueness=real_uniqueness, + penalty_applies_to_all=apply_to_every_proposer, + self_duplication_floored=not (is_primary_key or is_unique_constrained), + fronts=list(front_of), + scores=list(combined_scores), + weak_recommendation_warning=weak_recommendation_warning, + no_uniqueness_guarantee=no_uniqueness_guarantee, + ) + + +def format_ranking_display( + ranking: ProposalRanking, + results: Sequence[ProposalEvaluation], + profile: EvaluationProfile | None, +) -> ProposalRankingDisplay: + """Prepare user-facing strings for a ranked proposal table.""" + fid_w, nov_w, div_w = ranking.weights + recommendation = None + if ranking.no_uniqueness_guarantee: + recommendation = ( + "Recommended: none — no proposer can guarantee uniqueness for this column\n" + ) + elif ranking.recommended_index is not None: + rec_num = ranking.recommended_index + 1 + rec_name = results[ranking.recommended_index].proposer.name() + recommendation = ( + f"Recommended: {rec_num}. {rec_name} — {ranking.recommended_reason}\n" + ) + resample_scope = ( + "every generator (numeric column)" + if ranking.penalty_applies_to_all + else "resamplers only (dist_gen.choice/weighted_choice/zipf_choice)" + ) + self_dup_scope = ( + f"floored at {SELF_DUPLICATION_PENALTY_FLOOR:.1f} (no PK/UNIQUE constraint)" + if ranking.self_duplication_floored + else "full strength (PK/UNIQUE constrained)" + ) + profile_summary = ( + f"Profile: {profile.name if profile is not None else 'UNKNOWN'} | " + f"Real uniqueness: {ranking.real_uniqueness:.3f} | " + "Pareto front 1 rows are highlighted\n" + f"Score = clamp( ({fid_w:.2f}*Fidelity + {nov_w:.2f}*Novelty" + f" + {div_w:.2f}*Diversity + Keyword) " + "x Penalty , 0, 1 )\n" + " Keyword: +0.15 for a column-name hint (see Keyword column)\n" + " Penalty = resample_penalty x self_duplication_penalty" + " (see Copies / Synth.Uniq columns)\n" + f" resample_penalty (1 - real_uniqueness x copy_fraction): {resample_scope}\n" + " self_duplication_penalty (1 - real_uniqueness x (1 - synthetic_uniqueness)):" + f" {self_dup_scope}" + ) + return ProposalRankingDisplay( + recommendation=recommendation, + profile_summary=profile_summary, + rows=ranking.rows, + fronts=ranking.fronts, + weak_recommendation_warning=ranking.weak_recommendation_warning, + no_uniqueness_guarantee=ranking.no_uniqueness_guarantee, + ) diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py new file mode 100644 index 0000000..d40ab49 --- /dev/null +++ b/datafaker/evaluators/statistical_fidelity.py @@ -0,0 +1,297 @@ +"""Per-profile fidelity pipelines that score a proposer against a column.""" + +from dataclasses import dataclass + +from datafaker.evaluators.distribution_builders import ( + CategoryBuilder, + DistributionBuilder, + HistogramBuilder, +) +from datafaker.evaluators.evaluation_profile import EvaluationProfile +from datafaker.evaluators.feature_extractors import ( + CharacterBigramExtractor, + EmailDomainExtractor, + EmailLocalPartExtractor, + EmailTopLevelDomainExtractor, + EmailValidityExtractor, + FeatureExtractor, + FirstLetterExtractor, + IdentityExtractor, + LastLetterExtractor, + LengthExtractor, + SentenceCountExtractor, + TimestampExtractor, + VowelConsonantPatternExtractor, + WeekdayExtractor, + WordCountExtractor, + WordExtractor, +) +from datafaker.evaluators.metrics import JensenShannon, MeanSquaredError, Metric + + +@dataclass +class EvaluationPipeline: + """One named feature-comparison stage within a profile's fidelity score.""" + + name: str + builder: type[DistributionBuilder] + metric: Metric + extractor: FeatureExtractor + weight: float = 1.0 + + +# Short text profile (names, cities, etc.) +SHORT_TEXT_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.25, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="bigrams", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.35, + extractor=CharacterBigramExtractor(), + ), + EvaluationPipeline( + name="first_letter", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.10, + extractor=FirstLetterExtractor(), + ), + EvaluationPipeline( + name="last_letter", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.10, + extractor=LastLetterExtractor(), + ), + EvaluationPipeline( + name="pattern", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=VowelConsonantPatternExtractor(), + ), +] + +EMAIL_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.10, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="local_part", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.40, + extractor=EmailLocalPartExtractor(), + ), + EvaluationPipeline( + name="domain", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.15, + extractor=EmailDomainExtractor(), + ), + EvaluationPipeline( + name="tld", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.15, + extractor=EmailTopLevelDomainExtractor(), + ), + EvaluationPipeline( + name="format_validity", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=EmailValidityExtractor(), + ), +] + +# for country, gender, status, category +CATEGORICAL_PIPELINE = [ + EvaluationPipeline( + name="category", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=1.0, + extractor=IdentityExtractor(), + ) +] + +IDENTIFIER_PIPELINE = [ + EvaluationPipeline( + name="identifier", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=1.0, + extractor=IdentityExtractor(), + ) +] + +# Two pipelines instead of five independent year/month/day/hour dimensions: +# "timestamp" scores year+month+day jointly as one continuous quantity (so a +# generator is judged on the real calendar-position distribution as a whole, +# not on marginal components that can each look fine while their +# combination never does), and "day_of_week" is kept separately since it's a +# cyclic pattern a continuous days-since-epoch value can't capture on its +# own. +TEMPORAL_PIPELINE = [ + EvaluationPipeline( + name="timestamp", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.7, + extractor=TimestampExtractor(), + ), + EvaluationPipeline( + name="day_of_week", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.3, + extractor=WeekdayExtractor(), + ), +] + +# for comments, description, review +FREE_TEXT_PIPELINE = [ + EvaluationPipeline( + name="length", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.10, + extractor=LengthExtractor(), + ), + EvaluationPipeline( + name="word_count", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.25, + extractor=WordCountExtractor(), + ), + EvaluationPipeline( + name="sentence_count", + builder=HistogramBuilder, + metric=MeanSquaredError(), + weight=0.15, + extractor=SentenceCountExtractor(), + ), + EvaluationPipeline( + name="words", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.30, + extractor=WordExtractor(), + ), + EvaluationPipeline( + name="bigrams", + builder=CategoryBuilder, + metric=JensenShannon(), + weight=0.20, + extractor=CharacterBigramExtractor(), + ), +] + +PROFILE_PIPELINES = { + EvaluationProfile.IDENTIFIER: IDENTIFIER_PIPELINE, + EvaluationProfile.SHORT_TEXT: SHORT_TEXT_PIPELINE, + EvaluationProfile.EMAIL: EMAIL_PIPELINE, + EvaluationProfile.CATEGORICAL: CATEGORICAL_PIPELINE, + EvaluationProfile.FREE_TEXT: FREE_TEXT_PIPELINE, + EvaluationProfile.TEMPORAL: TEMPORAL_PIPELINE, +} + + +class StatisticalFidelity: + """ + Measure statistical fidelity of the synthetic sample. + + This class evaluates the statistical fidelity of a synthetic sample + against a real sample by comparing the distributions of features extracted + from the data. It uses a set of evaluation pipelines tailored to different + types of data (e.g., short text, categorical, free text, email, identifier, temporal) + to compute a weighted score that reflects how closely the synthetic data matches the real data + in terms of statistical properties. + """ + + def __init__(self, column, engine, sample_size=4000): + """Initialize with no pipelines set; call set_eval_pipelines() first.""" + self.engine = engine + self.column = column + self.table = self.column.table + self.eval_pipelines = {} + # caps how many real rows each pipeline's builder pulls into memory + # when computing the "real" distribution, so this scales to large + # tables instead of scanning the whole column per pipeline. + self.sample_size = sample_size + # (builder, real distribution) per pipeline name, populated lazily + # the first time calculate_scores runs each pipeline. The real + # distribution depends only on the column, not on which proposer is + # being scored, so it only needs computing once per column rather + # than once per calculate_scores() call (one per candidate proposer). + self._real_distributions = {} + + def set_eval_pipelines(self, profile): + """ + Set evaluation pipelines based on the specified profile. + + :param profile: The evaluation profile to use. + """ + self.eval_pipelines = PROFILE_PIPELINES[profile] + self._real_distributions = {} + + def calculate_scores(self, synthetic_samples): + """ + Calculate the overall fidelity score and each pipeline's own score. + + :param synthetic_samples: The synthetic samples to evaluate. + :return: A tuple of (overall score, {pipeline name: pipeline score}). + """ + pipeline_scores = {} + weighted_score = 0.0 + total_weight = 0.0 + + for pipeline in self.eval_pipelines: + cached = self._real_distributions.get(pipeline.name) + if cached is None: + builder = pipeline.builder( + self.engine, + self.table, + self.column, + extractor=pipeline.extractor, + sample_size=self.sample_size, + ) + # Distribution from real table - independent of the + # proposer being scored, so cache it (and the builder, + # which HistogramBuilder.build_from_values needs state + # from - e.g. self.edges - set by this same call). + real = builder.build_from_table() + self._real_distributions[pipeline.name] = (builder, real) + else: + builder, real = cached + + # Distribution from synthetic values + synthetic = builder.build_from_values(synthetic_samples) + + score = pipeline.metric.compare( + real, + synthetic, + ) + + pipeline_scores[pipeline.name] = score + + weighted_score += pipeline.weight * score + total_weight += pipeline.weight + + overall_score = weighted_score / total_weight if total_weight > 0 else 0.0 + + return overall_score, pipeline_scores diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index c6378d8..a2ca912 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -9,13 +9,26 @@ from typing import Any, Callable, Optional, cast from sqlalchemy import Column, and_, literal_column, select +from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private from datafaker.dialects import Random +from datafaker.evaluators.column_evaluator import ColumnEvaluator +from datafaker.evaluators.proposal_ranking import ( + ProposalRanking, + ProposalRankingDisplay, + format_ranking_display, + rank_proposals, +) from datafaker.interactive.base import DbCmd, TableEntry, fk_column_name, or_default from datafaker.proposers import everything_factory -from datafaker.proposers.base import PredefinedProposer, Proposer -from datafaker.theme import get_active_theme +from datafaker.proposers.base import PredefinedProposer, Proposer, get_column_type +from datafaker.proposers.continuous import ( + GaussianProposer, + LogNormalProposer, + UniformProposer, +) +from datafaker.theme import Theme, get_active_theme from datafaker.utils import ( get_columns_assigned, get_property, @@ -181,6 +194,10 @@ class GeneratorCmd(DbCmd): "{theme_reset}{index}. {theme_func}{name}:" " {theme_fit}{fit} {theme_data}{sample}{theme_reset} ..." ) + RANKED_SAMPLE_TEXT = ( + "{theme_reset}{index}. {theme_func}{name}:" + " {theme_data}{sample}{theme_reset} ..." + ) PRIMARY_PRIVATE_TEXT = "Primary Private" SECONDARY_PRIVATE_TEXT = "Secondary Private on columns {0}" NOT_PRIVATE_TEXT = "Not private" @@ -196,6 +213,11 @@ class GeneratorCmd(DbCmd): ERROR_CANNOT_UNMERGE_ALL = "You cannot unmerge all the generator's columns" PROPOSE_NOTHING = "No proposed generators, sorry." + # Cap on rows shown by default 'propose' (front 1 can otherwise hold more + # candidates than are worth scanning in a terminal table). 'propose all' + # is unaffected. + MAX_PROPOSERS_SHOWN = 10 + SRC_STAT_RE = re.compile( r'\bSRC_STATS\["([^"]+)"\](\["results"\]\[0\]\["([^"]+)"\])?' ) @@ -281,6 +303,9 @@ def __init__( self.proposers: list[Proposer] | None = None self.proposer_index = 0 self.proposers_valid_columns: Optional[tuple[int, list[str]]] = None + # whether the cached self.proposers list currently includes proposers + # that are type-incompatible with the column (e.g. from 'propose all') + self.proposers_include_all: bool = False self.set_prompt() self.roles: dict[str, dict[str, RoleEntry]] = { table_name: { @@ -763,17 +788,61 @@ def _proposers_valid(self) -> bool: self._get_column_names(), ) - def _get_proposer_proposals(self) -> list[Proposer]: - """Get a list of acceptable proposers, sorted by decreasing fit to the actual data.""" - if not self._proposers_valid(): - self.proposers = None - if self.proposers is None: + # Proposers that emit continuous float output (via random.uniform/ + # normalvariate/lognormvariate) with no integer rounding. That's a + # legitimate fit for a Numeric/decimal column, but not for a genuinely + # Integer-typed one: inserting a non-whole float into an Integer column + # is backend-dependent at best (silently stored as-is on SQLite, + # truncated or rejected on MySQL) and raises an error on PostgreSQL. + _CONTINUOUS_FLOAT_PROPOSER_TYPES = ( + GaussianProposer, + UniformProposer, + LogNormalProposer, + ) + + def _is_integer_incompatible( + self, proposer: Proposer, columns: list[Column] + ) -> bool: + """Check whether `proposer` produces float output invalid for an Integer column.""" + if not isinstance(proposer, self._CONTINUOUS_FLOAT_PROPOSER_TYPES): + return False + if len(columns) != 1: + return False + return isinstance(get_column_type(columns[0]), Integer) + + def _get_proposer_proposals( + self, include_all: bool | None = None + ) -> list[Proposer]: + """ + Get a list of acceptable proposers, sorted by decreasing fit to the actual data. + + By default (``include_all=None``), type-incompatible proposers (see + ``_is_integer_incompatible``) are filtered out - this is what + 'propose' shows - and the cached list from the most recent + 'propose'/'propose all' is reused if still valid, so that 'set + ' numbering matches whichever list was last displayed. Pass + an explicit ``True``/``False`` (from 'propose'/'propose all' + themselves) to force that mode, refetching if it differs from what's + cached. + """ + if include_all is None: + include_all = self.proposers_include_all + if ( + not self._proposers_valid() + or self.proposers is None + or include_all != self.proposers_include_all + ): columns = self._column_metadata() props = everything_factory(self.config, self.metadata).get_proposers( columns, self.sync_engine ) + if not include_all: + props = [ + p for p in props if not self._is_integer_incompatible(p, columns) + ] sorted_props = sorted(props, key=lambda g: g.fit(9999)) self.proposers = sorted_props + self.proposers_include_all = include_all self.proposers_valid_columns = ( self.table_index, self._get_column_names().copy(), @@ -978,23 +1047,26 @@ def _get_column_data( result = connection.execute(stmt) return [[to_str(x) for x in xs] for xs in result.all()] - def do_propose(self, _arg: str) -> None: + def do_propose( # pylint: disable=too-many-locals too-many-branches too-many-statements + self, _arg: str + ) -> None: """ Display a list of possible generators for this column. They will be listed in order of fit, the most likely matches first. The results can be compared (against a sample of the real data in the column and against each other) with the 'compare' command. + + By default, generators whose output type is incompatible with the + column (e.g. a continuous-float generator for an Integer column) + are left out. Run 'propose all' to include them anyway. """ theme = get_active_theme() limit = 5 - props = self._get_proposer_proposals() + include_all = _arg.strip().lower() == "all" + props = self._get_proposer_proposals(include_all) sample = self._get_column_data(limit) - if sample: - rep = [x[0] if len(x) == 1 else ",".join(x) for x in sample] - self.print(self.PROPOSE_SOURCE_SAMPLE_TEXT, "; ".join(rep), theme.data) - else: - self.print(self.PROPOSE_SOURCE_EMPTY_TEXT) + self._print_source_sample(sample, theme) if not props: self.print(self.PROPOSE_NOTHING) for index, prop in enumerate(props): @@ -1017,6 +1089,191 @@ def do_propose(self, _arg: str) -> None: theme_reset=theme.reset, ) + column_evaluator = ColumnEvaluator() + columns = self._column_metadata() + results = [] + column_evaluator.setup(columns, self.sync_engine) + for index, proposer in enumerate(props): + result = column_evaluator.evaluate(proposer) + results.append(result) + + self.print("\n***\n") + # self.print("\nProposal evaluation:\n", results) + + profile = getattr(column_evaluator, "profile", None) + column_name = ( + column_evaluator.column.name + if column_evaluator.column is not None + else None + ) + real_uniqueness = getattr(column_evaluator, "real_uniqueness", 0.0) + is_numeric_column = getattr(column_evaluator, "column_is_numeric", False) + is_primary_key = getattr(column_evaluator, "is_primary_key", False) + is_unique_constrained = getattr( + column_evaluator, "is_unique_constrained", False + ) + ranking = rank_proposals( + results, + profile, + theme, + column_name=column_name, + real_uniqueness=real_uniqueness, + is_numeric_column=is_numeric_column, + is_primary_key=is_primary_key, + is_unique_constrained=is_unique_constrained, + ) + display = format_ranking_display(ranking, results, profile) + + if display.recommendation is not None: + if ( + display.no_uniqueness_guarantee + or display.weak_recommendation_warning is not None + ): + # Don't highlight a fallback or non-existent pick - coloring + # it the same way as a solid recommendation would visually + # contradict the caveat that follows (or, for + # no_uniqueness_guarantee, claim confidence in a pick that + # doesn't exist at all). + self.print(display.recommendation) + else: + self.print(f"{theme.column}{display.recommendation}{theme.reset}") + if display.weak_recommendation_warning is not None: + self.print(display.weak_recommendation_warning) + if include_all and ranking.recommended_index is not None: + # 'propose all' deliberately bypasses the type-compatibility filter, + # so a generator whose output type doesn't match the column (e.g. a + # continuous-float generator for an Integer column) can still win - + # the statistical score has no way to detect that mismatch at all, + # which is exactly why the default 'propose' excludes it structurally + # instead of trying to penalize it. + recommended_proposer = results[ranking.recommended_index].proposer + if self._is_integer_incompatible(recommended_proposer, columns): + self.print( + "Warning: {0} produces output whose type doesn't match this" + " column - it's excluded from the default 'propose' list for" + " exactly this reason (run 'propose' without 'all'). The" + " statistics can't detect this mismatch, so a high score here" + " doesn't mean it's safe to use.", + recommended_proposer.name(), + ) + + self.print(display.profile_summary) + + indices_to_show = self._select_indices_to_show(display, ranking, include_all) + rows_to_show = [display.rows[i] for i in indices_to_show] + + self.print_table( + [ + "#", + "Proposer", + "Profile", + "Front", + "Score", + "Keyword", + "Fidelity", + "Novelty", + "Diversity", + "Uniqueness", + "Copies", + "Synth.Uniq", + "Penalty", + ], + rows_to_show, + ) + + self.print("\n") + self._print_source_sample(sample, theme) + for i in indices_to_show: + prop = props[i] + self.print( + self.RANKED_SAMPLE_TEXT, + index=i + 1, + name=prop.name(), + sample="; ".join(map(repr, prop.generate_data(limit))), + theme_func=theme.function, + theme_data=theme.data, + theme_reset=theme.reset, + ) + + print("\n\n") + + def _print_source_sample(self, sample: list[list[str]], theme: Theme) -> None: + """Print a sample of the real column values, or a note that there are none.""" + if sample: + rep = [x[0] if len(x) == 1 else ",".join(x) for x in sample] + self.print(self.PROPOSE_SOURCE_SAMPLE_TEXT, "; ".join(rep), theme.data) + else: + self.print(self.PROPOSE_SOURCE_EMPTY_TEXT) + + def _select_indices_to_show( + self, + display: ProposalRankingDisplay, + ranking: ProposalRanking, + include_all: bool, + ) -> list[int]: + """Pick which candidate rows to show in the propose table. + + By default, only show Pareto front 1: a front-2+ candidate is, by + definition, strictly worse than some front-1 one on fidelity, + novelty AND diversity simultaneously, so it adds nothing to a + shortlist. 'propose all' bypasses this (and the type-compatibility + filter) to show every candidate. The returned indices let the + sample listing after the table map each shown row back to its + proposer without parsing the (possibly theme-colored) '#' cell text. + """ + indices_to_show = list(range(len(display.rows))) + if include_all: + return indices_to_show + + indices_to_show = [i for i in indices_to_show if display.fronts[i] == 1] + # The recommendation is picked by combined score across *all* + # candidates, not just front 1 (see rank_proposals) - a + # dominated candidate can still win once the resample penalty + # and keyword boost are applied, since those are computed after + # fronts are derived from the raw fidelity/novelty/diversity + # values. Force it into the table even when front-2+, otherwise + # "Recommended: N. x" can point at a row the user never sees. + recommended_outside_front_1 = ( + ranking.recommended_index is not None + and ranking.recommended_index not in indices_to_show + ) + if recommended_outside_front_1: + indices_to_show.append(ranking.recommended_index) + capped = len(indices_to_show) > self.MAX_PROPOSERS_SHOWN + if capped: + # Keep the highest-scoring candidates - front 1 alone can + # still hold more rows than are worth scanning in a + # terminal table (nothing dominates them, but they're not + # all equally worth showing). Always keep the recommendation + # even if its score wouldn't otherwise make the cut, since + # it's referenced by name right above this table. + ranked = sorted( + indices_to_show, key=lambda i: ranking.scores[i], reverse=True + ) + keep = set(ranked[: self.MAX_PROPOSERS_SHOWN]) + if ranking.recommended_index is not None: + keep.add(ranking.recommended_index) + indices_to_show = sorted(keep) + else: + indices_to_show.sort() + if len(indices_to_show) < len(display.rows): + if capped: + qualifier = ( + f" (Pareto front 1, top {self.MAX_PROPOSERS_SHOWN} by score," + " plus the recommendation)" + ) + elif recommended_outside_front_1: + qualifier = " (Pareto front 1, plus the recommendation)" + else: + qualifier = " (Pareto front 1 only)" + self.print( + "Showing {0} of {1} candidates" + qualifier + "." + " Run 'propose all' to see the rest.", + len(indices_to_show), + len(display.rows), + ) + return indices_to_show + def do_p(self, arg: str) -> None: """Synonym for propose.""" self.do_propose(arg) diff --git a/datafaker/proposers/__init__.py b/datafaker/proposers/__init__.py index 1a333c3..bee4c53 100644 --- a/datafaker/proposers/__init__.py +++ b/datafaker/proposers/__init__.py @@ -30,6 +30,7 @@ NullPartitionedLogNormalProposerFactory, NullPartitionedNormalProposerFactory, ) +from datafaker.proposers.sequence import IncrementProposerFactory def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: @@ -57,4 +58,5 @@ def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: NullPartitionedLogNormalProposerFactory(config, metadata), DateAfterProposerFactory(config, metadata), DateComponentExtractProposerFactory(config, metadata), + IncrementProposerFactory(), ) diff --git a/datafaker/proposers/sequence.py b/datafaker/proposers/sequence.py new file mode 100644 index 0000000..99beb0e --- /dev/null +++ b/datafaker/proposers/sequence.py @@ -0,0 +1,91 @@ +"""Proposer for a monotonically-incrementing sequence past the observed max. + +This is the only kind of candidate that can actually guarantee fresh, unique +values for a genuine integer primary key: it can never collide with the real +data (every value it emits is strictly greater than any observed value) and +it can never collide with itself (it only ever increments). It deliberately +reuses ``generic.column_value_provider.increment`` - the same mechanism +``make.py``'s own default generator assignment already uses for integer +primary keys (see ``_integer_generator``) - rather than inventing a new +runtime function, so a column left on this proposer's recommendation behaves +identically to the framework's own established default. +""" + +from collections.abc import Sequence +from typing import Any + +from sqlalchemy import Column, Engine, func, select +from sqlalchemy.types import Integer + +from datafaker.proposers.base import Proposer, ProposerFactory, get_column_type + + +class IncrementProposer(Proposer): + """Generator continuing a sequence past the real data's observed maximum.""" + + def __init__(self, engine: Engine, column: Column): + """Initialise an IncrementProposer.""" + super().__init__() + self.engine = engine + self.column = column + self.table = column.table + + def function_name(self) -> str: + """Get the name of the generator function to call.""" + return "generic.column_value_provider.increment" + + def nominal_kwargs(self) -> dict[str, str]: + """Get the arguments to be entered into ``config.yaml``. + + These match the expressions ``make.py``'s own default integer + primary key generator emits (see ``_integer_generator``): not + SRC_STATS references, but Python expressions the generation + pipeline evaluates directly against the destination database and + its metadata at generation time. + """ + return { + "db_connection": "dst_db_conn", + "column": f'metadata.tables["{self.table.name}"].columns["{self.column.name}"]', + } + + def actual_kwargs(self) -> dict[str, Any]: + """Get the kwargs (summary statistics) this generator was instantiated with.""" + return {"column": f"{self.table.name}.{self.column.name}"} + + def generate_data(self, count: int) -> list[Any]: + """Generate ``count`` random data points for this column. + + For interactive preview purposes only: continues past the *source* + database's observed maximum. At actual generation time, the + ``nominal_kwargs`` above instead continue past whatever's already in + the *destination* database, via ``db_connection``/``column``. + """ + with self.engine.connect() as connection: + row = connection.execute(select(func.max(self.column))).first() + start = 0 if row is None or row[0] is None else row[0] + return [start + i + 1 for i in range(count)] + + +class IncrementProposerFactory(ProposerFactory): + """Propose an incrementing-sequence generator for integer primary keys. + + Deliberately narrow: only a genuine integer primary key (and one that + isn't also a foreign key - those should copy the referenced table's + values, not synthesize their own, matching ``make.py``'s own default + generator logic) gets this proposer. A non-key numeric column doesn't + need guaranteed-fresh values, and a non-integer key (a string/UUID + primary key) needs a different mechanism entirely - out of scope here. + """ + + def get_proposers( + self, columns: list[Column], engine: Engine + ) -> Sequence[Proposer]: + """Get the generators appropriate to these columns.""" + if len(columns) != 1: + return [] + column = columns[0] + if not column.primary_key or column.foreign_keys: + return [] + if not isinstance(get_column_type(column), Integer): + return [] + return [IncrementProposer(engine, column)] diff --git a/docs/source/builtin_generators.rst b/docs/source/builtin_generators.rst new file mode 100644 index 0000000..91ddf3b --- /dev/null +++ b/docs/source/builtin_generators.rst @@ -0,0 +1,679 @@ +Built-in generators and proposers +================================== + +This page is a reference for the generator functions that ship with ``datafaker``, +and for the *proposers* that suggest which of those functions (and which arguments) +to use for a given column. If you just want a quick pointer from "my column +looks like this" to "try this generator", see :doc:`choosing_a_generator` +instead. If you want to write your own generators, see +:doc:`custom_generators`. + +Generators vs. proposers +------------------------- + +A **generator** is the actual callable that produces fake values, referenced by name +in ``config.yaml`` (for example ``dist_gen.normal`` or ``generic.person.first_name``). +Generators are simple: given some arguments (often summary statistics pulled from +``src-stats.yaml``), they return one random value per call. + +A **proposer** is an internal ``datafaker`` object, not something you reference +directly in ``config.yaml``. When you run ``configure-generators`` and use the +``propose`` command on a column, ``datafaker`` runs every applicable proposer against +that column (and, for multi-column proposers, against that group of columns). Each +proposer: + +* decides whether it applies at all, based on the column's SQL type and (for some + proposers) properties of the source data or ``config.yaml``, +* works out what summary queries need to be added to ``src-stats.yaml`` (via + ``select_aggregate_clauses`` or ``custom_queries``), +* works out the generator function name and keyword arguments to write into + ``config.yaml`` if you ``set`` it. + +``set`` simply writes the winning proposer's generator name and arguments into +``config.yaml``; from then on, only the underlying generator function is used --- +the proposer itself is not needed again for that column. + +``propose`` now shows two different views of the same candidates: first the +plain, per-proposer ``(fit: ...)`` list described above, and then a second, +ranked table produced by a separate statistical evaluation pipeline (see +:ref:`evaluating-and-ranking-proposals` below) that scores every candidate on +fidelity, novelty and diversity and names a ``Recommended`` generator. The two +scores are independent and can disagree; the ranked table and its +recommendation are the more reliable of the two for most columns. + +Default generators assigned automatically +------------------------------------------ + +Before you run ``propose``/``set`` at all, ``configure-generators`` (via +``make.py``) already assigns a default generator to every column, purely from its +SQL type (and whether it is a foreign key or primary key). ``propose`` lets you +replace this default with something that better matches the real data. + +.. list-table:: Default generator by SQL column type + :widths: 25 45 30 + :header-rows: 1 + + * - SQL type + - Default generator + - Notes + * - Foreign key column (any type) + - ``generic.column_value_provider.column_value`` + - Picks a random existing value from the referenced column in the destination + database. + * - Integer primary key + - ``generic.column_value_provider.increment`` + - Continues counting up from the highest existing value in the destination + table; see :ref:`the increment proposer ` below for how + this is also offered as an explicit proposal. + * - Other ``Integer`` + - ``generic.numeric.integer_number`` + - + * - ``Numeric`` (decimal/float) + - ``generic.numeric.float_number`` + - If the column has a fixed ``scale``, the range is set to fit within that + many decimal digits. + * - ``String`` + - ``generic.person.password``, or ``generic.text.color`` if the column has no + maximum length + - The password length is set to the column's maximum length so the value + always fits. + * - ``Boolean`` + - ``generic.development.boolean`` + - + * - ``Date`` + - ``generic.datetime.date`` + - + * - ``DateTime`` + - ``generic.datetime.datetime`` + - + * - ``LargeBinary`` + - ``generic.bytes_provider.bytes`` + - + * - ``Uuid`` / PostgreSQL ``UUID`` / MS-SQL ``UNIQUEIDENTIFIER`` + - ``generic.cryptographic.uuid`` + - + * - Anything else unsupported + - ``generic.null_provider.null`` + - ``datafaker`` logs a warning; you should configure a real generator for + this column by hand. + +Built-in proposers +------------------- + +Everything below is registered in ``datafaker.proposers.everything_factory`` and is +tried automatically for every column (or, for the multi-column proposers, every +group of columns you select) when you run ``propose``. + +Single-column value proposers (Mimesis) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These proposers wrap `Mimesis `_ generator functions +(``generic.*``, using the ``en_GB`` locale) that produce a plausible but +unrelated fake value, with no reference to the real data's actual distribution. +Fit is estimated by comparing the length (for strings) or value (for numbers) of +generated samples against buckets built from the real column. + +.. list-table:: + :widths: 20 60 20 + :header-rows: 1 + + * - Applies to + - Generators proposed + - Notes + * - ``String`` + - Every name in :ref:`the Mimesis string generator list ` + (``generic.``) + - If the column has a maximum length, each candidate is wrapped in + ``dist_gen.truncated_string`` so the output never overflows the column. + * - ``Numeric`` + - ``generic.person.height`` + - + * - ``Numeric`` or ``Integer`` + - ``generic.person.weight`` + - + * - ``Date`` + - ``generic.datetime.date`` + - Range (``start``/``end`` years) is taken from the earliest/latest years + actually found in the column. + * - ``DateTime`` + - ``generic.datetime.datetime`` + - Same year-range behaviour as ``Date``. + * - ``Time`` + - ``generic.datetime.time`` + - + +.. _mimesis-string-list: + +The full list of Mimesis string generators tried for ``String`` columns: +``address.calling_code``, ``address.city``, ``address.continent``, +``address.country``, ``address.country_code``, ``address.postal_code``, +``address.province``, ``address.street_number``, ``address.street_name``, +``address.street_suffix``, ``person.blood_type``, ``person.email``, +``person.first_name``, ``person.last_name``, ``person.full_name``, +``person.gender``, ``person.language``, ``person.nationality``, +``person.occupation``, ``person.password``, ``person.title``, +``person.university``, ``person.username``, ``person.worldview``, +``text.answer``, ``text.color``, ``text.level``, ``text.quote``, +``text.sentence``, ``text.text``, ``text.word``. + +Continuous distribution proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These proposers fit a single numeric column to a well-known distribution, using +the column's actual mean and standard deviation (queried into ``src-stats.yaml`` +as ``mean__`` / ``stddev__``, or their log equivalents). + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Numeric`` or ``Integer`` + - ``dist_gen.normal`` + - Gaussian distribution with the observed mean and standard deviation. + * - ``Numeric`` or ``Integer`` + - ``dist_gen.uniform_ms`` + - Uniform distribution with the same mean and standard deviation as the real + data (rather than explicit min/max bounds). + * - ``Numeric`` or ``Integer``, values ``> 0`` + - ``dist_gen.lognormal`` + - Log-normal distribution, fitted to the mean/standard deviation of the logs + of the (positive) values. + +Choice proposers +^^^^^^^^^^^^^^^^^ + +Used for columns with a manageable number of distinct values (up to 500). Each of +the three distribution shapes below is proposed both for **all** distinct values +and, separately, for a version with rare values (seen 7 or fewer times) suppressed +--- and again for both of those computed from a random sample of up to 500 rows, +for large tables where scanning every row would be too slow. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Generator + - Function name + - Distribution + * - Uniform choice + - ``dist_gen.choice`` + - Every distinct value equally likely. + * - Zipf choice + - ``dist_gen.zipf_choice`` + - Values ranked by real-world frequency; the *n*\ th most common value is + chosen ``1/n`` as often as the most common one (a Zipf/power-law shape, + without needing to store every count). + * - Weighted choice + - ``dist_gen.weighted_choice`` + - Reproduces the real frequency of each value directly (stores a count + alongside each value in ``src-stats.yaml``). + +Constant proposer +^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Any single nullable column, or ``String``/``Numeric``/``Integer`` + - ``dist_gen.constant`` + - Always returns the same value: ``None`` if the column is nullable, + otherwise ``""``, ``0.0`` or ``0`` depending on type. Useful as a + placeholder, or a baseline to compare fit against. + +.. _increment-proposer: + +Sequence proposer +^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Integer primary key that is **not** also a foreign key + - ``generic.column_value_provider.increment`` + - Counts up past the highest value already present, guaranteeing fresh, + unique values. This is the same mechanism used as the automatic default + for integer primary keys (see above); proposing it explicitly just makes + it visible and comparable alongside other candidates. + +Multivariate proposers +^^^^^^^^^^^^^^^^^^^^^^^^ + +Offered when you select **two or more** numeric columns together (so that +``datafaker`` can preserve correlations between them, rather than generating each +column independently). A covariate (means/covariance) matrix is queried from the +source data. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - 2+ ``Numeric``/``Integer`` columns + - ``dist_gen.multivariate_normal`` + - Multivariate Gaussian distribution over all the selected columns. + * - 2+ ``Numeric``/``Integer`` columns, values ``> 0`` + - ``dist_gen.multivariate_lognormal`` + - Multivariate log-normal distribution (covariates computed on the logs of + the values). + +Null-partitioned multivariate proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A more powerful variant of the multivariate proposers above, for a group of +columns (numeric and/or categorical) that don't all follow the same pattern of +missing (``NULL``) values. The source data is split into partitions, one per +distinct combination of which columns in the group are ``NULL``; each partition +gets its own covariate matrix (or, if it has too few rows, is treated as a +suppressed group), and at generation time a partition is picked at random +(weighted by how common it was in the source data) before that partition's +distribution is sampled. + +.. list-table:: + :widths: 20 25 55 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - Group of columns with mixed nullability + - ``dist_gen.alternatives`` wrapping ``dist_gen.grouped_multivariate_normal`` + - Shown as *"null-partitioned grouped_multivariate_normal"* in ``propose``. + * - Group of columns with mixed nullability, values ``> 0`` + - ``dist_gen.alternatives`` wrapping ``dist_gen.grouped_multivariate_lognormal`` + - Shown as *"null-partitioned grouped_multivariate_lognormal"*. + * - (both of the above) + - *[sampled and suppressed]* variants + - Computed from a random sample of the source table with rare partitions + suppressed, for large tables. + +Date interval ("anchored date") proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Propose a date/datetime as an offset from another date, rather than independently. +This only appears once at least one ``Date``/``DateTime`` column in the same +table (or a directly related table) has been given the ``start`` role --- for +example an admission date. Every *other* date/datetime column in scope (an +implicit "end", such as a discharge date) is then offered a proposal anchored +to it; the anchor column itself is not offered a self-referential proposal. + +Roles are set per column and stored in ``config.yaml`` under +``tables..columns..roles``. The easiest way to set one is the +``role`` command inside ``configure-generators``: + +.. code-block:: shell + + (admission.start_date) role set start + (admission.start_date) role list + start + (discharge.end_date) role on admission.start_date list + +``role list`` shows the roles on the current column, ``role set `` / +``role delete `` add or remove one, and adding ``on `` (or +just ``on `` for a column in the current table) targets a different +column without navigating to it first. Like generator changes, role changes +are only written to ``config.yaml`` when you ``quit`` and confirm --- ``quit`` +will list any pending role changes alongside pending generator changes before +asking you to save. + +Two roles currently exist: ``start`` (consumed by this proposer, as described +above) and ``source`` (reserved for future use --- no built-in proposer reads +it yet). You can still set either role by hand-editing ``config.yaml`` instead +of using ``role``, if you prefer; both routes update the same field. + +.. list-table:: + :widths: 20 30 50 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Date``/``DateTime`` column with an anchor column in the *same* table + - ``generic.anchored_provider.normal_date`` + - Adds a clamped, normally-distributed number of seconds (mean/standard + deviation taken from the real interval lengths) to the anchor column's + generated value; never earlier than the anchor. The sample values shown + by ``propose``/``compare`` are built from real anchor values sampled + from the source database (recycled if fewer than requested), rather + than a single fixed dummy anchor, so the preview reflects realistic + intervals. + * - ``Date``/``DateTime`` column anchored to a column in a *related* table + (via a foreign key) + - ``generic.anchored_provider.normal_date_fk`` + - Same idea, but looks up the anchor value from the related row in the + destination database at generation time. + +Date component extraction proposers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Propose deriving a column's value from a ``DateTime`` column already generated +elsewhere in the same row, rather than generating it independently -- useful when, +for example, a ``year_of_birth`` column should always agree with a +``date_of_birth`` column. + +.. list-table:: + :widths: 20 30 50 + :header-rows: 1 + + * - Applies to + - Generator proposed + - Notes + * - ``Date``, based on a ``DateTime`` column in the same table + - ``generic.extract_provider.date`` + - Takes just the date part of the referenced ``DateTime`` value. + * - ``Integer``, based on a ``DateTime`` column in the same table + - ``generic.extract_provider.year``, ``generic.extract_provider.month``, + ``generic.extract_provider.day`` + - One proposal per component, per ``DateTime`` column found in the table. + +.. _evaluating-and-ranking-proposals: + +Evaluating and ranking proposals +---------------------------------- + +For a single-column ``propose``, every candidate proposer that survives the +type-compatibility filter (see below) is additionally scored by +``datafaker.evaluators.ColumnEvaluator`` and ranked by +``datafaker.evaluators.proposal_ranking.rank_proposals``. This produces a +table shown by ``propose``, with a ``Recommended: N. `` line +above it. + +Evaluation profile +^^^^^^^^^^^^^^^^^^^ + +The column is first classified into one ``EvaluationProfile``, based on a +sample of up to 4000 real rows: + +.. list-table:: + :widths: 25 75 + :header-rows: 1 + + * - Profile + - How it's chosen + * - ``TEMPORAL`` + - Column is ``Date``/``DateTime``/``Time``. + * - ``CATEGORICAL`` or ``IDENTIFIER`` + - Column is ``Numeric``/``Integer``: ``CATEGORICAL`` if fewer than 20% of + the sampled values are distinct (a status code, a small foreign key + range), otherwise ``IDENTIFIER``. + * - ``EMAIL`` + - Column is a string and more than half the sampled non-empty values + look like an email address (contain ``@``, with a non-empty local part + and a domain containing a ``.``). + * - ``SHORT_TEXT`` + - Column is a string, not email-like, average length under 30 + characters and more than 80% of sampled values are distinct. + * - ``CATEGORICAL`` + - Column is a string, not email-like or short-text, and fewer than 20% + of sampled values are distinct. + * - ``FREE_TEXT`` + - Remaining string columns with an average length over 50 characters or + a high space ratio (multi-word content). + * - ``SHORT_TEXT`` + - Any remaining string column (the fallback). + +Three scoring dimensions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each candidate is scored, from a 4000-value synthetic sample, on three +independent axes, each normalized to ``[0, 1]`` across the candidates being +compared (so these are *relative* scores, not absolute ones): + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Axis + - Meaning + * - **Fidelity** + - How closely the synthetic sample's distribution matches the real + column's, measured by a profile-specific pipeline of feature + extractors run through ``MeanSquaredError`` or ``JensenShannon`` + (see :ref:`statistical-fidelity-pipelines` below), then inverted so + higher is better. + * - **Novelty** + - The fraction of synthetic values (case/whitespace-normalized) that do + *not* already appear among the real sampled values. 1.0 means every + synthetic value is new. + * - **Diversity** + - How closely the synthetic sample's own internal variety (normalized + Shannon entropy over distinct values) matches the real sample's --- not + "more diverse is better", but "as diverse as the real data". + +.. _statistical-fidelity-pipelines: + +Fidelity pipelines by profile +"""""""""""""""""""""""""""""" + +Fidelity is computed by one of six fixed pipelines, chosen by profile, each a +weighted combination of feature extractors compared via ``MeanSquaredError`` +(on a length/count histogram) or ``JensenShannon`` divergence (on a category +distribution): + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Profile + - Pipeline (feature: weight) + * - ``IDENTIFIER`` + - identifier (value histogram): 1.0 + * - ``CATEGORICAL`` + - category (exact value): 1.0 + * - ``SHORT_TEXT`` + - length: 0.25, character bigrams: 0.35, first letter: 0.10, + last letter: 0.10, vowel/consonant pattern: 0.20 + * - ``EMAIL`` + - length: 0.10, local part: 0.40, domain: 0.15, top-level domain: 0.15, + address format validity: 0.20 + * - ``FREE_TEXT`` + - length: 0.10, word count: 0.25, sentence count: 0.15, words: 0.30, + character bigrams: 0.20 + * - ``TEMPORAL`` + - year+month+day, as one joint quantity: 0.7, day-of-week: 0.3 + +Combining the scores, penalties and a keyword hint +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The three normalized scores are combined into one ``Score`` per candidate as + +.. code-block:: text + + Score = clamp( (fid_w * Fidelity + nov_w * Novelty + div_w * Diversity + Keyword) * Penalty, 0, 1 ) + +* **Weights** (``fid_w``/``nov_w``/``div_w``) depend on the profile: 0.85 / + 0.05 / 0.10 for ``IDENTIFIER`` (fidelity dominates); 0.7 / 0.15 / 0.15 for + ``CATEGORICAL``, ``SHORT_TEXT``, ``EMAIL`` and ``TEMPORAL``; 0.5 / 0.3 / 0.2 + for ``FREE_TEXT`` (open-ended text tolerates, even rewards, novelty). +* **Keyword** is a flat ``+0.15`` boost when the column's own name hints at a + particular kind of value (e.g. a column named ``first_name`` or + ``customer_first_name`` boosts any ``person.first_name``-based candidate; + see ``KEYWORD_GENERATOR_HINTS`` in ``proposal_ranking.py`` for the full + list) -- a cheap, complementary signal used only to break near-ties, never + to override a clearly better statistical result. +* **Penalty** is the product of two independent factors, each scaled by how + unique the real column actually is: + + .. code-block:: text + + Penalty = resample_penalty * self_duplication_penalty + + * ``resample_penalty = 1 - real_uniqueness * copy_fraction`` is a privacy + safeguard against reproducing real values verbatim. ``copy_fraction`` is + the fraction of a candidate's synthetic sample that exactly matches a + real value. It only applies to choice-style proposers + (``dist_gen.choice``/``weighted_choice``/``zipf_choice``) -- or, for a + numeric column, *any* proposer -- since resampling real values is + expected and harmless for a low-uniqueness column (a gender, a status) + but a genuine leak for a near-unique one (an email, a real ID). It has no + effect (``1.0``) on other proposers for non-numeric columns, where an + overlap with real values is usually just a naturally shared vocabulary + (e.g. common first names), not memorization. + * ``self_duplication_penalty = 1 - real_uniqueness * (1 - synthetic_uniqueness)`` + guards against a candidate duplicating against its *own* output -- + ``synthetic_uniqueness`` is the fraction of distinct values in its 4000 + synthetic samples. This is a different failure mode from + ``resample_penalty``: a candidate can have a tiny fixed pool of possible + outputs (e.g. a canned-quote generator with only a few dozen distinct + strings) without ever coincidentally matching a *real* value, yet still + be unusable for a column that needs many distinct rows. It's computed + for every candidate, but only applied at full strength when the column + actually needs unique values (a primary key or a ``UNIQUE``-constrained + column) -- otherwise it's floored at ``0.4``, discounting a + self-duplicating-but-otherwise-plausible candidate rather than crushing + its score to near zero over a constraint that doesn't apply here. The + ``Synth.Uniq`` column in the ``propose`` table shows the raw value this + factor is computed from. + +Candidates are also grouped into Pareto fronts (front 1 = not +strictly dominated on fidelity, novelty *and* diversity simultaneously by any +other candidate); ``propose`` (without ``all``) shows only front 1, capped at +10 rows by ``Score`` (always keeping the recommended candidate even if it +would otherwise be cut), and tells you how many candidates were hidden. Run +``propose all`` to see every candidate --- including proposers whose output +type doesn't actually match the column (for example a continuous-float +proposer against an ``Integer`` column, invalid on most databases), which the +plain ``propose`` excludes structurally rather than relying on the score to +catch. + +The **recommendation** is simply the candidate with the highest ``Score`` +(ties broken by NSGA-II crowding distance, i.e. preferring a candidate that's +less redundant with its front-mates). Two caveats are surfaced when they +apply: + +* For a primary key or unique column, if *every* candidate's own synthetic + sample is duplicated enough that it couldn't satisfy a uniqueness + constraint, ``propose`` recommends nothing at all rather than a false + positive --- look for the sequence proposer instead (see + :ref:`increment-proposer` above). +* If the candidates that fit the real data best were heavily discounted by + the resample penalty above, leaving a poorer-fitting candidate to "win" by + elimination, ``propose`` prints a warning naming the discounted candidates + and flags the recommendation as a fallback rather than a confident pick. + +Generator function reference +------------------------------ + +The tables above name the generator functions that proposers can select. This +section documents each one, grouped by the Mimesis provider it belongs to, for +when you want to reference or combine them by hand in ``config.yaml`` (see +:doc:`custom_generators`). + +``dist_gen`` -- ``datafaker.providers.DistributionProvider`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Function + - Description + * - ``uniform(low, high)`` + - Uniform distribution between explicit bounds. + * - ``uniform_ms(mean, sd)`` + - Uniform distribution with a given mean and standard deviation. + * - ``normal(mean, sd)`` + - Gaussian (normal) distribution. + * - ``lognormal(logmean, logsd)`` + - Log-normal distribution. + * - ``choice(a)`` / ``choice_direct(a)`` + - Uniform choice between a list of values. ``choice`` takes + ``{"value": ...}`` dicts (as stored in ``src-stats.yaml``); + ``choice_direct`` takes plain values. + * - ``zipf_choice(a, n=None)`` / ``zipf_choice_direct(a, n=None)`` + - Choice following a Zipf distribution over values ranked most-to-least + frequent. + * - ``weighted_choice(a)`` + - Choice weighted by an explicit ``count`` stored alongside each value. + * - ``constant(value)`` + - Always returns ``value``. + * - ``multivariate_normal(cov)`` / ``multivariate_lognormal(cov)`` + - Draws a list of correlated values from a covariate matrix (means and + covariances, keyed as ``mN``/``cN_M``). + * - ``grouped_multivariate_normal(covs)`` / ``grouped_multivariate_lognormal(covs)`` + - As above, but first picks one covariate matrix from a list, weighted by + each group's ``count``. Used for the null-partitioned proposers. + * - ``alternatives(alternative_configs, counts=None)`` + - Picks between other named generators, weighted by count; this is how the + null-partitioned proposers choose a missingness pattern before delegating + to ``grouped_multivariate_normal``/``grouped_multivariate_lognormal``. + * - ``with_constants_at(constants_at, subgen, params)`` + - Runs another generator and splices fixed values into the result list at + given positions (used to reinsert ``NULL``/category columns alongside + generated numeric ones). + * - ``truncated_string(subgen_fn, params, length)`` + - Runs a string-producing generator and truncates the result to ``length`` + characters. + +``generic`` -- Mimesis providers (locale ``en_GB``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Most of ``generic.address.*``, ``generic.person.*``, ``generic.text.*``, +``generic.datetime.*`` and ``generic.numeric.*`` come straight from the +`Mimesis library `_ and aren't +listed again here; see :ref:`the Mimesis string generator list +` above for the ones ``propose`` tries automatically, +plus ``person.height``, ``person.weight``, ``datetime.date``, +``datetime.datetime`` and ``datetime.time``. + +``datafaker`` adds the following extra Mimesis providers of its own +(``datafaker/providers.py``): + +.. list-table:: + :widths: 30 70 + :header-rows: 1 + + * - Function + - Description + * - ``column_value_provider.column_value(db_connection, orm_class, column_name)`` + - Returns a random existing value from another column -- the default + generator for foreign key columns. + * - ``column_value_provider.increment(db_connection, column)`` + - Returns a number one higher than the previous call for this column, + starting one above the highest existing value in the destination + database -- the default generator for integer primary keys. + * - ``bytes_provider.bytes()`` + - Random binary data. + * - ``timedelta_provider.timedelta(...)`` + - Random ``datetime.timedelta`` value. + * - ``timespan_provider.timespan(...)`` + - Random pair of dates/times forming a span. + * - ``weighted_boolean_provider.bool(probability)`` + - ``True`` with the given probability. + * - ``sql_group_by_provider.sample(...)`` + - Samples rows grouped and filtered according to SQL-like criteria. + * - ``null_provider.null()`` + - Always returns ``None``. Used as the last-resort default for column + types ``datafaker`` doesn't otherwise recognise. + * - ``extract_provider.year(extract_from)`` / ``month(...)`` / ``day(...)`` / ``date(...)`` + - Pulls a component out of a ``datetime`` produced elsewhere in the same + row (see the date component extraction proposers above). + * - ``anchored_provider.normal_date(mean_seconds, sd_seconds, anchor)`` + - A date offset from ``anchor`` in the same table by a clamped normally + distributed number of seconds. + * - ``anchored_provider.normal_date_fk(dst_db_conn, mean_seconds, sd_seconds, table, on_column, anchor_row, anchor_column)`` + - As above, but ``anchor`` is looked up from a related table via a foreign + key at generation time. + +See also +--------- + +* :doc:`choosing_a_generator` -- a quick lookup from column shape to + generator, for when you want more than the ``propose`` recommendation. +* :doc:`custom_generators` -- writing your own row generators and story + generators. +* :doc:`quickstart` and :doc:`introduction` -- walkthroughs of + ``configure-generators``, ``propose``, ``compare`` and ``set`` in action. +* :doc:`glossary` diff --git a/docs/source/choosing_a_generator.rst b/docs/source/choosing_a_generator.rst new file mode 100644 index 0000000..a47c2dc --- /dev/null +++ b/docs/source/choosing_a_generator.rst @@ -0,0 +1,123 @@ +Choosing a generator for a column +================================== + +This page is a quick decision aid for the ``propose`` command: "my column looks +like *this* -- which generator should I expect, and why?" For the full list of +generator functions and proposers, and how ``propose`` scores and ranks them, +see :doc:`builtin_generators`. + +Start here +---------- + +In almost all cases, just run ``propose`` on the column and take the +``Recommended`` generator -- it already scores every applicable candidate on +fidelity, novelty and diversity against your real data, which is more +reliable than guessing from the column's type alone. The table below is for +the two situations where you need more than the recommendation: + +* you want to sanity-check *why* a particular generator was recommended, or +* the recommendation looks wrong (or none was given) and you want to know + what else is worth trying. + +.. list-table:: + :widths: 30 40 30 + :header-rows: 1 + + * - Your column looks like... + - Try this generator + - Why + * - A unique ID, e.g. an integer primary key + - ``generic.column_value_provider.increment`` + - Guarantees fresh, unique values by counting up past the highest + existing one; ``propose`` offers this explicitly and it's also the + automatic default for integer primary keys. See + :ref:`the sequence proposer `. + * - A foreign key to another table + - ``generic.column_value_provider.column_value`` + - Picks a random existing value from the referenced column, so + referential integrity is preserved. This is the automatic default for + any foreign key column. + * - A near-unique value that must look real but not *be* real, e.g. an + email address or free-text ID + - Whichever ``generic.*`` value proposer ``propose`` recommends + - ``propose`` applies an extra privacy penalty here: a candidate that + reproduces real values verbatim is scored down in proportion to how + unique the real column is, so the recommendation already accounts for + leak risk. See :ref:`evaluating-and-ranking-proposals`. + * - A small set of repeating values, e.g. a status code, category or flag + - ``dist_gen.weighted_choice`` (matches real frequencies exactly), or + ``dist_gen.zipf_choice`` / ``dist_gen.choice`` for a looser fit + - These are the *choice proposers*, offered for any column with up to + 500 distinct values; they differ in how closely they copy the real + frequency of each value. ``weighted_choice`` stores and reproduces + the exact real frequency of every value -- the closest fit, but the + most exposing of how common each value really is. ``zipf_choice`` + instead only ranks values from most- to least-common and assigns + each a Zipf/power-law frequency (the *n*\ th most common value is + chosen ``1/n`` as often as the most common one), which approximates + a typical real-world skew without storing per-value counts. + ``choice`` drops frequency information entirely and picks uniformly + at random -- the loosest fit, but the simplest and most private + option if flattening the distribution is acceptable. + * - A continuous measurement, e.g. a length, weight or price + - ``dist_gen.normal``, ``dist_gen.lognormal`` (if all values are + positive and skewed), or ``dist_gen.uniform_ms`` + - The *continuous distribution proposers* fit directly to the real + column's mean and standard deviation. + * - Two or more numeric columns that vary together, e.g. width and height + - ``dist_gen.multivariate_normal`` / ``dist_gen.multivariate_lognormal`` + - Select all the columns together (``merge``) before running + ``propose`` so it can offer these -- they preserve the correlation + between columns rather than generating each independently. + * - The same, but some rows have one column ``NULL`` and others don't + - the *null-partitioned* generators, e.g. ``dist_gen.alternatives`` + wrapping ``dist_gen.grouped_multivariate_normal`` + - Real data is split into partitions by which columns are ``NULL``; + each partition keeps its own correlation, and one is picked at + generation time weighted by how common it was in the source data. + * - A recognisable human-ish string, e.g. a first name, city or username + - The matching ``generic.person.*`` / ``generic.address.*`` / + ``generic.text.*`` generator + - ``propose`` gives a flat ``+0.15`` boost when the column's *name* + hints at its content (e.g. a column called ``first_name`` boosts + ``generic.person.first_name``) -- a tie-breaker, not an override of a + clearly better statistical fit. + * - A date/time that should stay consistent with another date on the same + row, e.g. a ``discharge_date`` after an ``admission_date`` + - ``generic.anchored_provider.normal_date`` (same table) or + ``.normal_date_fk`` (related table via foreign key) + - Only offered once the earlier date has been given the ``start`` role + (``role set start``). The offset is a clamped, normally distributed + number of seconds taken from the real interval lengths, so the + generated date is never earlier than its anchor. + * - A column that should just agree with a ``datetime`` column elsewhere + in the row, e.g. ``year_of_birth`` alongside ``date_of_birth`` + - ``generic.extract_provider.date`` / ``.year`` / ``.month`` / ``.day`` + - Derives the value from the other column instead of generating it + independently, so the two can never disagree. + * - Nothing else fits, or you just want a placeholder + - ``dist_gen.constant`` + - Always returns the same value (``None`` if the column is nullable). + Also useful as a baseline to compare other candidates' fit against. + +If a column doesn't match any row above, it's still worth running +``propose`` -- the ranking in :doc:`builtin_generators` covers cases (and +combinations of ``String``/``Numeric``/``Date`` type with real-data shape) +that are awkward to summarise as a simple lookup table. + +When ``propose`` recommends nothing +------------------------------------ + +For a primary key or other unique column, ``propose`` will withhold a +recommendation entirely if every candidate's synthetic sample duplicates too +often to satisfy uniqueness -- rather than confidently suggesting something +that would break a constraint. Look for the sequence proposer +(``generic.column_value_provider.increment``) instead. + +See also +--------- + +* :doc:`builtin_generators` -- full generator and proposer reference, plus + how the ``Recommended`` score is computed. +* :doc:`quickstart` -- walkthrough of ``propose``, ``compare`` and ``set`` in + the ``configure-generators`` CLI. diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 87b9889..a80edb6 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -11,6 +11,12 @@ Glossary - A user-defined Python function which will provide one or more random column values for a single table when called. * - Story generator - A user-defined Python generator function that ``yields`` rows, possibly multiple rows for multiple tables. + * - Generator (function) + - A callable, referenced by name in ``config.yaml`` (e.g. ``dist_gen.normal``), that produces one random value (or tuple of values) per call. See :doc:`builtin_generators` for the ones built into `datafaker`, or :doc:`custom_generators` to write your own. + * - Proposer + - An internal `datafaker` object, used by the ``propose`` command of ``configure-generators``, that suggests a generator function and arguments for a column (or group of columns) and estimates how well it fits the real data. See :doc:`builtin_generators`. + * - Role + - A tag (``start`` or ``source``) set on a column, via the ``role`` command in ``configure-generators`` or by hand in ``config.yaml``, that some proposers use to find related columns -- for example, the ``start`` role marks an anchor date that other dates in the same table can be generated as an offset from. See :doc:`builtin_generators`. * - Destination database and destination schema - A database and a schema within that database where `datafaker` creates the synthetic data tables and inserts the synthetic data it generates. * - Source database and source schema diff --git a/docs/source/index.rst b/docs/source/index.rst index f411c60..1233728 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -36,7 +36,9 @@ Contents: orm configuration health_data + Choosing a Generator Custom Generators + Built-in Generators and Proposers api .. toctree:: diff --git a/docs/source/tutorial_parquet.rst b/docs/source/tutorial_parquet.rst index d28d951..205e4ae 100644 --- a/docs/source/tutorial_parquet.rst +++ b/docs/source/tutorial_parquet.rst @@ -219,7 +219,7 @@ and source statistics: datafaker configure-tables datafaker configure-generators - datafaker configure-missingness + datafaker configure-missing datafaker make-stats This creates: @@ -328,7 +328,7 @@ Generate configuration: datafaker configure-tables datafaker configure-generators - datafaker configure-missingness + datafaker configure-missing datafaker make-stats Create schema and generate data: @@ -356,7 +356,10 @@ For a minimal end-to-end workflow: export DST_DSN=duckdb:///./fake.db datafaker make-tables --parquet-dir ./input_parquet - + datafaker configure-tables + datafaker configure-generators + datafaker configure-missing + datafaker make-stats datafaker create-tables datafaker create-data --num-passes 10 diff --git a/tests/test_evaluators_column_evaluator.py b/tests/test_evaluators_column_evaluator.py new file mode 100644 index 0000000..61e8cab --- /dev/null +++ b/tests/test_evaluators_column_evaluator.py @@ -0,0 +1,233 @@ +"""Unit tests for datafaker.evaluators.column_evaluator.""" +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.column_evaluator import ( + ColumnEvaluator, + ColumnStats, + EvaluationProfile, + ProposalEvaluation, + analyse_column, + choose_numeric_profile, + choose_profile, + looks_like_email, +) +from datafaker.proposers.base import ConstantProposer +from tests.utils import DatafakerTestCase + + +class AnalyseColumnTests(DatafakerTestCase): + """Test case for analyse_column.""" + + def test_counts_rows_and_unique_values(self) -> None: + """row_count and unique_count reflect the (non-None) input values.""" + stats = analyse_column(["a", "b", "a"]) + self.assertEqual(3, stats.row_count) + self.assertEqual(2, stats.unique_count) + + def test_none_values_are_excluded(self) -> None: + """None entries don't count towards row_count or unique_count.""" + stats = analyse_column(["a", None, None]) + self.assertEqual(1, stats.row_count) + self.assertEqual(1, stats.unique_count) + + def test_avg_length_and_ratios(self) -> None: + """avg_length, space_ratio, digit_ratio and punctuation_ratio are computed.""" + stats = analyse_column(["ab 1!"]) + self.assertEqual(5, stats.avg_length) + self.assertAlmostEqual(1 / 5, stats.space_ratio) + self.assertAlmostEqual(1 / 5, stats.digit_ratio) + self.assertAlmostEqual(1 / 5, stats.punctuation_ratio) + + def test_empty_input_does_not_divide_by_zero(self) -> None: + """An empty (or all-None) column yields all-zero stats, not an error.""" + stats = analyse_column([None, None]) + self.assertEqual(0, stats.row_count) + self.assertEqual(0, stats.avg_length) + self.assertEqual(0, stats.space_ratio) + self.assertEqual(0, stats.digit_ratio) + self.assertEqual(0, stats.punctuation_ratio) + + def test_non_string_values_are_stringified(self) -> None: + """Non-string values (e.g. ints) are converted to str first.""" + stats = analyse_column([12345]) + self.assertEqual(5, stats.avg_length) + self.assertEqual(1.0, stats.digit_ratio) + + +class ColumnStatsTests(DatafakerTestCase): + """Test case for the ColumnStats dataclass.""" + + def test_uniqueness_is_unique_over_row_count(self) -> None: + """uniqueness divides unique_count by row_count.""" + stats = ColumnStats( + row_count=4, + unique_count=2, + avg_length=0, + space_ratio=0, + digit_ratio=0, + punctuation_ratio=0, + ) + self.assertEqual(0.5, stats.uniqueness) + + def test_uniqueness_with_zero_rows_does_not_divide_by_zero(self) -> None: + """A zero row_count is floored to 1 to avoid a ZeroDivisionError.""" + stats = ColumnStats( + row_count=0, + unique_count=0, + avg_length=0, + space_ratio=0, + digit_ratio=0, + punctuation_ratio=0, + ) + self.assertEqual(0.0, stats.uniqueness) + + +class ChooseProfileTests(DatafakerTestCase): + """Test case for choose_profile.""" + + def _stats( + self, avg_length=10.0, uniqueness_row=(8, 10), space_ratio=0.0 + ) -> ColumnStats: + unique_count, row_count = uniqueness_row + return ColumnStats( + row_count=row_count, + unique_count=unique_count, + avg_length=avg_length, + space_ratio=space_ratio, + digit_ratio=0.0, + punctuation_ratio=0.0, + ) + + def test_short_and_mostly_unique_is_short_text(self) -> None: + """Short, mostly-distinct values look like names/identifiers.""" + stats = self._stats(avg_length=10, uniqueness_row=(9, 10)) + self.assertEqual(EvaluationProfile.SHORT_TEXT, choose_profile(stats)) + + def test_low_uniqueness_is_categorical(self) -> None: + """Values repeated often enough look like a small category set.""" + stats = self._stats(avg_length=10, uniqueness_row=(1, 100)) + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_profile(stats)) + + def test_long_average_length_is_free_text(self) -> None: + """Long values that aren't near-unique look like free text.""" + stats = self._stats(avg_length=60, uniqueness_row=(5, 10)) + self.assertEqual(EvaluationProfile.FREE_TEXT, choose_profile(stats)) + + def test_high_space_ratio_is_free_text(self) -> None: + """Many spaces per character looks like prose even if average length is short.""" + stats = self._stats(avg_length=10, uniqueness_row=(5, 10), space_ratio=0.2) + self.assertEqual(EvaluationProfile.FREE_TEXT, choose_profile(stats)) + + def test_fallback_is_short_text(self) -> None: + """Anything not matching the other rules falls back to SHORT_TEXT.""" + stats = self._stats(avg_length=35, uniqueness_row=(5, 10), space_ratio=0.05) + self.assertEqual(EvaluationProfile.SHORT_TEXT, choose_profile(stats)) + + +class ChooseNumericProfileTests(DatafakerTestCase): + """Test case for choose_numeric_profile.""" + + def test_low_cardinality_is_categorical(self) -> None: + """Few distinct values among many rows looks like a status/flag column.""" + values = [1, 2] * 10 # 2 distinct out of 20: uniqueness 0.1 + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_numeric_profile(values)) + + def test_high_cardinality_is_identifier(self) -> None: + """Mostly-distinct numeric values look like an id/age/salary column.""" + values = list(range(100)) + self.assertEqual(EvaluationProfile.IDENTIFIER, choose_numeric_profile(values)) + + def test_all_none_defaults_to_identifier(self) -> None: + """An empty/all-None column can't be judged, so defaults to IDENTIFIER.""" + self.assertEqual( + EvaluationProfile.IDENTIFIER, choose_numeric_profile([None, None]) + ) + + def test_none_values_are_excluded_from_the_ratio(self) -> None: + """None entries don't count as either distinct or total values.""" + # 2 distinct values among 20 non-null entries: uniqueness 0.1. If the + # Nones were counted in the denominator without being excluded first, + # this would look even more categorical, not less - the point here is + # just that the Nones don't crash or skew the ratio unexpectedly. + values = [1] * 19 + [2] + [None] * 5 + self.assertEqual(EvaluationProfile.CATEGORICAL, choose_numeric_profile(values)) + + +class LooksLikeEmailTests(DatafakerTestCase): + """Test case for looks_like_email.""" + + def test_mostly_email_values_is_true(self) -> None: + """A column where most values look like emails is detected.""" + values = ["a@example.com", "b@example.com", "not-an-email"] + self.assertTrue(looks_like_email(values)) + + def test_mostly_non_email_values_is_false(self) -> None: + """A column where most values don't look like emails is rejected.""" + values = ["a@example.com", "plain text", "another plain value"] + self.assertFalse(looks_like_email(values)) + + def test_empty_input_is_false(self) -> None: + """No values at all can't look like emails.""" + self.assertFalse(looks_like_email([])) + self.assertFalse(looks_like_email([None, " "])) + + def test_value_without_local_part_or_domain_is_not_email_like(self) -> None: + """'@' alone, or with an empty side, doesn't count as email-like.""" + values = ["@example.com", "a@", "@"] + self.assertFalse(looks_like_email(values)) + + +class ProposalEvaluationTests(DatafakerTestCase): + """Test case for the ProposalEvaluation dataclass's display string.""" + + def test_str_includes_key_metrics_and_pipeline_scores(self) -> None: + """The string form surfaces the headline scores and per-pipeline detail.""" + evaluation = ProposalEvaluation( + proposer=ConstantProposer("x"), + novelty=0.5, + diversity=0.25, + overall_score=0.125, + pipeline_scores={"length": 0.1, "words": 0.2}, + copy_fraction=0.0, + synthetic_uniqueness=1.0, + ) + text = str(evaluation) + self.assert_str_in("dist_gen.constant", text) + self.assert_str_in("0.125000", text) + self.assert_str_in("0.500000", text) + self.assert_str_in("0.250000", text) + self.assert_str_in("length", text) + self.assert_str_in("words", text) + + +class ColumnEvaluatorIntegrationTests(DatafakerTestCase): + """End-to-end test of ColumnEvaluator against a real (DuckDB) table.""" + + def test_evaluate_a_constant_proposer_against_a_categorical_column(self) -> None: + """setup() profiles the column and evaluate() scores a real proposer.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table( + "statuses", + metadata, + Column("id", Integer, primary_key=True), + Column("status", String), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute( + table.insert(), + [{"id": i, "status": ["active", "inactive"][i % 2]} for i in range(20)], + ) + + evaluator = ColumnEvaluator() + evaluator.setup([table.c.status], engine) + self.assertEqual(EvaluationProfile.CATEGORICAL, evaluator.profile) + + result = evaluator.evaluate(ConstantProposer("active")) + self.assertIsInstance(result, ProposalEvaluation) + self.assertEqual(1.0, result.copy_fraction) + # A constant proposer emits the same value every time: only one + # distinct value out of the whole synthetic sample. + self.assertLess(result.synthetic_uniqueness, 0.01) + self.assertIn("category", result.pipeline_scores) diff --git a/tests/test_evaluators_distribution_builders.py b/tests/test_evaluators_distribution_builders.py new file mode 100644 index 0000000..4b79e82 --- /dev/null +++ b/tests/test_evaluators_distribution_builders.py @@ -0,0 +1,201 @@ +"""Unit tests for datafaker.evaluators.distribution_builders.""" +from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.distribution_builders import ( + CategoryBuilder, + Distribution, + HistogramBuilder, +) +from datafaker.evaluators.feature_extractors import ( + FirstLetterExtractor, + LengthExtractor, +) +from tests.utils import DatafakerTestCase + + +class DistributionTests(DatafakerTestCase): + """Test case for the Distribution dataclass.""" + + def test_vocabulary_is_the_set_of_keys(self) -> None: + """vocabulary exposes the keys of the probability mapping.""" + dist = Distribution({"a": 0.5, "b": 0.5}) + self.assertEqual({"a", "b"}, dist.vocabulary) + + def test_probability_of_known_key(self) -> None: + """probability() returns the stored value for a known key.""" + dist = Distribution({"a": 0.75}) + self.assertEqual(0.75, dist.probability("a")) + + def test_probability_of_unknown_key_is_zero(self) -> None: + """probability() returns 0.0 for a key that isn't in the distribution.""" + dist = Distribution({"a": 0.75}) + self.assertEqual(0.0, dist.probability("missing")) + + def test_as_vector_orders_by_given_vocabulary(self) -> None: + """as_vector() maps a vocabulary list to matching probabilities, in order.""" + dist = Distribution({"a": 0.2, "b": 0.8}) + self.assertEqual([0.8, 0.2, 0.0], dist.as_vector(["b", "a", "c"])) + + +def _make_duckdb_table(rows: list[dict]) -> tuple: + """Create an in-memory DuckDB engine with one populated table.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table( + "people", + metadata, + Column("age", Integer), + Column("name", String), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute(table.insert(), rows) + return engine, table + + +class HistogramBuilderBuildFromTableTests(DatafakerTestCase): + """Test case for HistogramBuilder.build_from_table.""" + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """The resulting histogram is a valid probability distribution.""" + engine, table = _make_duckdb_table( + [{"age": 20 + (i % 10), "name": "x"} for i in range(100)] + ) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertIsNotNone(builder.mean) + self.assertIsNotNone(builder.stddev) + + def test_empty_table_yields_empty_distribution(self) -> None: + """An empty table produces an empty distribution rather than an error.""" + engine, table = _make_duckdb_table([]) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertEqual({}, dist.probabilities) + + def test_constant_column_does_not_crash_on_zero_stddev(self) -> None: + """A column with a single repeated value has zero stddev but still works.""" + engine, table = _make_duckdb_table( + [{"age": 42, "name": "x"} for _ in range(10)] + ) + builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertEqual(0.0, builder.stddev) + + def test_uses_feature_extractor_expression(self) -> None: + """The configured extractor's SQL expression is used, not the raw column.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": name} for name in ["ab", "cde", "fghi"] * 5] + ) + builder = HistogramBuilder( + engine, table, table.c.name, extractor=LengthExtractor(), sample_size=1000 + ) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + + +class HistogramBuilderBuildFromValuesTests(DatafakerTestCase): + """Test case for HistogramBuilder.build_from_values.""" + + def setUp(self) -> None: + super().setUp() + engine, table = _make_duckdb_table( + [{"age": 20 + (i % 10), "name": "x"} for i in range(100)] + ) + self.builder = HistogramBuilder(engine, table, table.c.age, sample_size=1000) + # Learn edges from the real data first, as calculate_scores() does. + self.builder.build_from_table() + + def test_result_sums_to_one(self) -> None: + """Bucketed synthetic values form a valid probability distribution.""" + dist = self.builder.build_from_values([20, 21, 22, 23, 24, 25]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + + def test_empty_values_yield_empty_distribution(self) -> None: + """No synthetic values produces an empty distribution.""" + dist = self.builder.build_from_values([]) + self.assertEqual({}, dist.probabilities) + + def test_values_outside_edges_are_clamped_to_end_buckets(self) -> None: + """Values far below/above the learned range land in the outer buckets.""" + dist = self.builder.build_from_values([-1000, 1000]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + buckets = set(dist.probabilities.keys()) + assert self.builder.edges is not None + self.assertTrue(all(0 <= b <= len(self.builder.edges) - 2 for b in buckets)) + + +class CategoryBuilderBuildFromTableTests(DatafakerTestCase): + """Test case for CategoryBuilder.build_from_table.""" + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """The resulting category histogram sums to (approximately) 1.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": n} for n in ["alice", "bob", "carol"] * 10] + ) + builder = CategoryBuilder(engine, table, table.c.name, sample_size=1000) + dist = builder.build_from_table() + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertEqual({"alice", "bob", "carol"}, dist.vocabulary) + + def test_empty_table_yields_empty_distribution(self) -> None: + """An empty table produces an empty distribution.""" + engine, table = _make_duckdb_table([]) + builder = CategoryBuilder(engine, table, table.c.name, sample_size=1000) + dist = builder.build_from_table() + self.assertEqual({}, dist.probabilities) + + def test_uses_feature_extractor(self) -> None: + """The category counts are keyed by extracted features, not raw values.""" + engine, table = _make_duckdb_table( + [{"age": 0, "name": n} for n in ["Alice", "Anna", "Bob"]] + ) + builder = CategoryBuilder( + engine, + table, + table.c.name, + extractor=FirstLetterExtractor(), + sample_size=1000, + ) + dist = builder.build_from_table() + self.assertEqual({"a", "b"}, dist.vocabulary) + self.assertAlmostEqual(2 / 3, dist.probability("a")) + self.assertAlmostEqual(1 / 3, dist.probability("b")) + + +class CategoryBuilderBuildFromValuesTests(DatafakerTestCase): + """Test case for CategoryBuilder.build_from_values.""" + + def setUp(self) -> None: + super().setUp() + self.builder = CategoryBuilder( + engine=None, table=None, column=None, sample_size=1000 + ) + + def test_builds_a_distribution_that_sums_to_one(self) -> None: + """Category counts from a plain Python list sum to one.""" + dist = self.builder.build_from_values(["a", "a", "b"]) + self.assertAlmostEqual(1.0, sum(dist.probabilities.values())) + self.assertAlmostEqual(2 / 3, dist.probability("a")) + self.assertAlmostEqual(1 / 3, dist.probability("b")) + + def test_none_values_are_ignored(self) -> None: + """None entries don't contribute counts.""" + dist = self.builder.build_from_values(["a", None, "a"]) + self.assertEqual({"a": 1.0}, dist.probabilities) + + def test_no_extracted_features_yields_sentinel_missing_distribution(self) -> None: + """An empty result set is represented with an explicit __MISSING__ token.""" + dist = self.builder.build_from_values([None, None]) + self.assertEqual({"__MISSING__": 1.0}, dist.probabilities) + + def test_uses_feature_extractor(self) -> None: + """Values are converted through the configured extractor before counting.""" + builder = CategoryBuilder( + engine=None, table=None, column=None, extractor=LengthExtractor() + ) + dist = builder.build_from_values(["ab", "cd", "efg"]) + self.assertAlmostEqual(2 / 3, dist.probability(2)) + self.assertAlmostEqual(1 / 3, dist.probability(3)) diff --git a/tests/test_evaluators_feature_extractors.py b/tests/test_evaluators_feature_extractors.py new file mode 100644 index 0000000..014b9fa --- /dev/null +++ b/tests/test_evaluators_feature_extractors.py @@ -0,0 +1,458 @@ +"""Unit tests for datafaker.evaluators.feature_extractors.""" +from datetime import date, datetime, timezone + +from datafaker.evaluators.feature_extractors import ( + CharacterBigramExtractor, + CharacterExtractor, + CharacterTrigramExtractor, + EmailDomainExtractor, + EmailLocalPartExtractor, + EmailTopLevelDomainExtractor, + EmailValidityExtractor, + FirstLetterExtractor, + IdentityExtractor, + LastLetterExtractor, + LengthExtractor, + PrefixExtractor, + SentenceCountExtractor, + SuffixExtractor, + TimestampExtractor, + VowelConsonantPatternExtractor, + WeekdayExtractor, + WordCountExtractor, + WordExtractor, +) +from tests.utils import DatafakerTestCase + + +class IdentityExtractorTests(DatafakerTestCase): + """Test case for IdentityExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = IdentityExtractor() + + def test_yields_the_value_unchanged(self) -> None: + """Non-None values pass through untouched.""" + self.assertEqual([42], list(self.extractor.extract(42))) + self.assertEqual(["hello"], list(self.extractor.extract("hello"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_expression_returns_the_column_itself(self) -> None: + """The SQL expression is just the column, unmodified.""" + column = object() + self.assertIs(column, self.extractor.expression(column)) + + +class FirstLetterExtractorTests(DatafakerTestCase): + """Test case for FirstLetterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = FirstLetterExtractor() + + def test_extracts_lowercased_first_letter(self) -> None: + """The first character is extracted and lowercased.""" + self.assertEqual(["a"], list(self.extractor.extract("Alice"))) + + def test_strips_surrounding_whitespace_first(self) -> None: + """Leading whitespace does not become the 'first letter'.""" + self.assertEqual(["a"], list(self.extractor.extract(" alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + def test_expression_does_not_raise(self) -> None: + """A SQL expression is implemented for this extractor.""" + self.extractor.expression("col") + + +class LastLetterExtractorTests(DatafakerTestCase): + """Test case for LastLetterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = LastLetterExtractor() + + def test_extracts_lowercased_last_letter(self) -> None: + """The last character is extracted and lowercased.""" + self.assertEqual(["e"], list(self.extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class LengthExtractorTests(DatafakerTestCase): + """Test case for LengthExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = LengthExtractor() + + def test_extracts_string_length(self) -> None: + """The length of str(value) is extracted.""" + self.assertEqual([5], list(self.extractor.extract("Alice"))) + + def test_extracts_length_of_non_string_values(self) -> None: + """Non-string values are stringified before measuring length.""" + self.assertEqual([len(str(12345))], list(self.extractor.extract(12345))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class WordExtractorTests(DatafakerTestCase): + """Test case for WordExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WordExtractor() + + def test_tokenizes_and_lowercases_words(self) -> None: + """Words are split on non-word characters and lowercased.""" + self.assertEqual( + ["the", "quick", "fox"], + list(self.extractor.extract("The quick, fox!")), + ) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_no_sql_expression_is_implemented(self) -> None: + """Word extraction has no SQL equivalent and must raise.""" + with self.assertRaises(NotImplementedError): + self.extractor.expression("col") + + +class CharacterExtractorTests(DatafakerTestCase): + """Test case for CharacterExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterExtractor() + + def test_extracts_each_character(self) -> None: + """Each character of the value is yielded separately.""" + self.assertEqual(["a", "b", "c"], list(self.extractor.extract("abc"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_no_sql_expression_is_implemented(self) -> None: + """Per-character extraction has no SQL equivalent and must raise.""" + with self.assertRaises(NotImplementedError): + self.extractor.expression("col") + + +class CharacterBigramExtractorTests(DatafakerTestCase): + """Test case for CharacterBigramExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterBigramExtractor() + + def test_extracts_overlapping_bigrams(self) -> None: + """Adjacent character pairs are extracted, lowercased.""" + self.assertEqual(["ab", "bc"], list(self.extractor.extract("ABc"))) + + def test_short_strings_yield_no_bigrams(self) -> None: + """A string shorter than 2 characters has no bigrams.""" + self.assertEqual([], list(self.extractor.extract("a"))) + self.assertEqual([], list(self.extractor.extract(""))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class CharacterTrigramExtractorTests(DatafakerTestCase): + """Test case for CharacterTrigramExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = CharacterTrigramExtractor() + + def test_extracts_overlapping_trigrams(self) -> None: + """Adjacent character triples are extracted, lowercased.""" + self.assertEqual(["abc", "bcd"], list(self.extractor.extract("ABCd"))) + + def test_short_strings_yield_no_trigrams(self) -> None: + """A string shorter than 3 characters has no trigrams.""" + self.assertEqual([], list(self.extractor.extract("ab"))) + + +class PrefixExtractorTests(DatafakerTestCase): + """Test case for PrefixExtractor.""" + + def test_default_length_is_two(self) -> None: + """With no explicit length, a 2-character prefix is extracted.""" + extractor = PrefixExtractor() + self.assertEqual(["al"], list(extractor.extract("Alice"))) + + def test_custom_length(self) -> None: + """A custom prefix length is honored.""" + extractor = PrefixExtractor(length=4) + self.assertEqual(["alic"], list(extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(PrefixExtractor().extract(None))) + + def test_empty_string_yields_nothing(self) -> None: + """A blank/whitespace-only string produces no features.""" + self.assertEqual([], list(PrefixExtractor().extract(" "))) + + +class SuffixExtractorTests(DatafakerTestCase): + """Test case for SuffixExtractor.""" + + def test_default_length_is_two(self) -> None: + """With no explicit length, a 2-character suffix is extracted.""" + extractor = SuffixExtractor() + self.assertEqual(["ce"], list(extractor.extract("Alice"))) + + def test_custom_length(self) -> None: + """A custom suffix length is honored.""" + extractor = SuffixExtractor(length=3) + self.assertEqual(["ice"], list(extractor.extract("Alice"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(SuffixExtractor().extract(None))) + + +class VowelConsonantPatternExtractorTests(DatafakerTestCase): + """Test case for VowelConsonantPatternExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = VowelConsonantPatternExtractor() + + def test_extracts_vowel_consonant_pattern(self) -> None: + """Each alphabetic character is classified as V or C.""" + self.assertEqual(["VCVCV"], list(self.extractor.extract("Alice"))) + + def test_non_alphabetic_characters_are_skipped(self) -> None: + """Digits and punctuation don't appear in the pattern.""" + self.assertEqual(["VCC"], list(self.extractor.extract("a1-b2c!"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_all_non_alphabetic_yields_nothing(self) -> None: + """A value with no letters at all produces no pattern.""" + self.assertEqual([], list(self.extractor.extract("123!!"))) + + +class TimestampExtractorTests(DatafakerTestCase): + """Test case for TimestampExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = TimestampExtractor() + + def test_epoch_datetime_extracts_zero(self) -> None: + """The Unix epoch itself extracts to day 0.""" + self.assertEqual([0.0], list(self.extractor.extract(datetime(1970, 1, 1)))) + + def test_extracts_days_since_epoch(self) -> None: + """A later date extracts to the correct day offset.""" + self.assertEqual([1.0], list(self.extractor.extract(datetime(1970, 1, 2)))) + + def test_accepts_date_objects(self) -> None: + """A plain date (no time component) is also supported.""" + self.assertEqual([1.0], list(self.extractor.extract(date(1970, 1, 2)))) + + def test_accepts_iso_format_strings(self) -> None: + """A string that came from a query losing type info is coerced.""" + self.assertEqual([1.0], list(self.extractor.extract("1970-01-02"))) + + def test_invalid_string_yields_nothing(self) -> None: + """A string that isn't a valid date produces no feature.""" + self.assertEqual([], list(self.extractor.extract("not-a-date"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_non_date_non_string_value_yields_nothing(self) -> None: + """A value with no date meaning at all (e.g. an int) is dropped.""" + self.assertEqual([], list(self.extractor.extract(42))) + + def test_timezone_aware_datetime_is_normalized(self) -> None: + """Timezone info is stripped rather than causing a crash or a shift.""" + aware = datetime(1970, 1, 2, tzinfo=timezone.utc) + self.assertEqual([1.0], list(self.extractor.extract(aware))) + + +class WeekdayExtractorTests(DatafakerTestCase): + """Test case for WeekdayExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WeekdayExtractor() + + def test_extracts_iso_weekday_index(self) -> None: + """1970-01-01 was a Thursday: weekday() == 3.""" + self.assertEqual([3], list(self.extractor.extract(datetime(1970, 1, 1)))) + + def test_accepts_iso_format_strings(self) -> None: + """String dates are coerced the same way as TimestampExtractor.""" + self.assertEqual([3], list(self.extractor.extract("1970-01-01"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_invalid_string_yields_nothing(self) -> None: + """An unparseable string produces no feature.""" + self.assertEqual([], list(self.extractor.extract("nope"))) + + +class EmailLocalPartExtractorTests(DatafakerTestCase): + """Test case for EmailLocalPartExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailLocalPartExtractor() + + def test_extracts_lowercased_local_part(self) -> None: + """The portion before '@' is extracted and lowercased.""" + self.assertEqual( + ["alice.smith"], list(self.extractor.extract("Alice.Smith@Example.com")) + ) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no local part.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + +class EmailDomainExtractorTests(DatafakerTestCase): + """Test case for EmailDomainExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailDomainExtractor() + + def test_extracts_lowercased_domain(self) -> None: + """The portion after '@' is extracted and lowercased.""" + self.assertEqual( + ["example.com"], list(self.extractor.extract("alice@Example.COM")) + ) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no domain.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + +class EmailTopLevelDomainExtractorTests(DatafakerTestCase): + """Test case for EmailTopLevelDomainExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailTopLevelDomainExtractor() + + def test_extracts_lowercased_tld(self) -> None: + """The final domain segment is extracted and lowercased.""" + self.assertEqual( + ["com"], list(self.extractor.extract("alice@mail.example.COM")) + ) + + def test_domain_without_dot_yields_nothing(self) -> None: + """A domain with no TLD separator produces no feature.""" + self.assertEqual([], list(self.extractor.extract("alice@localhost"))) + + def test_value_without_at_sign_yields_nothing(self) -> None: + """A non-email string has no TLD.""" + self.assertEqual([], list(self.extractor.extract("not-an-email"))) + + +class EmailValidityExtractorTests(DatafakerTestCase): + """Test case for EmailValidityExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = EmailValidityExtractor() + + def test_well_formed_email_is_valid(self) -> None: + """A syntactically valid email is classified 'valid'.""" + self.assertEqual(["valid"], list(self.extractor.extract("alice@example.com"))) + + def test_malformed_value_is_invalid(self) -> None: + """A string with no '@' or domain is classified 'invalid'.""" + self.assertEqual(["invalid"], list(self.extractor.extract("not-an-email"))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class WordCountExtractorTests(DatafakerTestCase): + """Test case for WordCountExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = WordCountExtractor() + + def test_counts_words(self) -> None: + """Word count matches the number of \\w+ tokens.""" + self.assertEqual([4], list(self.extractor.extract("The quick brown fox."))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) + + +class SentenceCountExtractorTests(DatafakerTestCase): + """Test case for SentenceCountExtractor.""" + + def setUp(self) -> None: + super().setUp() + self.extractor = SentenceCountExtractor() + + def test_counts_sentences_by_terminal_punctuation(self) -> None: + """Sentences are split on ./!/? and counted if they contain a word.""" + self.assertEqual( + [3], list(self.extractor.extract("Hi there! How are you? I am fine.")) + ) + + def test_trailing_punctuation_without_words_is_not_a_sentence(self) -> None: + """A run of terminal punctuation with no word content doesn't count.""" + self.assertEqual([1], list(self.extractor.extract("Hello there..."))) + + def test_none_yields_nothing(self) -> None: + """None produces no features.""" + self.assertEqual([], list(self.extractor.extract(None))) + + def test_blank_string_yields_nothing(self) -> None: + """An empty/whitespace-only string produces no features.""" + self.assertEqual([], list(self.extractor.extract(" "))) diff --git a/tests/test_evaluators_metrics.py b/tests/test_evaluators_metrics.py new file mode 100644 index 0000000..bf4610b --- /dev/null +++ b/tests/test_evaluators_metrics.py @@ -0,0 +1,163 @@ +"""Unit tests for datafaker.evaluators.metrics.""" +import math + +from datafaker.evaluators.distribution_builders import Distribution +from datafaker.evaluators.metrics import ( + DiversityMetric, + JensenShannon, + MeanSquaredError, + NoveltyMetric, +) +from tests.utils import DatafakerTestCase + + +class MeanSquaredErrorTests(DatafakerTestCase): + """Test case for the MeanSquaredError metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = MeanSquaredError() + + def test_identical_distributions_score_zero(self) -> None: + """Two identical distributions should have zero error.""" + dist = Distribution({"a": 0.5, "b": 0.5}) + self.assertEqual(0.0, self.metric.compare(dist, dist)) + + def test_completely_disjoint_distributions(self) -> None: + """Disjoint single-value distributions score the maximum of 1.0.""" + real = Distribution({"a": 1.0}) + synthetic = Distribution({"b": 1.0}) + # squared error = (1-0)**2 + (0-1)**2 = 2, halved = 1.0 + self.assertAlmostEqual(1.0, self.metric.compare(real, synthetic)) + + def test_partial_overlap(self) -> None: + """Score reflects the squared difference in shared probability mass.""" + real = Distribution({"a": 0.6, "b": 0.4}) + synthetic = Distribution({"a": 0.4, "b": 0.6}) + expected = ((0.2) ** 2 + (0.2) ** 2) / 2 + self.assertAlmostEqual(expected, self.metric.compare(real, synthetic)) + + def test_empty_vocabulary_scores_zero(self) -> None: + """Two empty distributions have no vocabulary and score zero.""" + self.assertEqual(0.0, self.metric.compare(Distribution({}), Distribution({}))) + + def test_none_and_mixed_type_keys_are_canonicalized(self) -> None: + """None and non-string keys are normalized so they compare correctly.""" + real = Distribution({None: 1.0}) + synthetic = Distribution({"": 1.0}) + self.assertEqual(0.0, self.metric.compare(real, synthetic)) + + real_int_key = Distribution({3: 1.0}) + synthetic_str_key = Distribution({"3": 1.0}) + self.assertEqual(0.0, self.metric.compare(real_int_key, synthetic_str_key)) + + +class JensenShannonTests(DatafakerTestCase): + """Test case for the JensenShannon metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = JensenShannon() + + def test_identical_distributions_score_zero(self) -> None: + """Two identical distributions have zero divergence.""" + dist = Distribution({"a": 0.3, "b": 0.7}) + self.assertAlmostEqual(0.0, self.metric.compare(dist, dist)) + + def test_completely_disjoint_distributions_score_ln2(self) -> None: + """Disjoint distributions hit the theoretical maximum of ln(2).""" + real = Distribution({"a": 1.0}) + synthetic = Distribution({"b": 1.0}) + self.assertAlmostEqual(math.log(2), self.metric.compare(real, synthetic)) + + def test_partial_overlap_between_zero_and_max(self) -> None: + """A partially-overlapping pair scores strictly between 0 and ln(2).""" + real = Distribution({"a": 0.9, "b": 0.1}) + synthetic = Distribution({"a": 0.1, "b": 0.9}) + score = self.metric.compare(real, synthetic) + self.assertGreater(score, 0.0) + self.assertLess(score, math.log(2)) + + def test_empty_vocabulary_scores_zero(self) -> None: + """Two empty distributions score zero.""" + self.assertEqual(0.0, self.metric.compare(Distribution({}), Distribution({}))) + + def test_is_symmetric(self) -> None: + """Jensen-Shannon divergence is symmetric in its two arguments.""" + real = Distribution({"a": 0.2, "b": 0.8}) + synthetic = Distribution({"a": 0.7, "b": 0.3}) + # pylint: disable-next=arguments-out-of-order + reversed_score = self.metric.compare(synthetic, real) + self.assertAlmostEqual(self.metric.compare(real, synthetic), reversed_score) + + +class NoveltyMetricTests(DatafakerTestCase): + """Test case for the NoveltyMetric metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = NoveltyMetric() + + def test_no_overlap_is_fully_novel(self) -> None: + """Synthetic values sharing nothing with the real data score 1.0.""" + self.assertEqual(1.0, self.metric.compare(["a", "b"], ["c", "d"])) + + def test_full_overlap_is_not_novel(self) -> None: + """Synthetic values that are all copies of real values score 0.0.""" + self.assertEqual(0.0, self.metric.compare(["a", "b"], ["a", "b"])) + + def test_partial_overlap(self) -> None: + """Score is the fraction of distinct synthetic values not seen in real.""" + real = ["a", "b"] + synthetic = ["a", "c"] + self.assertAlmostEqual(0.5, self.metric.compare(real, synthetic)) + + def test_empty_synthetic_scores_zero(self) -> None: + """No synthetic values at all is treated as no novelty.""" + self.assertEqual(0.0, self.metric.compare(["a"], [])) + + def test_comparison_is_case_and_whitespace_insensitive(self) -> None: + """Values are compared after stripping whitespace and lowercasing.""" + self.assertEqual(0.0, self.metric.compare(["Alice"], [" alice "])) + + def test_none_values_are_ignored(self) -> None: + """None entries in either sequence are dropped before comparison.""" + self.assertEqual(0.0, self.metric.compare(["a", None], ["a"])) + + +class DiversityMetricTests(DatafakerTestCase): + """Test case for the DiversityMetric metric.""" + + def setUp(self) -> None: + super().setUp() + self.metric = DiversityMetric() + + def test_matching_diversity_scores_one(self) -> None: + """Equally-diverse real and synthetic samples score 1.0.""" + real = ["a", "b", "c", "d"] + synthetic = ["w", "x", "y", "z"] + self.assertAlmostEqual(1.0, self.metric.compare(real, synthetic)) + + def test_single_repeated_value_has_zero_entropy_on_both_sides(self) -> None: + """A single repeated value on both sides is a diversity match.""" + self.assertAlmostEqual(1.0, self.metric.compare(["a", "a", "a"], ["b", "b"])) + + def test_less_diverse_synthetic_scores_below_one(self) -> None: + """A synthetic sample far less diverse than the real one is penalized.""" + real = ["a", "b", "c", "d"] + synthetic = ["x", "x", "x", "x"] + score = self.metric.compare(real, synthetic) + self.assertLess(score, 1.0) + self.assertGreaterEqual(score, 0.0) + + def test_empty_values_treated_as_zero_entropy(self) -> None: + """Empty or all-None sequences have zero normalized entropy.""" + self.assertAlmostEqual(1.0, self.metric.compare([], [])) + self.assertAlmostEqual(1.0, self.metric.compare([None, None], [None])) + + def test_score_is_bounded_below_by_zero(self) -> None: + """The score never goes negative even for maximally different diversity.""" + real = ["a", "a", "a", "a"] + synthetic = ["a", "b", "c", "d"] + score = self.metric.compare(real, synthetic) + self.assertGreaterEqual(score, 0.0) diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py new file mode 100644 index 0000000..2e8b73c --- /dev/null +++ b/tests/test_evaluators_proposal_ranking.py @@ -0,0 +1,398 @@ +"""Unit tests for datafaker.evaluators.proposal_ranking.""" +from typing import Any + +from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation +from datafaker.evaluators.proposal_ranking import ( + ProposalRanking, + _crowding_distances, + _pareto_fronts, + format_ranking_display, + keyword_match, + normalize_list, + rank_proposals, +) +from datafaker.proposers.base import Proposer +from datafaker.proposers.choice import UniformChoiceProposer +from tests.utils import DatafakerTestCase + + +class FakeProposer(Proposer): + """A minimal, DB-free stand-in for a real Proposer, for ranking tests.""" + + def __init__(self, name: str) -> None: + self._name = name + + def function_name(self) -> str: + return self._name + + def nominal_kwargs(self) -> dict[str, str]: + return {} + + def actual_kwargs(self) -> dict[str, Any]: + return {} + + def generate_data(self, count: int) -> list[Any]: + return [None] * count + + +def make_result( # pylint: disable=too-many-arguments,too-many-positional-arguments + name: str = "generic.text", + overall_score: float = 0.5, + novelty: float = 0.5, + diversity: float = 0.5, + copy_fraction: float = 0.0, + synthetic_uniqueness: float = 1.0, + proposer: Proposer | None = None, +) -> ProposalEvaluation: + """Build a ProposalEvaluation for ranking tests.""" + return ProposalEvaluation( + proposer=proposer or FakeProposer(name), + novelty=novelty, + diversity=diversity, + overall_score=overall_score, + pipeline_scores={}, + copy_fraction=copy_fraction, + synthetic_uniqueness=synthetic_uniqueness, + ) + + +class KeywordMatchTests(DatafakerTestCase): + """Test case for keyword_match.""" + + def test_matching_keyword_and_generator_gives_boost(self) -> None: + """A column/generator pair that both mention 'email' gets the boost.""" + boost, keyword = keyword_match("customer_email", "generic.person.email") + self.assertEqual(0.15, boost) + self.assertEqual("email", keyword) + + def test_compound_column_name_matches_by_substring(self) -> None: + """Substring matching catches compound names like 'customer_first_name'.""" + boost, keyword = keyword_match( + "customer_first_name", "generic.person.first_name" + ) + self.assertEqual(0.15, boost) + self.assertEqual("first_name", keyword) + + def test_no_column_name_gives_no_boost(self) -> None: + """A missing column name can't match anything.""" + self.assertEqual((0.0, None), keyword_match(None, "generic.person.email")) + self.assertEqual((0.0, None), keyword_match("", "generic.person.email")) + + def test_column_matches_but_generator_does_not(self) -> None: + """Column name hints at a kind, but this generator isn't of that kind.""" + boost, keyword = keyword_match("email", "generic.text.word") + self.assertEqual(0.0, boost) + self.assertIsNone(keyword) + + def test_is_case_insensitive(self) -> None: + """Matching ignores case in both the column and generator names.""" + boost, keyword = keyword_match("EMAIL", "GENERIC.PERSON.EMAIL") + self.assertEqual(0.15, boost) + self.assertEqual("email", keyword) + + +class NormalizeListTests(DatafakerTestCase): + """Test case for normalize_list.""" + + def test_empty_list(self) -> None: + """An empty input returns an empty output.""" + self.assertEqual([], normalize_list([])) + + def test_spread_out_values_map_to_zero_one(self) -> None: + """Values are linearly rescaled so the min is 0 and the max is 1.""" + self.assertEqual([0.0, 0.5, 1.0], normalize_list([10, 20, 30])) + + def test_all_identical_values_are_treated_as_a_tie(self) -> None: + """Exactly equal values become 0.5 rather than dividing by zero.""" + self.assertEqual([0.5, 0.5, 0.5], normalize_list([3.0, 3.0, 3.0])) + + def test_near_identical_values_are_also_treated_as_a_tie(self) -> None: + """Floating-point noise around a shared value doesn't get stretched out.""" + result = normalize_list([0.6931471805599453, 0.6931471805599454]) + self.assertEqual([0.5, 0.5], result) + + def test_single_value_is_a_tie(self) -> None: + """A single value has no spread, so it's treated as a tie too.""" + self.assertEqual([0.5], normalize_list([42.0])) + + +class ParetoFrontsTests(DatafakerTestCase): + """Test case for the internal _pareto_fronts helper.""" + + def test_strictly_dominated_point_is_in_a_later_front(self) -> None: + """A point beaten or matched on every dimension lands behind the winner.""" + points = [(1.0, 1.0, 1.0), (0.5, 0.5, 0.5)] + fronts = _pareto_fronts(points) + self.assertEqual([[0], [1]], fronts) + + def test_non_dominated_points_share_the_first_front(self) -> None: + """Points that each win on a different dimension are mutually non-dominated.""" + points = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)] + fronts = _pareto_fronts(points) + self.assertEqual([[0, 1, 2]], fronts) + + def test_empty_points(self) -> None: + """No points means no fronts.""" + self.assertEqual([], _pareto_fronts([])) + + +class CrowdingDistancesTests(DatafakerTestCase): + """Test case for the internal _crowding_distances helper.""" + + def test_front_boundary_points_get_infinite_crowding(self) -> None: + """The extreme points of a front are always maximally 'crowded out'.""" + points = [(0.0, 0.0, 0.0), (0.5, 0.5, 0.5), (1.0, 1.0, 1.0)] + fronts = [[0, 1, 2]] + objectives = ( + [p[0] for p in points], + [p[1] for p in points], + [p[2] for p in points], + ) + crowding = _crowding_distances(points, fronts, objectives) + self.assertEqual(float("inf"), crowding[0]) + self.assertEqual(float("inf"), crowding[2]) + self.assertLess(crowding[1], float("inf")) + + def test_singleton_front_gets_infinite_crowding(self) -> None: + """A front with just one point has nothing to be crowded by.""" + points = [(0.5, 0.5, 0.5)] + fronts = [[0]] + objectives = ([0.5], [0.5], [0.5]) + crowding = _crowding_distances(points, fronts, objectives) + self.assertEqual([float("inf")], crowding) + + +class RankProposalsTests(DatafakerTestCase): + """Test case for rank_proposals.""" + + def test_no_results_gives_an_empty_ranking(self) -> None: + """With nothing to rank, no recommendation is made.""" + ranking = rank_proposals([], EvaluationProfile.SHORT_TEXT) + self.assertIsNone(ranking.recommended_index) + self.assertIsNone(ranking.recommended_reason) + self.assertEqual([], ranking.rows) + self.assertEqual((0.5, 0.25, 0.25), ranking.weights) + + def test_single_result_is_trivially_recommended(self) -> None: + """A single candidate is always the recommendation.""" + results = [make_result("only_one")] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + self.assertEqual(0, ranking.recommended_index) + self.assertEqual("best combined score", ranking.recommended_reason) + self.assertEqual(1, len(ranking.rows)) + + def test_lower_error_wins_on_fidelity(self) -> None: + """Among otherwise-equal candidates, the one with lower overall_score wins.""" + results = [ + make_result("bad_fit", overall_score=0.9, novelty=0.5, diversity=0.5), + make_result("good_fit", overall_score=0.1, novelty=0.5, diversity=0.5), + ] + ranking = rank_proposals(results, EvaluationProfile.IDENTIFIER) + self.assertEqual(1, ranking.recommended_index) + + def test_identifier_profile_weighs_fidelity_heavily(self) -> None: + """IDENTIFIER profile weights are (0.85, 0.05, 0.10).""" + ranking = rank_proposals([make_result()], EvaluationProfile.IDENTIFIER) + self.assertEqual((0.85, 0.05, 0.10), ranking.weights) + + def test_unknown_profile_falls_back_to_default_weights(self) -> None: + """A profile with no explicit entry uses the (0.5, 0.25, 0.25) default.""" + ranking = rank_proposals([make_result()], None) + self.assertEqual((0.5, 0.25, 0.25), ranking.weights) + + def test_exact_tie_breaks_towards_the_earlier_candidate(self) -> None: + """Identical candidates are both max-crowding; the first one wins.""" + results = [make_result("first"), make_result("second")] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + self.assertEqual(0, ranking.recommended_index) + self.assertEqual("tiebreak by crowding distance", ranking.recommended_reason) + + def test_keyword_boost_can_flip_the_recommendation(self) -> None: + """A column-name/generator match can outweigh an otherwise-tied fidelity.""" + results = [ + make_result( + "generic.text.word", overall_score=0.1, novelty=0.5, diversity=0.5 + ), + make_result( + "generic.person.email", overall_score=0.1, novelty=0.5, diversity=0.5 + ), + ] + # Without the keyword boost this is an exact tie, decided by crowding + # distance (which favors the first candidate); the boost should flip it. + ranking = rank_proposals( + results, EvaluationProfile.EMAIL, column_name="user_email" + ) + self.assertEqual(1, ranking.recommended_index) + self.assertEqual("best combined score", ranking.recommended_reason) + + def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: + """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" + choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + results = [ + make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), + make_result(overall_score=0.5, copy_fraction=0.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + # The ChoiceProposer perfectly fits but is fully discounted (penalty + # multiplier 0), leaving the other candidate to win despite a worse fit. + self.assertEqual(1, ranking.recommended_index) + + def test_non_choice_proposer_is_not_resample_penalized_on_string_column( + self, + ) -> None: + """Only ChoiceProposer (or a numeric column) is hit by the resample penalty.""" + results = [make_result(overall_score=0.0, copy_fraction=1.0)] + ranking = rank_proposals( + results, EvaluationProfile.SHORT_TEXT, real_uniqueness=1.0 + ) + self.assertFalse(ranking.penalty_applies_to_all) + # combined score should be unaffected by the (non-applicable) penalty: + # a single candidate normalizes every dimension to a 0.5 tie. + fid_w, nov_w, div_w = ranking.weights + expected = (fid_w + nov_w + div_w) * 0.5 + self.assertAlmostEqual(expected, ranking.scores[0]) + + def test_numeric_column_penalizes_every_proposer(self) -> None: + """A numeric column applies the resample-style penalty to all candidates.""" + results = [make_result(overall_score=0.0, copy_fraction=1.0)] + ranking = rank_proposals( + results, + EvaluationProfile.IDENTIFIER, + real_uniqueness=1.0, + is_numeric_column=True, + ) + self.assertTrue(ranking.penalty_applies_to_all) + self.assertEqual(0.0, ranking.scores[0]) + + def test_self_duplication_penalty_applies_regardless_of_proposer_type(self) -> None: + """Low synthetic_uniqueness is penalized even for a non-resampling proposer.""" + results = [ + make_result("full_of_dupes", overall_score=0.0, synthetic_uniqueness=0.0), + make_result("all_unique", overall_score=0.0, synthetic_uniqueness=1.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + self.assertEqual(1, ranking.recommended_index) + + def test_no_uniqueness_guarantee_when_every_candidate_duplicates(self) -> None: + """A primary key column with only duplicate-prone candidates has no safe pick.""" + results = [ + make_result("dupe_a", synthetic_uniqueness=0.5), + make_result("dupe_b", synthetic_uniqueness=0.9), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + self.assertTrue(ranking.no_uniqueness_guarantee) + + def test_uniqueness_guarantee_satisfied_by_one_candidate(self) -> None: + """If at least one candidate guarantees uniqueness, there's no warning.""" + results = [ + make_result("dupe", synthetic_uniqueness=0.5), + make_result("unique", synthetic_uniqueness=1.0), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + self.assertFalse(ranking.no_uniqueness_guarantee) + + def test_combined_score_is_clamped_to_one(self) -> None: + """The keyword boost can't push a score above 1.0.""" + results = [make_result("generic.person.email", overall_score=0.0)] + ranking = rank_proposals(results, EvaluationProfile.EMAIL, column_name="email") + self.assertLessEqual(ranking.scores[0], 1.0) + + def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: + """A heavily-discounted best-fit candidate triggers the fallback warning.""" + choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + results = [ + # Fits almost perfectly but is a resampler on a near-unique column: + # gets discounted to (near) zero. + make_result( + overall_score=0.0, + novelty=0.0, + diversity=0.0, + copy_fraction=1.0, + synthetic_uniqueness=1.0, + proposer=choice_proposer, + ), + # A poor fit that isn't penalized, so it "wins" by elimination. + # synthetic_uniqueness is deliberately below the guarantee + # threshold so the weak-recommendation warning isn't suppressed + # by the "winner guarantees uniqueness" exemption. + make_result( + overall_score=1.0, + novelty=0.0, + diversity=0.0, + copy_fraction=0.0, + synthetic_uniqueness=0.5, + proposer=FakeProposer("generic.text.word"), + ), + ] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, real_uniqueness=1.0 + ) + self.assertEqual(1, ranking.recommended_index) + self.assertIsNotNone(ranking.weak_recommendation_warning) + self.assert_str_in("resample penalty", ranking.weak_recommendation_warning) + + +class FormatRankingDisplayTests(DatafakerTestCase): + """Test case for format_ranking_display.""" + + def test_recommendation_line_names_the_winner(self) -> None: + """The recommendation string names the recommended candidate by number/name.""" + results = [make_result("winner", overall_score=0.0)] + ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + display = format_ranking_display(ranking, results, EvaluationProfile.SHORT_TEXT) + self.assert_str_in("Recommended: 1. winner", display.recommendation) + self.assert_str_in("best combined score", display.recommendation) + + def test_no_uniqueness_guarantee_overrides_recommendation_text(self) -> None: + """When no candidate can guarantee uniqueness, no winner is named.""" + results = [make_result("a", synthetic_uniqueness=0.5)] + ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_primary_key=True + ) + display = format_ranking_display(ranking, results, EvaluationProfile.IDENTIFIER) + self.assert_str_in("Recommended: none", display.recommendation) + + def test_penalty_scope_text_reflects_numeric_flag(self) -> None: + """The profile summary explains whether the penalty hit every candidate.""" + results = [make_result("a")] + numeric_ranking = rank_proposals( + results, EvaluationProfile.IDENTIFIER, is_numeric_column=True + ) + numeric_display = format_ranking_display( + numeric_ranking, results, EvaluationProfile.IDENTIFIER + ) + self.assert_str_in( + "every generator (numeric column)", numeric_display.profile_summary + ) + + string_ranking = rank_proposals(results, EvaluationProfile.SHORT_TEXT) + string_display = format_ranking_display( + string_ranking, results, EvaluationProfile.SHORT_TEXT + ) + self.assert_str_in( + "resamplers only (dist_gen.choice/weighted_choice/zipf_choice)", + string_display.profile_summary, + ) + + def test_rows_and_fronts_pass_through_unchanged(self) -> None: + """rows and fronts are carried over verbatim from the ProposalRanking.""" + results = [make_result("winner")] + ranking = ProposalRanking( + recommended_index=0, + recommended_reason="best combined score", + rows=[("1", "x")], + profile=EvaluationProfile.SHORT_TEXT, + weights=(0.5, 0.25, 0.25), + fronts=[1], + ) + display = format_ranking_display(ranking, results, EvaluationProfile.SHORT_TEXT) + self.assertEqual([("1", "x")], display.rows) + self.assertEqual([1], display.fronts) diff --git a/tests/test_evaluators_statistical_fidelity.py b/tests/test_evaluators_statistical_fidelity.py new file mode 100644 index 0000000..fac7aa6 --- /dev/null +++ b/tests/test_evaluators_statistical_fidelity.py @@ -0,0 +1,108 @@ +"""Unit tests for datafaker.evaluators.statistical_fidelity.""" +from datetime import datetime + +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, create_engine + +from datafaker.evaluators.column_evaluator import EvaluationProfile +from datafaker.evaluators.statistical_fidelity import ( + EMAIL_PIPELINE, + PROFILE_PIPELINES, + StatisticalFidelity, +) +from tests.utils import DatafakerTestCase + + +class ProfilePipelinesTests(DatafakerTestCase): + """Structural sanity checks for the built-in evaluation pipelines.""" + + def test_every_evaluation_profile_has_a_pipeline(self) -> None: + """Each EvaluationProfile enum member maps to a configured pipeline.""" + for profile in EvaluationProfile: + self.assertIn(profile, PROFILE_PIPELINES) + self.assertTrue(PROFILE_PIPELINES[profile]) + + def test_pipeline_weights_sum_to_one(self) -> None: + """Each profile's pipeline weights add up to 1.0, for a sane weighted average.""" + for profile, pipelines in PROFILE_PIPELINES.items(): + with self.subTest(profile=profile.name): + total_weight = sum(p.weight for p in pipelines) + self.assertAlmostEqual(1.0, total_weight) + + def test_pipeline_names_are_unique_within_a_profile(self) -> None: + """Pipeline names double as dict keys in calculate_scores, so must be unique.""" + for profile, pipelines in PROFILE_PIPELINES.items(): + with self.subTest(profile=profile.name): + names = [p.name for p in pipelines] + self.assertEqual(len(names), len(set(names))) + + +def _make_table(columns: list[Column], rows: list[dict]) -> tuple: + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("data", metadata, *columns) + metadata.create_all(engine) + if rows: + with engine.begin() as conn: + conn.execute(table.insert(), rows) + return engine, table + + +class StatisticalFidelityTests(DatafakerTestCase): + """Test case for StatisticalFidelity.""" + + def test_identical_real_and_synthetic_data_scores_perfectly(self) -> None: + """A synthetic sample identical to the real data has zero divergence.""" + engine, table = _make_table( + [Column("email", String)], + [{"email": f"user{i}@example.com"} for i in range(30)], + ) + fidelity = StatisticalFidelity(table.c.email, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.EMAIL) + + synthetic = [f"user{i}@example.com" for i in range(30)] + overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + + self.assertAlmostEqual(0.0, overall_score) + self.assertEqual({p.name for p in EMAIL_PIPELINE}, set(pipeline_scores)) + for name, score in pipeline_scores.items(): + with self.subTest(pipeline=name): + self.assertAlmostEqual(0.0, score) + + def test_completely_different_synthetic_data_scores_worse(self) -> None: + """A synthetic sample sharing nothing with the real data scores worse.""" + engine, table = _make_table( + [Column("category", String)], + [{"category": "alpha"} for _ in range(30)], + ) + fidelity = StatisticalFidelity(table.c.category, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.CATEGORICAL) + + matching_score, _ = fidelity.calculate_scores(["alpha"] * 30) + different_score, _ = fidelity.calculate_scores(["zzz_never_seen"] * 30) + + self.assertGreater(different_score, matching_score) + + def test_temporal_profile_scores_a_matching_date_column(self) -> None: + """The TEMPORAL profile's two pipelines run cleanly against a date column.""" + engine, table = _make_table( + [Column("ts", DateTime)], + [{"ts": datetime(2020, 1, (i % 27) + 1)} for i in range(40)], + ) + fidelity = StatisticalFidelity(table.c.ts, engine, sample_size=1000) + fidelity.set_eval_pipelines(EvaluationProfile.TEMPORAL) + + synthetic = [datetime(2020, 1, (i % 27) + 1) for i in range(40)] + overall_score, pipeline_scores = fidelity.calculate_scores(synthetic) + + self.assertAlmostEqual(0.0, overall_score) + self.assertEqual({"timestamp", "day_of_week"}, set(pipeline_scores)) + + def test_no_pipelines_set_scores_zero(self) -> None: + """Without calling set_eval_pipelines, there's nothing to score.""" + engine, table = _make_table( + [Column("n", Integer)], [{"n": i} for i in range(10)] + ) + fidelity = StatisticalFidelity(table.c.n, engine, sample_size=1000) + overall_score, pipeline_scores = fidelity.calculate_scores([1, 2, 3]) + self.assertEqual(0.0, overall_score) + self.assertEqual({}, pipeline_scores) diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 08d4587..1a59e4c 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -427,7 +427,11 @@ def test_aggregate_queries_merge(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - gc.do_propose("") + # "position" is an Integer column, and dist_gen.uniform_ms produces + # float output that default 'propose' now excludes as type- + # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' + # bypasses that filter so this generator is still selectable here. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[f"{generator}"][0])) gc.do_quit("") @@ -604,7 +608,11 @@ def test_existing_configuration_remains(self) -> None: column = "position" generator = "dist_gen.uniform_ms" gc.do_next(f"string.{column}") - gc.do_propose("") + # "position" is an Integer column, and dist_gen.uniform_ms produces + # float output that default 'propose' now excludes as type- + # incompatible (see GeneratorCmd._is_integer_incompatible) - 'all' + # bypasses that filter so this generator is still selectable here. + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) gc.do_quit("") diff --git a/tests/test_rst.py b/tests/test_rst.py index ee89b5b..edd87ac 100644 --- a/tests/test_rst.py +++ b/tests/test_rst.py @@ -40,6 +40,7 @@ def test_dir(self) -> None: # Ignore errors if they contain any of these strings allowed_errors = [ 'No role entry for "ref" in module', + 'No role entry for "doc" in module', 'No directive entry for "toctree"', 'No directive entry for "automodule"', 'No directive entry for "literalinclude"', @@ -51,6 +52,10 @@ def test_dir(self) -> None: 'Hyperlink target "page-quickstart" is not referenced.', 'Hyperlink target "page-installation" is not referenced.', 'Hyperlink target "story-generators" is not referenced.', + 'Hyperlink target "mimesis-string-list" is not referenced', + 'Hyperlink target "increment-proposer" is not referenced', + 'Hyperlink target "evaluating-and-ranking-proposals" is not referenced', + 'Hyperlink target "statistical-fidelity-pipelines" is not referenced', ] filtered_errors = [ file_error