diff --git a/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py b/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py new file mode 100644 index 000000000..bd88a8d23 --- /dev/null +++ b/DashAI/alembic/versions/f4a91c62d8e7_add_preprocessing_to_model_session.py @@ -0,0 +1,60 @@ +"""add preprocessing columns to model_session + +Revision ID: f4a91c62d8e7 +Revises: m6n7o8p9q0r1 +Create Date: 2026-09-08 10:00:00.000000 + +Sessions created before this column existed have no preprocessing, so an +empty/None value means exactly that — no backfill is needed. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "f4a91c62d8e7" +down_revision: Union[str, None] = "m6n7o8p9q0r1" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_NEW_COLUMNS = [ + ("preprocessing", sa.JSON(), True, None), + ("input_column_refs", sa.JSON(), True, None), + ("preprocessing_status", sa.String(), False, "ready"), + ("preprocessing_error", sa.String(), True, None), + ("preprocessing_artifacts_path", sa.String(), True, None), + ("preprocessing_job_id", sa.String(), True, None), +] + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "model_session" not in inspector.get_table_names(): + return + existing = {column["name"] for column in inspector.get_columns("model_session")} + for name, col_type, nullable, server_default in _NEW_COLUMNS: + if name in existing: + continue + op.add_column( + "model_session", + sa.Column( + name, + col_type, + nullable=nullable, + server_default=server_default, + ), + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "model_session" not in inspector.get_table_names(): + return + existing = {column["name"] for column in inspector.get_columns("model_session")} + for name, _, _, _ in _NEW_COLUMNS: + if name not in existing: + continue + op.drop_column("model_session", name) diff --git a/DashAI/back/api/api_v1/endpoints/model_sessions.py b/DashAI/back/api/api_v1/endpoints/model_sessions.py index c68153df9..aad2cc72b 100644 --- a/DashAI/back/api/api_v1/endpoints/model_sessions.py +++ b/DashAI/back/api/api_v1/endpoints/model_sessions.py @@ -14,6 +14,8 @@ ) from DashAI.back.api.utils import remove_path from DashAI.back.dependencies.database.models import Dataset, ModelSession, Run +from DashAI.back.job.preprocessing_job import PreprocessingJob +from DashAI.back.preprocessing.column_ref import ConverterSequence, RawColumnRef from DashAI.back.splitters.splits_payload import ( META_KEYS, normalize_splits_payload, @@ -23,6 +25,7 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker + from DashAI.back.dependencies.job_queues import BaseJobQueue from DashAI.back.dependencies.registry import ComponentRegistry from DashAI.back.tasks.base_task import BaseTask @@ -32,6 +35,22 @@ router = APIRouter() +# Every concrete value type name that DashAIValue's wildcard covers (i.e. every +# DashAIValue subclass in DashAI/back/types/value_types.py), used so a task +# declaring the wildcard (e.g. ClassificationTask) still accepts a converter +# group whose declared type is one of these, not just "DashAIValue" itself. +_DASHAI_VALUE_TYPE_NAMES = { + "Integer", + "Float", + "Text", + "Time", + "Timestamp", + "Duration", + "Decimal", + "Date", + "Binary", +} + @router.get("/") @inject @@ -139,13 +158,23 @@ async def validate_columns( column_names = minimal_dataset.column_names - if len(params.inputs_columns + params.outputs_columns) > len(column_names): + group_refs = [ + ref for ref in (params.input_refs or []) if ref.kind == "group" + ] + + if not group_refs and len( + params.inputs_columns + params.outputs_columns + ) > len(column_names): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Column index out of range", ) - inputs_names = params.inputs_columns + inputs_names = ( + [r.name for r in params.input_refs if r.kind == "raw"] + if params.input_refs + else params.inputs_columns + ) outputs_names = params.outputs_columns except exc.SQLAlchemyError as e: @@ -162,6 +191,33 @@ async def validate_columns( ) task: "BaseTask" = component_registry[params.task_name]["class"]() + + if group_refs: + declared_types = params.converter_output_types or {} + task_metadata = task.get_metadata() + allowed_input_types = set(task_metadata.get("inputs_types", [])) + for ref in group_refs: + # A step with a heterogeneous scope (see SessionPreprocessor. + # _classify_by_type) can declare more than one type, one per + # slot — "{step}:{slot}" disambiguates which one a slotted ref + # means; an unslotted ref (the whole step) keeps the plain + # "{step}" key, unchanged from before slots existed. + key = str(ref.step) if ref.slot is None else f"{ref.step}:{ref.slot}" + declared_type = declared_types.get(key) + type_ok = declared_type in allowed_input_types or ( + "DashAIValue" in allowed_input_types + and declared_type in _DASHAI_VALUE_TYPE_NAMES + ) + if allowed_input_types and not type_ok: + return { + "dataset_status": "invalid", + "error": ( + f"Converter step {ref.step} declares output type " + f"'{declared_type}', which is not one of the task's " + f"allowed input types {sorted(allowed_input_types)}." + ), + } + validation_response = {} try: @@ -236,6 +292,7 @@ async def create_model_session( params: ModelSessionParams, session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]), component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]), + job_queue: "BaseJobQueue" = Depends(lambda: di["job_queue"]), ): """Create a new model session. @@ -268,6 +325,28 @@ async def create_model_session( _validate_splits(params.splits, component_registry) + sequence = ConverterSequence(steps=params.preprocessing) + try: + sequence.validate_scopes() + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) from e + has_preprocessing = len(sequence.steps) > 0 + + if has_preprocessing and not params.input_column_refs: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=( + "input_column_refs is required when preprocessing steps are provided" + ), + ) + + input_column_refs = params.input_column_refs or [ + RawColumnRef(name=name) for name in params.input_columns + ] + with session_factory() as db: try: dataset = db.get(Dataset, params.dataset_id) @@ -283,7 +362,12 @@ async def create_model_session( schema = reader.schema column_names = schema.names - if len(params.input_columns + params.output_columns) > len(column_names): + # When preprocessing is configured, input_columns is only a + # placeholder until PreprocessingJob resolves the real ones, so + # it cannot be checked against the raw dataset's column count. + if not has_preprocessing and len( + params.input_columns + params.output_columns + ) > len(column_names): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Column index out of range", @@ -300,10 +384,24 @@ async def create_model_session( test_metrics=params.test_metrics, evaluation_strategy=params.evaluation_strategy, splits=params.splits, + preprocessing=sequence.model_dump(mode="json"), + input_column_refs=[ + ref.model_dump(mode="json") for ref in input_column_refs + ], + preprocessing_status="pending" if has_preprocessing else "ready", ) db.add(model_session) db.commit() db.refresh(model_session) + + if has_preprocessing: + job_id = job_queue.put( + PreprocessingJob(model_session_id=model_session.id) + ).id + model_session.preprocessing_job_id = str(job_id) + db.commit() + db.refresh(model_session) + return model_session except exc.IntegrityError as e: db.rollback() diff --git a/DashAI/back/api/api_v1/endpoints/runs.py b/DashAI/back/api/api_v1/endpoints/runs.py index fa847f3e8..5cb4cc0cf 100644 --- a/DashAI/back/api/api_v1/endpoints/runs.py +++ b/DashAI/back/api/api_v1/endpoints/runs.py @@ -295,6 +295,19 @@ async def upload_run( status_code=status.HTTP_404_NOT_FOUND, detail="Model session not found", ) + if model_session.preprocessing_status == "pending": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="This session's preprocessing has not finished yet.", + ) + if model_session.preprocessing_status == "failed": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "This session's preprocessing failed: " + f"{model_session.preprocessing_error}" + ), + ) # REQUIRES_DOWNLOAD is the static contract; the download state is # reconciled against the filesystem so a model downloaded after # startup (in the worker process) is recognised without a restart. diff --git a/DashAI/back/api/api_v1/schemas/model_sessions_params.py b/DashAI/back/api/api_v1/schemas/model_sessions_params.py index 5de8f632f..f5e35e415 100644 --- a/DashAI/back/api/api_v1/schemas/model_sessions_params.py +++ b/DashAI/back/api/api_v1/schemas/model_sessions_params.py @@ -1,6 +1,8 @@ -from typing import List +from typing import Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field + +from DashAI.back.preprocessing.column_ref import ColumnRef, ConverterStep class ModelSessionParams(BaseModel): @@ -14,6 +16,8 @@ class ModelSessionParams(BaseModel): test_metrics: List[str] evaluation_strategy: str splits: str + preprocessing: List[ConverterStep] = Field(default_factory=list) + input_column_refs: Optional[List[ColumnRef]] = None class ColumnsValidationParams(BaseModel): @@ -21,6 +25,8 @@ class ColumnsValidationParams(BaseModel): dataset_id: int inputs_columns: List[str] outputs_columns: List[str] + input_refs: Optional[List[ColumnRef]] = None + converter_output_types: Optional[Dict[str, str]] = None class ModelSessionBulkDeleteParams(BaseModel): diff --git a/DashAI/back/config.py b/DashAI/back/config.py index ef88d6031..98c1dc989 100644 --- a/DashAI/back/config.py +++ b/DashAI/back/config.py @@ -19,6 +19,7 @@ class DefaultSettings(BaseSettings): DATASETS_PATH: str = "datasets" IMAGES_PATH: str = "images" RUNS_PATH: str = "runs" + PREPROCESSING_PATH: str = "preprocessing" EXPLANATIONS_PATH: str = "explanations" EXPLORATIONS_PATH: str = "explorations" DOCUMENTS_PATH: str = "documents" diff --git a/DashAI/back/converters/base_converter.py b/DashAI/back/converters/base_converter.py index 8b636356b..79306a3dc 100644 --- a/DashAI/back/converters/base_converter.py +++ b/DashAI/back/converters/base_converter.py @@ -42,6 +42,15 @@ class BaseConverter(ConfigObject, ABC): COLOR: Final[str] = "rgb(255, 255, 255)" SUPERVISED: bool = False CHANGES_ROW_COUNT: bool = False + # True for converters that never transform values, only keep or drop + # whole columns as-is (feature selection, variance thresholding): the + # output type of a surviving column is always exactly its input type, no + # arithmetic involved. Lets a caller that already knows the real input + # type (e.g. the Models-module wizard, once a real scope is chosen) use + # that instead of this class's own best-effort get_output_type() guess, + # which — called on a bare unfitted instance — has no idea what column + # it will actually run on. + PRESERVES_INPUT_TYPE: bool = False SCHEMA: BaseConverterSchema @classmethod @@ -74,6 +83,7 @@ def get_metadata(cls) -> Dict[str, Any]: meta["download_size_bytes"] = getattr(cls, "DOWNLOAD_SIZE_BYTES", None) meta["supervised"] = cls.SUPERVISED meta["changes_row_count"] = cls.CHANGES_ROW_COUNT + meta["preserves_input_type"] = cls.PRESERVES_INPUT_TYPE meta["n_components_features_bounded"] = getattr( cls, "N_COMPONENTS_FEATURES_BOUNDED", False ) @@ -108,6 +118,30 @@ def get_metadata(cls) -> Dict[str, Any]: # Drop restricted_dtypes (no converter uses it; it is always []) meta.pop("restricted_dtypes", None) + # A representative output type, so the Models-module wizard can show + # "this converter's group is typed X" before any real fit exists. + # Not every converter can be instantiated with no arguments (some + # require constructor params with no default), so this is + # best-effort: None means "unknown until configured". + try: + output_type = cls().get_output_type() + meta["output_type"] = ( + output_type.display_name() + if output_type is not None and hasattr(output_type, "display_name") + else None + ) + # The concrete storage dtype (e.g. "int64"), so a group column can + # show one instead of "unknown" before any real fit exists — same + # best-effort default-constructed instance as output_type above. + meta["output_dtype"] = ( + output_type.to_string().get("dtype") + if output_type is not None and hasattr(output_type, "to_string") + else None + ) + except Exception: + meta["output_type"] = None + meta["output_dtype"] = None + return meta @abstractmethod diff --git a/DashAI/back/converters/category/feature_selection.py b/DashAI/back/converters/category/feature_selection.py index f090ae69a..a8c9919e2 100644 --- a/DashAI/back/converters/category/feature_selection.py +++ b/DashAI/back/converters/category/feature_selection.py @@ -34,6 +34,7 @@ class FeatureSelectionConverter(BaseConverter): ) ICON: Final[str] = Icon.FilterList.value COLOR: Final[str] = "rgb(255, 206, 86)" + PRESERVES_INPUT_TYPE = True def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/converters/dataset_columns.py b/DashAI/back/converters/dataset_columns.py new file mode 100644 index 000000000..cad8406c5 --- /dev/null +++ b/DashAI/back/converters/dataset_columns.py @@ -0,0 +1,108 @@ +"""Shared helper for splicing a converter's transformed columns back into a +dataset, used both by the Notebooks converter job and by session-level +preprocessing (SessionPreprocessor). +""" + +from typing import TYPE_CHECKING, List + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +def rebuild_dataset_with_transformed_columns( + base: "DashAIDataset", + transformed: "DashAIDataset", + scope_column_names: List[str], + scope_column_indexes: List[int], +) -> "DashAIDataset": + """ + Replaces specific columns in the base dataset with columns from the transformed + dataset, preserving their original positions. Also appends any additional columns + that were generated by the transformer at the end. Keeps the features and metadata + consistent. + + Parameters + ---------- + base : DashAIDataset + The original dataset before transformation. + + transformed : DashAIDataset + The dataset resulting from applying a transformer, containing updated and/or + new columns. + + scope_column_names : List[str] + Names of the columns that were originally selected for transformation. + + scope_column_indexes : List[int] + The indices of the columns in the base dataset that were replaced. + Must match the order of scope_column_names. + + Returns + ------- + DashAIDataset + A new dataset with the specified columns replaced in place, new columns + appended, and original metadata and split information preserved. + """ + from DashAI.back.dataloaders.classes.dashai_dataset import modify_table + + original_columns = base.column_names + transformed_cols = transformed.column_names + + transformed_cols_set = set(transformed_cols) + scope_column_names_set = set(scope_column_names) + + removed_cols = [ + col for col in scope_column_names if col not in transformed_cols_set + ] + replacement_cols = [ + col for col in scope_column_names if col in transformed_cols_set + ] + new_cols = [col for col in transformed_cols if col not in scope_column_names_set] + + removed_cols_set = set(removed_cols) + + new_columns_order = [] + seen_cols = set() + for col in original_columns: + if col in removed_cols_set: + continue + if col not in seen_cols: + new_columns_order.append(col) + seen_cols.add(col) + + col_name_mapping = {} + for col in new_cols: + unique_col = col + counter = 1 + while unique_col in seen_cols: + unique_col = f"{col}_{counter}" + counter += 1 + new_columns_order.append(unique_col) + seen_cols.add(unique_col) + col_name_mapping[col] = unique_col + + updated_arrays = {} + for col in replacement_cols: + if col in transformed_cols_set: + updated_arrays[col] = transformed.arrow_table[col] + for col, unique_col in col_name_mapping.items(): + if col in transformed_cols_set: + updated_arrays[unique_col] = transformed.arrow_table[col] + + updated_types = base.types.copy() + + for col in removed_cols: + if col in updated_types: + del updated_types[col] + + for col in replacement_cols: + if col in transformed.types: + updated_types[col] = transformed.types[col] + for col, unique_col in col_name_mapping.items(): + if col in transformed.types: + updated_types[unique_col] = transformed.types[col] + + modified_dataset = modify_table(base, updated_arrays, types=updated_types) + modified_dataset = modified_dataset.select_columns(new_columns_order) + + return modified_dataset diff --git a/DashAI/back/converters/scikit_learn/variance_threshold.py b/DashAI/back/converters/scikit_learn/variance_threshold.py index 216e755d7..335893904 100644 --- a/DashAI/back/converters/scikit_learn/variance_threshold.py +++ b/DashAI/back/converters/scikit_learn/variance_threshold.py @@ -92,6 +92,7 @@ class VarianceThreshold( de="Varianz-Schwellenwert", zh="方差阈值", ) + PRESERVES_INPUT_TYPE = True def fit( self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None diff --git a/DashAI/back/dependencies/config_builder.py b/DashAI/back/dependencies/config_builder.py index ef6c1ba2a..6a57c52cd 100644 --- a/DashAI/back/dependencies/config_builder.py +++ b/DashAI/back/dependencies/config_builder.py @@ -57,6 +57,7 @@ def build_config_dict( config["EXPLANATIONS_PATH"] = local_path / config["EXPLANATIONS_PATH"] config["NOTEBOOK_PATH"] = local_path / config["NOTEBOOK_PATH"] config["RUNS_PATH"] = local_path / config["RUNS_PATH"] + config["PREPROCESSING_PATH"] = local_path / config["PREPROCESSING_PATH"] config["IMAGES_PATH"] = local_path / config["IMAGES_PATH"] config["DATAFILE_PATH"] = local_path / config["DATAFILE_PATH"] config["CREDENTIALS_KEY_PATH"] = local_path / config["CREDENTIALS_KEY_PATH"] diff --git a/DashAI/back/dependencies/database/models.py b/DashAI/back/dependencies/database/models.py index bf878f36d..2b808f20f 100644 --- a/DashAI/back/dependencies/database/models.py +++ b/DashAI/back/dependencies/database/models.py @@ -146,6 +146,20 @@ class ModelSession(Base): evaluation_strategy: Mapped[str] = mapped_column(String, nullable=False) splits: Mapped[str] = mapped_column(JSON, nullable=False) + preprocessing: Mapped[dict] = mapped_column(JSON, nullable=True) + input_column_refs: Mapped[list] = mapped_column(JSON, nullable=True) + preprocessing_status: Mapped[str] = mapped_column( + String, nullable=False, default="ready", server_default="ready" + ) + preprocessing_error: Mapped[str] = mapped_column(String, nullable=True) + preprocessing_artifacts_path: Mapped[str] = mapped_column(String, nullable=True) + # The Huey task id PreprocessingJob was enqueued with — lets the frontend + # track this specific job via the same shared job-polling mechanism the + # Job Queue widget uses (useJobTracker/startJobPolling), instead of + # polling preprocessing_status on its own separate timer. Keeping both + # "is it done" signals on the same underlying poll loop is what keeps + # them from ever drifting out of sync with each other. + preprocessing_job_id: Mapped[str] = mapped_column(String, nullable=True) created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now) last_modified: Mapped[DateTime] = mapped_column( DateTime, diff --git a/DashAI/back/job/converter_job.py b/DashAI/back/job/converter_job.py index fcbfc4d68..3c6f964f0 100644 --- a/DashAI/back/job/converter_job.py +++ b/DashAI/back/job/converter_job.py @@ -1,10 +1,13 @@ import logging -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from kink import inject from sqlalchemy import exc from DashAI.back.api.api_v1.schemas.converter_params import ConverterParams +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) from DashAI.back.dependencies.database.models import Converter from DashAI.back.dependencies.database.models import Dataset as DatasetModel from DashAI.back.job.base_job import BaseJob, JobError @@ -12,112 +15,10 @@ if TYPE_CHECKING: from sqlalchemy.orm import sessionmaker - from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset - logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) -def _rebuild_dataset_with_transformed_columns( - base: "DashAIDataset", - transformed: "DashAIDataset", - scope_column_names: List[str], - scope_column_indexes: List[int], -) -> "DashAIDataset": - """ - Replaces specific columns in the base dataset with columns from the transformed - dataset, preserving their original positions. Also appends any additional columns - that were generated by the transformer at the end. Keeps the features and metadata - consistent. - - Parameters - ---------- - base : DashAIDataset - The original dataset before transformation. - - transformed : DashAIDataset - The dataset resulting from applying a transformer, containing updated and/or - new columns. - - scope_column_names : List[str] - Names of the columns that were originally selected for transformation. - - scope_column_indexes : List[int] - The indices of the columns in the base dataset that were replaced. - Must match the order of scope_column_names. - - Returns - ------- - DashAIDataset - A new dataset with the specified columns replaced in place, new columns - appended, and original metadata and split information preserved. - """ - from DashAI.back.dataloaders.classes.dashai_dataset import modify_table - - original_columns = base.column_names - transformed_cols = transformed.column_names - - transformed_cols_set = set(transformed_cols) - scope_column_names_set = set(scope_column_names) - - removed_cols = [ - col for col in scope_column_names if col not in transformed_cols_set - ] - replacement_cols = [ - col for col in scope_column_names if col in transformed_cols_set - ] - new_cols = [col for col in transformed_cols if col not in scope_column_names_set] - - removed_cols_set = set(removed_cols) - - new_columns_order = [] - seen_cols = set() - for col in original_columns: - if col in removed_cols_set: - continue - if col not in seen_cols: - new_columns_order.append(col) - seen_cols.add(col) - - col_name_mapping = {} - for col in new_cols: - unique_col = col - counter = 1 - while unique_col in seen_cols: - unique_col = f"{col}_{counter}" - counter += 1 - new_columns_order.append(unique_col) - seen_cols.add(unique_col) - col_name_mapping[col] = unique_col - - updated_arrays = {} - for col in replacement_cols: - if col in transformed_cols_set: - updated_arrays[col] = transformed.arrow_table[col] - for col, unique_col in col_name_mapping.items(): - if col in transformed_cols_set: - updated_arrays[unique_col] = transformed.arrow_table[col] - - updated_types = base.types.copy() - - for col in removed_cols: - if col in updated_types: - del updated_types[col] - - for col in replacement_cols: - if col in transformed.types: - updated_types[col] = transformed.types[col] - for col, unique_col in col_name_mapping.items(): - if col in transformed.types: - updated_types[unique_col] = transformed.types[col] - - # Use existing modify_table (imported at module level) - modified_dataset = modify_table(base, updated_arrays, types=updated_types) - modified_dataset = modified_dataset.select_columns(new_columns_order) - - return modified_dataset - - class ConverterJob(BaseJob): """ConverterJob class to modify a dataset by applying a sequence of converters.""" @@ -410,7 +311,7 @@ def instantiate_converters( if type(converter_instance).CHANGES_ROW_COUNT: loaded_dataset = transformed_dataset else: - loaded_dataset = _rebuild_dataset_with_transformed_columns( + loaded_dataset = rebuild_dataset_with_transformed_columns( loaded_dataset, transformed_dataset, scope_column_names, diff --git a/DashAI/back/job/explainer_job.py b/DashAI/back/job/explainer_job.py index 2033353e1..8a9fb64dd 100644 --- a/DashAI/back/job/explainer_job.py +++ b/DashAI/back/job/explainer_job.py @@ -186,6 +186,7 @@ def _generate_local_explanation( splits: Dict[str, Any], task: BaseTask, same_dataset: bool, + preprocessor: Any = None, ) -> None: import json import os @@ -242,12 +243,20 @@ def _generate_local_explanation( manual_input_data, f"{instance.file_path}/dataset", ) + if preprocessor is not None: + prepared_instance = preprocessor.transform_dataset( + prepared_instance + ) # Manual input carries only the input columns (no target), so # keep just those instead of the standard input/output split. # select_columns returns a DashAIDataset (same shape the # split path produces), which is what the explainers expect. X = prepared_instance.select_columns(self.input_columns) else: + if preprocessor is not None: + loaded_instance = preprocessor.transform_dataset( + loaded_instance + ) prepared_instance = task.prepare_for_task( loaded_instance, input_columns=self.input_columns, @@ -445,6 +454,16 @@ def run( self.input_columns = model_session.input_columns self.output_columns = model_session.output_columns + preprocessor = None + if model_session.preprocessing and model_session.preprocessing.get( + "steps" + ): + from DashAI.back.preprocessing.session_preprocessor import ( + load_final_preprocessor, + ) + + preprocessor = load_final_preprocessor(model_session) + try: run_model_class = component_registry[run.model_name]["class"] except Exception as e: @@ -527,6 +546,9 @@ def run( log.exception(e) raise JobError(str(e)) from e try: + if preprocessor is not None: + loaded_dataset = preprocessor.transform_dataset(loaded_dataset) + loaded_dataset = split_dataset( loaded_dataset, train_indexes=train_idx, @@ -596,6 +618,7 @@ def run( splits=splits, task=task, same_dataset=same_dataset, + preprocessor=preprocessor, ) else: raise JobError(f"{explainer_scope} is an invalid explainer type") diff --git a/DashAI/back/job/model_job.py b/DashAI/back/job/model_job.py index 7ae8528c7..2fe6a7047 100644 --- a/DashAI/back/job/model_job.py +++ b/DashAI/back/job/model_job.py @@ -25,6 +25,64 @@ log = logging.getLogger(__name__) +def apply_persisted_preprocessing(model_session: ModelSession, x, y): + """Transform already-split fold data using the SessionPreprocessor that + PreprocessingJob fit on this session's training data, then narrow to the + resolved concrete input columns. y is returned unchanged: output columns + are always raw in v1, so the splitter already narrowed y correctly. + + Parameters + ---------- + model_session : ModelSession + The session whose preprocessing_artifacts_path holds one fitted + SessionPreprocessor per fold (plus a "final" one), persisted by + PreprocessingJob. + x : list of DatasetDict | DatasetDict + The splitter's output for the input side: a list of per-fold dicts + for Cross-Validation, or a single dict for Holdout. + y : list of DatasetDict | DatasetDict + The splitter's output for the output side, already narrowed to + model_session.output_columns. Returned unchanged. + + Returns + ------- + tuple + (x, y) with x's datasets transformed and narrowed to the resolved + input columns for each fold/holdout split. + """ + import os + import pickle + + from DashAI.back.preprocessing.column_ref import parse_column_refs, resolve_refs + + input_refs = parse_column_refs(model_session.input_column_refs or []) + is_cv = isinstance(x, list) + x_folds = x if is_cv else [x] + total_folds = len(x_folds) - 1 if is_cv else 0 + fold_names = [f"fold_{i}" for i in range(total_folds)] + ["final"] + + new_x = [] + for split_dict, fold_name in zip(x_folds, fold_names, strict=True): + artifact_path = os.path.join( + model_session.preprocessing_artifacts_path, f"{fold_name}.pkl" + ) + with open(artifact_path, "rb") as f: + preprocessor = pickle.load(f) + + transformed = preprocessor.transform_only(split_dict) + resolved_input_columns = resolve_refs( + input_refs, preprocessor.resolved_columns, preprocessor.resolved_slots + ) + + fold_x = { + split_name: dataset.select_columns(resolved_input_columns) + for split_name, dataset in transformed.items() + } + new_x.append(fold_x) + + return (new_x, y) if is_cv else (new_x[0], y) + + class ModelJob(BaseJob): """ModelJob class to run the model training.""" @@ -134,6 +192,12 @@ def run( # save the obtained splits into the database run.split_indexes = json.dumps(splits) + + model_session = preparation_results["model_session"] + if model_session.preprocessing and model_session.preprocessing.get( + "steps" + ): + x, y = apply_persisted_preprocessing(model_session, x, y) except Exception as e: log.exception(e) raise JobError( @@ -301,16 +365,56 @@ def _prepare_dataset_and_components( ), ) from e + has_preprocessing = bool( + model_session.preprocessing and model_session.preprocessing.get("steps") + ) + try: - # Prepare dataset for the task and get number of labels of the task - prepared_dataset = task.prepare_for_task( - dataset=loaded_dataset, - input_columns=model_session.input_columns, - output_columns=model_session.output_columns, - ) - n_labels = task.num_labels( - prepared_dataset, model_session.output_columns[0] - ) + if has_preprocessing: + from DashAI.back.preprocessing.column_ref import parse_column_refs + + input_refs = parse_column_refs(model_session.input_column_refs or []) + raw_input_names = [r.name for r in input_refs if r.kind == "raw"] + # Group-produced columns do not exist in loaded_dataset yet: + # only the raw subset can go through prepare_for_task before + # the converters run per fold (see apply_persisted_preprocessing, + # called from run() after splitting). + prepared_dataset = task.prepare_for_task( + dataset=loaded_dataset, + input_columns=raw_input_names, + output_columns=model_session.output_columns, + ) + n_labels = task.num_labels( + prepared_dataset, model_session.output_columns[0] + ) + # X keeps every raw column (not just the input ones): the + # converters may need columns that are not themselves final + # inputs. Y is safe to narrow now because v1 requires every + # output ColumnRef to be raw. + X = prepared_dataset + Y = prepared_dataset.select_columns(model_session.output_columns) + else: + # Prepare dataset for the task and get number of labels of the task + prepared_dataset = task.prepare_for_task( + dataset=loaded_dataset, + input_columns=model_session.input_columns, + output_columns=model_session.output_columns, + ) + n_labels = task.num_labels( + prepared_dataset, model_session.output_columns[0] + ) + # Divide the dataset into two datasets: + # one with the input columns and another with the output column. + # This reads the prepared dataset rather than the loaded one: a + # task may reorder or otherwise adjust the rows, and forecasting + # does, sorting them by date so the temporal splitter carves real + # periods of time. Selecting from the loaded dataset would drop + # that work on the floor. + X, Y = select_columns( + prepared_dataset, + model_session.input_columns, + model_session.output_columns, + ) except Exception as e: log.exception(e) raise JobError( @@ -318,25 +422,6 @@ def _prepare_dataset_and_components( for Task {model_session.task_name}""", ) from e - try: - # Divide the dataset into two datasets: - # one with the input columns and another with the output column. - # This reads the prepared dataset rather than the loaded one: a - # task may reorder or otherwise adjust the rows, and forecasting - # does, sorting them by date so the temporal splitter carves real - # periods of time. Selecting from the loaded dataset would drop - # that work on the floor. - X, Y = select_columns( - prepared_dataset, - model_session.input_columns, - model_session.output_columns, - ) - except Exception as e: - log.exception(e) - raise JobError( - f"Error selecting input and output columns from dataset {dataset.id}", - ) from e - try: # Get splits data from model session splits_data = json.loads(model_session.splits) @@ -465,4 +550,5 @@ def _prepare_dataset_and_components( "Y": Y, "splitter": splitter, "evaluation_strategy": evaluation_strategy, + "model_session": model_session, } diff --git a/DashAI/back/job/predict_job.py b/DashAI/back/job/predict_job.py index f509c101d..085e37537 100644 --- a/DashAI/back/job/predict_job.py +++ b/DashAI/back/job/predict_job.py @@ -53,6 +53,14 @@ def _run_prediction_pipeline( """ import numpy as np + if model_session.preprocessing and model_session.preprocessing.get("steps"): + from DashAI.back.preprocessing.session_preprocessor import ( + load_final_preprocessor, + ) + + preprocessor = load_final_preprocessor(model_session) + loaded_dataset = preprocessor.transform_dataset(loaded_dataset) + prepared_dataset = loaded_dataset.select_columns(model_session.input_columns) y_pred_proba = np.array(trained_model.predict(prepared_dataset)) y_pred = task.process_predictions( diff --git a/DashAI/back/job/preprocessing_job.py b/DashAI/back/job/preprocessing_job.py new file mode 100644 index 000000000..993d997d9 --- /dev/null +++ b/DashAI/back/job/preprocessing_job.py @@ -0,0 +1,189 @@ +import json +import logging +import os +import pickle +from typing import TYPE_CHECKING + +from kink import inject +from sqlalchemy import exc + +from DashAI.back.core.atomic import atomic_directory +from DashAI.back.dependencies.database.models import Dataset, ModelSession +from DashAI.back.job.base_job import BaseJob, JobError +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + parse_column_refs, + resolve_refs, +) +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.splitters.splits_payload import normalize_splits_payload + +if TYPE_CHECKING: + from sqlalchemy.orm import sessionmaker + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +class PreprocessingJob(BaseJob): + """Fits a session's ConverterSequence once, ahead of any model training. + + Fits per fold for Cross-Validation (plus a final fit on the full + training pool) or once for Holdout, and persists each fitted + SessionPreprocessor so ModelJob, predict_job and explainer_job can + reuse it without ever re-fitting on new data. + """ + + @inject + def set_status_as_delivered( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + model_session_id = self.kwargs["model_session_id"] + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + if model_session is None: + raise JobError( + f"Model session {model_session_id} does not exist in DB." + ) + try: + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + raise JobError("Error setting preprocessing status as delivered") from e + + @inject + def set_status_as_error( + self, session_factory: "sessionmaker" = lambda di: di["session_factory"] + ) -> None: + model_session_id = self.kwargs.get("model_session_id") + if model_session_id is None: + return + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + if model_session is None: + return + model_session.preprocessing_status = "failed" + try: + db.commit() + except exc.SQLAlchemyError as e: + log.exception(e) + + @inject + def get_job_name(self) -> str: + model_session_id = self.kwargs.get("model_session_id") + return f"Preprocessing: session {model_session_id}" + + @inject + def run(self) -> None: + from kink import di + + from DashAI.back.dataloaders.classes.dashai_dataset import load_dataset + + session_factory = di["session_factory"] + component_registry = di["component_registry"] + config = di["config"] + + model_session_id = self.kwargs["model_session_id"] + + with session_factory() as db: + model_session: ModelSession = db.get(ModelSession, model_session_id) + if not model_session: + raise JobError( + f"Model session {model_session_id} does not exist in DB." + ) + + try: + model_session.preprocessing_status = "pending" + db.commit() + + dataset = db.get(Dataset, model_session.dataset_id) + if not dataset: + raise JobError( + f"Dataset {model_session.dataset_id} does not exist in DB." + ) + loaded_dataset = load_dataset(f"{dataset.file_path}/dataset") + + sequence = ConverterSequence.model_validate( + model_session.preprocessing or {"steps": []} + ) + sequence.validate_scopes() + input_refs = parse_column_refs(model_session.input_column_refs or []) + + self.report_progress(0.05, "Splitting dataset") + splits_data = normalize_splits_payload(json.loads(model_session.splits)) + splitter_name = splits_data.get("splitter_name") + splitter = component_registry[splitter_name]["class"]( + splits_data=splits_data + ) + + y_for_split = loaded_dataset.select_columns( + model_session.output_columns + ) + x, _, _ = splitter.split(loaded_dataset, y_for_split) + + is_cv = isinstance(x, list) + x_folds = x if is_cv else [x] + total_folds = len(x_folds) - 1 if is_cv else 0 + + artifacts_dir = os.path.join( + str(config["PREPROCESSING_PATH"]), str(model_session.id) + ) + + final_transformed = None + final_resolved = None + with atomic_directory(artifacts_dir) as tmp_dir: + for i in range(total_folds): + self.report_progress( + 0.1 + 0.7 * (i / max(total_folds, 1)), + f"Fitting preprocessing for fold {i + 1}/{total_folds}", + ) + fold_preprocessor = SessionPreprocessor( + sequence, component_registry + ) + fold_preprocessor.fit_transform(x_folds[i]) + with open(os.path.join(tmp_dir, f"fold_{i}.pkl"), "wb") as f: + pickle.dump(fold_preprocessor, f) + + self.report_progress(0.85, "Fitting final preprocessing") + final_preprocessor = SessionPreprocessor( + sequence, component_registry + ) + final_transformed, final_resolved = ( + final_preprocessor.fit_transform(x_folds[-1]) + ) + with open(os.path.join(tmp_dir, "final.pkl"), "wb") as f: + pickle.dump(final_preprocessor, f) + + resolved_input_columns = resolve_refs( + input_refs, final_resolved, final_preprocessor.resolved_slots + ) + + self.report_progress(0.95, "Validating against the task") + task = component_registry[model_session.task_name]["class"]() + task.prepare_for_task( + dataset=final_transformed["train"], + input_columns=resolved_input_columns, + output_columns=model_session.output_columns, + ) + + model_session.input_columns = resolved_input_columns + model_session.preprocessing_artifacts_path = artifacts_dir + model_session.preprocessing_status = "ready" + model_session.preprocessing_error = None + db.commit() + except (TypeError, ValueError) as e: + log.exception(e) + model_session.preprocessing_status = "failed" + model_session.preprocessing_error = str(e) + db.commit() + raise JobError( + f"Preprocessing produced columns invalid for the task: {e}" + ) from e + except Exception as e: + log.exception(e) + model_session.preprocessing_status = "failed" + model_session.preprocessing_error = str(e) + db.commit() + raise JobError( + f"Error running preprocessing for session {model_session_id}: {e}" + ) from e diff --git a/DashAI/back/preprocessing/__init__.py b/DashAI/back/preprocessing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/DashAI/back/preprocessing/column_ref.py b/DashAI/back/preprocessing/column_ref.py new file mode 100644 index 000000000..652860a64 --- /dev/null +++ b/DashAI/back/preprocessing/column_ref.py @@ -0,0 +1,112 @@ +"""Column references used by session-level preprocessing. + +A ColumnRef identifies either a real column already present in a dataset +(RawColumnRef) or the not-yet-materialized output of a converter step in a +ConverterSequence (GroupColumnRef). Group references let the Models-module +wizard offer "whatever this converter produces" as an input column before +any fit has happened — the concrete names only exist once the +PreprocessingJob has fit the sequence (see session_preprocessor.py). +""" + +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field, TypeAdapter +from typing_extensions import Annotated + + +class RawColumnRef(BaseModel): + kind: Literal["raw"] = "raw" + name: str + + +class GroupColumnRef(BaseModel): + kind: Literal["group"] = "group" + step: int + # None (the default) means "every column step produced" — unchanged, + # backward-compatible behavior. A converter step's real output columns + # aren't always one homogeneous type (e.g. SimpleImputer with a scope + # that mixes categorical and numeric columns just preserves each one's + # own type), so `slot` lets a ref pick out only the columns of one + # declared type from that step's output, once a real fit has classified + # them (see SessionPreprocessor._classify_by_type). The slot name is a + # DashAI type's display_name(), e.g. "Categorical" or "Integer". + slot: Optional[str] = None + + +ColumnRef = Annotated[Union[RawColumnRef, GroupColumnRef], Field(discriminator="kind")] + +_ColumnRefListAdapter = TypeAdapter(List[ColumnRef]) + + +class ConverterStep(BaseModel): + converter: str + params: Dict[str, Any] = Field(default_factory=dict) + scope: List[ColumnRef] = Field(default_factory=list) + + +class ConverterSequence(BaseModel): + steps: List[ConverterStep] = Field(default_factory=list) + + def validate_scopes(self) -> None: + """Raise ValueError if any step's scope references itself or a later step. + + A step may only reference the output group of a step strictly before + it — this is what makes chaining acyclic without a separate graph + structure. + """ + for index, step in enumerate(self.steps): + for ref in step.scope: + if isinstance(ref, GroupColumnRef) and ref.step >= index: + raise ValueError( + f"Step {index} ('{step.converter}') scope references " + f"step {ref.step}, which is not strictly before it." + ) + + +def resolve_refs( + refs: List[Union[RawColumnRef, GroupColumnRef]], + resolved_columns: Dict[int, List[str]], + resolved_slots: Optional[Dict[int, Dict[str, List[str]]]] = None, +) -> List[str]: + """Flatten a list of ColumnRef into concrete column names. + + Parameters + ---------- + refs : list of RawColumnRef | GroupColumnRef + References to resolve, in the order they should appear in the result. + resolved_columns : dict + Maps a ConverterSequence step index to the concrete column names that + step produced in one specific fit (see SessionPreprocessor). + resolved_slots : dict, optional + Maps a step index to {type_name: [column names]}, the same step + output classified by real per-column type (see SessionPreprocessor. + _classify_by_type). Required only if some ref has a non-None `slot`. + + Returns + ------- + list of str + Concrete column names, in order. A GroupColumnRef with `slot=None` + expands to every column its step produced; with a `slot` set, only + to that step's columns of that declared type. + + Raises + ------ + KeyError + If a GroupColumnRef names a step with no entry in resolved_columns + (or, for a slot ref, no matching entry in resolved_slots) — the step + has not been fit yet, or produced no column of that type. + """ + names: List[str] = [] + for ref in refs: + if ref.kind == "raw": + names.append(ref.name) + elif ref.slot is None: + names.extend(resolved_columns[ref.step]) + else: + names.extend((resolved_slots or {})[ref.step][ref.slot]) + return names + + +def parse_column_refs(raw: List[dict]) -> List[Union[RawColumnRef, GroupColumnRef]]: + """Parse a list of plain dicts (as stored in JSON columns) into ColumnRef.""" + return _ColumnRefListAdapter.validate_python(raw) diff --git a/DashAI/back/preprocessing/session_preprocessor.py b/DashAI/back/preprocessing/session_preprocessor.py new file mode 100644 index 000000000..7ca6ff801 --- /dev/null +++ b/DashAI/back/preprocessing/session_preprocessor.py @@ -0,0 +1,247 @@ +"""Fits and applies a ConverterSequence against dataset partitions. + +SessionPreprocessor is fit once per session (per fold for Cross-Validation, +once for Holdout) by PreprocessingJob, then persisted so training, +prediction and explanation can reuse the exact fit without ever re-fitting +on new data (see load_final_preprocessor at the bottom of this module). +""" + +import os +import pickle +from typing import TYPE_CHECKING, Any, Dict, List, Tuple + +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) +from DashAI.back.preprocessing.column_ref import ConverterSequence, resolve_refs + +if TYPE_CHECKING: + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + +class SessionPreprocessor: + """Fits and applies converters from a ConverterSequence. + + Each step's scope may reference raw dataset columns or the output group + of an earlier step. Fitting always uses only the "train" entry of + whatever split dict is passed in, so no step ever sees validation or + test rows during fit. + """ + + def __init__(self, sequence: ConverterSequence, component_registry: Any): + self.sequence = sequence + self.component_registry = component_registry + self.fitted_converters: List[Any] = [] + self.resolved_columns: Dict[int, List[str]] = {} + self.resolved_slots: Dict[int, Dict[str, List[str]]] = {} + + def _instantiate(self, step) -> Any: + converter_class = self.component_registry[step.converter]["class"] + return converter_class(**step.params) + + @staticmethod + def _classify_by_type( + converter: Any, column_names: List[str] + ) -> Dict[str, List[str]]: + """Group a step's real output columns by their real, per-column type. + + Calls the now-fitted converter's own get_output_type(column_name) — + already implemented by every converter, and already accurate once + fitted (e.g. SimpleImputer's preserves the input column's own type + for "most_frequent"/"constant"). Most converters produce one + homogeneous type, so this is a single slot; a converter whose scope + mixed column types (e.g. SimpleImputer imputing a categorical and a + numeric column together) naturally splits into one slot per type, + with no converter-specific code needed here or in the converter + itself. + """ + slots: Dict[str, List[str]] = {} + for name in column_names: + output_type = converter.get_output_type(name) + type_name = ( + output_type.display_name() + if output_type is not None and hasattr(output_type, "display_name") + else "unknown" + ) + slots.setdefault(type_name, []).append(name) + return slots + + @staticmethod + def _transform_split(converter, dataset, scope_names, train_transformed): + """Transform one split's scoped columns, without crashing on 0 rows. + + A "test" (or similar) partition can legitimately have 0 rows — e.g. + a session that reserved nothing for the final refit — and several + sklearn transformers raise on an empty array. Since train_transformed + (computed first) has the same columns any non-empty split would + produce, an empty split reuses that shape with 0 rows instead of + calling the converter at all. + """ + scoped = dataset.select_columns(scope_names) + if scoped.num_rows == 0 and train_transformed is not None: + return train_transformed.select([]) + return converter.transform(scoped) + + def __getstate__(self): + """Exclude component_registry from pickling. + + The registry is only needed to instantiate converters during + fit_transform; a fitted preprocessor is persisted precisely so that + step never runs again. The registry itself is not picklable (it + holds RelationshipManager lambdas), so it must never travel with the + pickled object. + """ + state = self.__dict__.copy() + state["component_registry"] = None + return state + + def __setstate__(self, state): + self.__dict__.update(state) + + def fit_transform( + self, split: Dict[str, "DashAIDataset"] + ) -> Tuple[Dict[str, "DashAIDataset"], Dict[int, List[str]]]: + """Fit every step on split["train"] and transform every split present. + + Parameters + ---------- + split : dict + Whatever partitions are present (e.g. {"train", "validation"}, + {"train", "validation", "test"} or {"train", "test"}) are + transformed, but only "train" is ever used to fit. + + Returns + ------- + tuple + (transformed_split, resolved_columns) where transformed_split + has the same keys as split and resolved_columns maps each + step's index to the concrete column names it produced. + """ + current: Dict[str, "DashAIDataset"] = dict(split) + self.fitted_converters = [] + self.resolved_columns = {} + self.resolved_slots = {} + + for index, step in enumerate(self.sequence.steps): + scope_names = resolve_refs( + step.scope, self.resolved_columns, self.resolved_slots + ) + converter = self._instantiate(step) + + train_scope = current["train"].select_columns(scope_names) + converter = converter.fit(train_scope) + + train_transformed = converter.transform(train_scope) + transformed_by_split = {"train": train_transformed} + for split_name, dataset in current.items(): + if split_name == "train": + continue + transformed_by_split[split_name] = self._transform_split( + converter, dataset, scope_names, train_transformed + ) + + # A converter that only rewrites its scope columns in place (e.g. a + # scaler: "age" in, scaled "age" out) has no other way to expose + # its result, so the scope names ARE the group. But a converter + # like Bag-of-Words additionally keeps its scope column verbatim + # alongside brand-new derived columns (see BagOfWordsConverter. + # transform's docstring: "the source text column is preserved + # unchanged") — for those, the untouched scope column is a + # passthrough, not this step's own output, so it must not leak + # into the group a later step or the wizard's input selection can + # reference (it would still carry the pre-conversion dtype, e.g. + # Text, which is never valid as a resolved input column). + new_columns = [ + name + for name in train_transformed.column_names + if name not in scope_names + ] + self.resolved_columns[index] = ( + new_columns if new_columns else list(train_transformed.column_names) + ) + self.resolved_slots[index] = self._classify_by_type( + converter, self.resolved_columns[index] + ) + + new_current = {} + for split_name, dataset in current.items(): + if type(converter).CHANGES_ROW_COUNT: + new_current[split_name] = transformed_by_split[split_name] + else: + scope_indexes = [ + dataset.column_names.index(name) for name in scope_names + ] + new_current[split_name] = rebuild_dataset_with_transformed_columns( + dataset, + transformed_by_split[split_name], + scope_names, + scope_indexes, + ) + current = new_current + self.fitted_converters.append(converter) + + return current, self.resolved_columns + + def transform_only( + self, split: Dict[str, "DashAIDataset"] + ) -> Dict[str, "DashAIDataset"]: + """Apply already-fitted converters to new data, without fitting. + + Used after unpickling a SessionPreprocessor that was fit earlier (by + PreprocessingJob), to transform fold data at training time, or a + prediction/explanation input. + """ + current: Dict[str, "DashAIDataset"] = dict(split) + for index, converter in enumerate(self.fitted_converters): + scope_names = resolve_refs( + self.sequence.steps[index].scope, + self.resolved_columns, + self.resolved_slots, + ) + + train_transformed = None + if "train" in current: + train_transformed = converter.transform( + current["train"].select_columns(scope_names) + ) + + transformed_by_split = {} + if train_transformed is not None: + transformed_by_split["train"] = train_transformed + for split_name, dataset in current.items(): + if split_name == "train": + continue + transformed_by_split[split_name] = self._transform_split( + converter, dataset, scope_names, train_transformed + ) + + new_current = {} + for split_name, dataset in current.items(): + transformed = transformed_by_split[split_name] + if type(converter).CHANGES_ROW_COUNT: + new_current[split_name] = transformed + else: + scope_indexes = [ + dataset.column_names.index(name) for name in scope_names + ] + new_current[split_name] = rebuild_dataset_with_transformed_columns( + dataset, transformed, scope_names, scope_indexes + ) + current = new_current + return current + + def transform_dataset(self, dataset: "DashAIDataset") -> "DashAIDataset": + """Convenience wrapper for a single dataset (predict/explain use).""" + return self.transform_only({"train": dataset})["train"] + + +def load_final_preprocessor(model_session: Any) -> "SessionPreprocessor": + """Load the SessionPreprocessor fitted on the session's full training pool. + + Used by prediction and explanation, which must transform new raw data + the exact same way the model's training data was transformed, without + ever re-fitting on that new data. + """ + path = os.path.join(model_session.preprocessing_artifacts_path, "final.pkl") + with open(path, "rb") as f: + return pickle.load(f) diff --git a/DashAI/front/src/api/modelSession.test.ts b/DashAI/front/src/api/modelSession.test.ts new file mode 100644 index 000000000..6a13f142d --- /dev/null +++ b/DashAI/front/src/api/modelSession.test.ts @@ -0,0 +1,68 @@ +jest.mock("./api"); + +import api from "./api"; +import { createModelSession } from "./modelSession"; + +describe("createModelSession", () => { + beforeEach(() => { + jest.clearAllMocks(); + (api.post as jest.Mock).mockResolvedValue({ data: { id: "session-1" } }); + }); + + it("includes preprocessing and input_column_refs in the request body", async () => { + await createModelSession( + 1, + "TabularClassificationTask", + "session-1", + [], + ["label"], + [], + [], + [], + "holdout", + JSON.parse("{}"), + [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + }, + ], + [{ kind: "group", step: 0 }], + ); + + expect(api.post).toHaveBeenCalledWith( + "/v1/model-session/", + expect.objectContaining({ + preprocessing: [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + }, + ], + input_column_refs: [{ kind: "group", step: 0 }], + }), + ); + }); + + it("defaults preprocessing and input_column_refs to empty arrays", async () => { + await createModelSession( + 1, + "TabularClassificationTask", + "session-2", + ["age"], + ["label"], + [], + [], + [], + "holdout", + JSON.parse("{}"), + ); + + expect(api.post).toHaveBeenCalledWith( + "/v1/model-session/", + expect.objectContaining({ preprocessing: [], input_column_refs: [] }), + ); + }); +}); diff --git a/DashAI/front/src/api/modelSession.ts b/DashAI/front/src/api/modelSession.ts index bcdf0b89c..0c85e6f9e 100644 --- a/DashAI/front/src/api/modelSession.ts +++ b/DashAI/front/src/api/modelSession.ts @@ -1,5 +1,9 @@ import api from "./api"; -import type { IModelSession } from "../types/modelSession"; +import type { + IColumnRef, + IConverterStep, + IModelSession, +} from "../types/modelSession"; const endpointURL = "/v1/model-session"; @@ -26,6 +30,8 @@ export const createModelSession = async ( testMetrics: string[], evaluationStrategy: string, splitsValue: JSON, + preprocessing: IConverterStep[] = [], + inputColumnRefs: IColumnRef[] = [], ): Promise => { const data = { dataset_id: datasetId, @@ -38,6 +44,8 @@ export const createModelSession = async ( test_metrics: testMetrics, evaluation_strategy: evaluationStrategy, splits: splitsValue, + preprocessing: preprocessing, + input_column_refs: inputColumnRefs, }; const response = await api.post("/v1/model-session/", data); @@ -75,12 +83,16 @@ export const validateColumns = async ( datasetId: number, inputColumns: string[], outputColumns: string[], + inputRefs?: IColumnRef[], + converterOutputTypes?: Record, ): Promise => { const formData = { task_name: taskName, dataset_id: datasetId, inputs_columns: inputColumns, outputs_columns: outputColumns, + input_refs: inputRefs, + converter_output_types: converterOutputTypes, }; const response = await api.post( "/v1/model-session/validation", diff --git a/DashAI/front/src/components/models/AddModelDialog.jsx b/DashAI/front/src/components/models/AddModelDialog.jsx index 02e5c5e54..8812f3dcb 100644 --- a/DashAI/front/src/components/models/AddModelDialog.jsx +++ b/DashAI/front/src/components/models/AddModelDialog.jsx @@ -38,6 +38,7 @@ import { useTourContext } from "../tour/TourProvider"; import { checkIfHaveOptimazers } from "../../utils/schema"; import { getDatasetInfo } from "../../api/datasets"; import { useModels } from "./ModelsContext"; +import { getApiErrorMessage } from "../../utils/apiError"; const DEFAULT_INNER_CONFIG = { splitterType: null, // null means: derive from outer splitter on mount @@ -328,9 +329,16 @@ function AddModelDialog({ console.error("Unknown Error", error.message); } - enqueueSnackbar(t("models:error.createRun", { name }), { - variant: "error", - }); + enqueueSnackbar( + t("models:error.createRunReason", { + name, + reason: getApiErrorMessage( + error, + t("models:error.createRun", { name }), + ), + }), + { variant: "error" }, + ); } finally { setLoading(false); } diff --git a/DashAI/front/src/components/models/CreateSessionSteps.jsx b/DashAI/front/src/components/models/CreateSessionSteps.jsx index 0d16c3008..05d5a0da9 100644 --- a/DashAI/front/src/components/models/CreateSessionSteps.jsx +++ b/DashAI/front/src/components/models/CreateSessionSteps.jsx @@ -1,14 +1,21 @@ import { useState, useMemo, useEffect, useRef } from "react"; import PropTypes from "prop-types"; -import { Box, Typography } from "@mui/material"; +import { Box, Typography, CircularProgress } from "@mui/material"; import { useSnackbar } from "notistack"; import { useFormik } from "formik"; import { useTourContext } from "../tour/TourProvider"; import SetNameAndDatasetStep from "./SetNameAndDatasetStep"; import PrepareDatasetStep from "./modelSession/PrepareDatasetStep"; +import PreprocessingStep from "./modelSession/PreprocessingStep"; +import SelectColumnsStep from "./modelSession/SelectColumnsStep"; import DatasetAutocomplete from "../notebooks/notebookCreation/DatasetAutocomplete"; import { createModelSession } from "../../api/modelSession"; +import { forceRefreshNow } from "../../utils/jobPoller"; import { getComponents } from "../../api/component"; +import { + getDatasetInfo as getDatasetInfoRequest, + getDatasetTypes as getDatasetTypesRequest, +} from "../../api/datasets"; import { generateSequentialName, getNextAvailableName, @@ -18,6 +25,10 @@ import { useModels } from "./ModelsContext"; import StepperNavigationFooter from "../shared/StepperNavigationFooter"; import { hasPartition } from "../../utils/splitsPayload"; +const STEP_PREPARE_DATASET = "prepareDataset"; +const STEP_PREPROCESSING = "preprocessing"; +const STEP_SELECT_COLUMNS = "selectColumns"; + function CreateSessionSteps({ backHome, selectedTask, @@ -54,9 +65,60 @@ function CreateSessionSteps({ evaluation_strategy: "", splits: {}, runs: [], + applyPreprocessing: false, + preprocessing: [], + input_column_refs: [], }); const [nextEnabled, setNextEnabled] = useState(false); + const [currentStep, setCurrentStep] = useState(STEP_PREPARE_DATASET); + + const [datasetInfo, setDatasetInfo] = useState({}); + const [datasetTypes, setDatasetTypes] = useState({}); + const [infoLoading, setInfoLoading] = useState(false); + + useEffect(() => { + if (!selectedDataset?.id) { + setDatasetInfo({}); + setDatasetTypes({}); + return; + } + let cancelled = false; + setInfoLoading(true); + (async () => { + try { + const [fetchedInfo, fetchedTypes] = await Promise.all([ + getDatasetInfoRequest(selectedDataset.id), + getDatasetTypesRequest(selectedDataset.id), + ]); + if (cancelled) return; + setDatasetInfo(fetchedInfo); + setDatasetTypes(fetchedTypes); + } catch (error) { + if (!cancelled) { + enqueueSnackbar(t("experiments:error.errorFetchingDatasetInfo")); + console.error("Error fetching dataset info:", error); + } + } finally { + if (!cancelled) setInfoLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [selectedDataset?.id]); + + const steps = useMemo( + () => [ + STEP_PREPARE_DATASET, + ...(newExp.applyPreprocessing ? [STEP_PREPROCESSING] : []), + STEP_SELECT_COLUMNS, + ], + [newExp.applyPreprocessing], + ); + const currentStepIndex = Math.max(0, steps.indexOf(currentStep)); + const isFirstStep = currentStepIndex === 0; + const isLastStep = currentStepIndex === steps.length - 1; const handleDatasetChange = (newDataset) => { setSelectedDataset(newDataset); @@ -66,6 +128,7 @@ function CreateSessionSteps({ dataset: newDataset, input_columns: [], output_columns: [], + input_column_refs: [], splits: {}, })); if ( @@ -144,10 +207,32 @@ function CreateSessionSteps({ return () => setSessionRightContent(null); }, [selectedDataset]); - const isNextEnabled = - formik.values.name.trim().length >= 4 && - selectedDataset !== null && - nextEnabled; + const isNextEnabled = isFirstStep + ? formik.values.name.trim().length >= 4 && + selectedDataset !== null && + nextEnabled + : nextEnabled; + + const goToStep = (step) => { + setNextEnabled(false); + setCurrentStep(step); + }; + + const handleFooterBack = () => { + if (isFirstStep) { + backHome(); + return; + } + goToStep(steps[currentStepIndex - 1]); + }; + + const handleFooterNext = () => { + if (isLastStep) { + formik.handleSubmit(); + return; + } + goToStep(steps[currentStepIndex + 1]); + }; const createSession = async (sessionName) => { try { @@ -182,6 +267,8 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), + newExp.preprocessing, + newExp.input_column_refs, ); } catch (createError) { if (createError?.response?.status === 409) { @@ -198,12 +285,21 @@ function CreateSessionSteps({ hasTest ? allMetricNames : [], newExp.evaluation_strategy, JSON.stringify(newExp.splits), + newExp.preprocessing, + newExp.input_column_refs, ); } else { throw createError; } } + // The session's PreprocessingJob (if any) is already enqueued by the + // time this response comes back — force an immediate poll so the job + // queue widget picks it up right away instead of waiting for its next + // 1s tick, which the job can easily finish before (see + // ConfigureAndUploadDatasetStep.jsx for the same pattern). + forceRefreshNow(); + enqueueSnackbar(t("models:message.sessionCreatedSuccess"), { variant: "success", }); @@ -226,6 +322,18 @@ function CreateSessionSteps({ } }; + const stepTitle = { + [STEP_PREPARE_DATASET]: t("models:label.prepareDataset"), + [STEP_PREPROCESSING]: t("models:label.preprocessingOptional"), + [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsTitle"), + }[currentStep]; + + const stepSubtitle = { + [STEP_PREPARE_DATASET]: t("models:label.selectDatasetAndPrepare"), + [STEP_PREPROCESSING]: t("models:label.preprocessingOptionalDescription"), + [STEP_SELECT_COLUMNS]: t("models:label.selectColumnsDescription"), + }[currentStep]; + return ( - {t("models:label.prepareDataset")} + {stepTitle} - {t("models:label.selectDatasetAndPrepare")} + {stepSubtitle} @@ -255,30 +363,67 @@ function CreateSessionSteps({ gap: 4, }} > - - - {selectedDataset && ( - + + + {selectedDataset && ( + + )} + + )} + + {currentStep === STEP_PREPROCESSING && selectedDataset && ( + )} + + {currentStep === STEP_SELECT_COLUMNS && selectedDataset && ( + <> + {infoLoading ? ( + + + + ) : ( + + )} + + )} diff --git a/DashAI/front/src/components/models/ManualPredictionsTable.jsx b/DashAI/front/src/components/models/ManualPredictionsTable.jsx index 76646adcb..84054d0f4 100644 --- a/DashAI/front/src/components/models/ManualPredictionsTable.jsx +++ b/DashAI/front/src/components/models/ManualPredictionsTable.jsx @@ -38,6 +38,7 @@ import { } from "../../api/predict"; import { enqueuePredictionJob } from "../../api/job"; import { getModelSessionById } from "../../api/modelSession"; +import { rawColumnsNeededFor } from "./modelSession/sessionColumnRefs"; import { startJobPolling } from "../../utils/jobPoller"; import { getTargetDecimals, @@ -245,13 +246,33 @@ export default function ManualPredictionsTable({ }; }, [run, session]); - const inputColumns = modelSession?.input_columns ?? EMPTY_ARRAY; + // Manual prediction submits raw values that the backend runs through the + // session's persisted preprocessor before predicting — it only ever + // accepts real dataset columns (BaseTask.process_manual_input rejects + // anything else), never a converter's already-resolved output name like + // "pca_1". A session with no preprocessing has input_column_refs already + // wrapping input_columns 1:1, so this resolves to the same thing then. + const inputColumns = useMemo(() => { + if (!modelSession) return EMPTY_ARRAY; + const refs = modelSession.input_column_refs; + if (!refs || refs.length === 0) + return modelSession.input_columns ?? EMPTY_ARRAY; + return rawColumnsNeededFor(refs, modelSession.preprocessing?.steps || []); + }, [modelSession]); const createEmptyRow = useCallback(() => { if (!inputSample || inputColumns.length === 0) return {}; - const randomIndex = Math.floor( - Math.random() * inputSample[inputColumns[0]].length, + // Defensive: a column missing from the sample (shouldn't happen once + // inputColumns is raw-keyed to match inputSample/inputTypes, but this is + // exactly the shape of bug that used to crash the whole tab) falls back + // to an empty prefill instead of throwing — the user can still type a + // value by hand. + const sampleColumn = inputColumns.find( + (col) => inputSample[col]?.length > 0, ); + const randomIndex = sampleColumn + ? Math.floor(Math.random() * inputSample[sampleColumn].length) + : 0; const row = {}; inputColumns.forEach((col) => { const typeInfo = inputTypes[col]; @@ -264,7 +285,7 @@ export default function ManualPredictionsTable({ row[col] = typeInfo.categories[randomIndex % typeInfo.categories.length]; } else { - row[col] = inputSample[col][randomIndex]; + row[col] = inputSample[col]?.[randomIndex]; } }); return row; diff --git a/DashAI/front/src/components/models/ModelsContext.jsx b/DashAI/front/src/components/models/ModelsContext.jsx index 8b6f3b2e7..f76df41e4 100644 --- a/DashAI/front/src/components/models/ModelsContext.jsx +++ b/DashAI/front/src/components/models/ModelsContext.jsx @@ -10,6 +10,7 @@ import { useTranslation } from "react-i18next"; import { useSharedDatasets } from "../../contexts/DatasetsContext"; import { useSessions } from "../../hooks/models/useSessions"; import { useModelComponents } from "../../hooks/models/useModelComponents"; +import { useJobTracker } from "../../hooks/useJobPolling"; const ModelsContext = createContext(null); export const useModels = () => useContext(ModelsContext); @@ -165,6 +166,25 @@ export function ModelsProvider({ children }) { fetchTasks(); }, [i18n.language]); + // Track the selected session's PreprocessingJob through the same shared + // job-polling mechanism the Job Queue widget itself uses (jobPoller.js), + // instead of polling preprocessing_status on an independent timer — this + // is what every other job-backed "processing" indicator in the app does + // (RunnerDialog, ComponentDownloadControl, prediction/explainer panels, + // ...). Sharing the exact same poll loop for the exact same job id is what + // keeps this indicator and the widget from ever showing contradictory + // states. Refreshing the whole session list on success/error is enough, + // since ModelsContent re-derives `selectedSession` from it. No-op for + // sessions with no preprocessing steps (preprocessing_job_id stays null). + const hasPendingPreprocessing = + (selectedSession?.preprocessing?.steps || []).length > 0 && + selectedSession?.preprocessing_status === "pending"; + useJobTracker( + hasPendingPreprocessing ? selectedSession?.preprocessing_job_id : null, + fetchSessions, + fetchSessions, + ); + // Memoized — this context wraps the entire models page tree, so a fresh // object literal every render would force every consumer (RunCard, // ModelDetailView, ModelsRightBar, ...) to re-render whenever ANY field diff --git a/DashAI/front/src/components/models/SessionVisualization.jsx b/DashAI/front/src/components/models/SessionVisualization.jsx index 63d54c407..f268842fd 100644 --- a/DashAI/front/src/components/models/SessionVisualization.jsx +++ b/DashAI/front/src/components/models/SessionVisualization.jsx @@ -1,7 +1,14 @@ import React, { useState, useEffect } from "react"; import { useStrategyKind } from "../../hooks/useStrategyKind"; import { STRATEGY_KINDS } from "../../utils/splitsPayload"; -import { Box, Typography, Divider, Button, ToggleButton } from "@mui/material"; +import { + Box, + Typography, + Divider, + Button, + ToggleButton, + CircularProgress, +} from "@mui/material"; import { useTheme } from "@mui/material/styles"; import { useParams, useNavigate } from "react-router-dom"; import { PlayArrow } from "@mui/icons-material"; @@ -300,6 +307,55 @@ export default function SessionVisualization() { ); } + // Sessions with preprocessing steps run a PreprocessingJob (fit/transform + // on train, persist to disk) right after creation — no Run can train, and + // nothing about the session is safe to show, until it finishes. + const hasPreprocessing = (session.preprocessing?.steps || []).length > 0; + if (hasPreprocessing && session.preprocessing_status === "failed") { + return ( + + + {t("models:label.preprocessingFailed")} + + {session.preprocessing_error && ( + + {session.preprocessing_error} + + )} + + ); + } + if (hasPreprocessing && session.preprocessing_status === "pending") { + return ( + + + + {t("models:label.preprocessingInProgress")} + + + ); + } + return ( <> getColorByColumnType(type, theme), + color: "#fff", + fontWeight: 600, + fontSize: "0.65rem", + height: "18px", + ml: 1, + }} + /> + ); +} + +TypeChip.propTypes = { type: PropTypes.string }; + +function RefChip({ refKey, label, columnTypes }) { + return ( + + {label} + + + } + /> + ); +} + +RefChip.propTypes = { + refKey: PropTypes.string.isRequired, + label: PropTypes.string.isRequired, + columnTypes: PropTypes.object.isRequired, +}; + +/** + * Mirrors the notebook's own ConverterParametersTable (same columns, same + * MaterialReactTable setup): a compact two-column table with one row per + * converter attribute, values rendered as wrapped chips (each with a + * colored type badge) instead of plain text, since a session converter's + * scope/output are references to columns or groups, not literal strings. + */ +function SessionConverterParametersTable({ + step, + optionLabels, + columnTypes, + outputEntries, + t, + localization, +}) { + const paramColumns = [ + { accessorKey: "key", header: t("common:parameter"), grow: 1 }, + { accessorKey: "value", header: t("common:value"), grow: 4 }, + ]; + + const paramRows = [ + { + // Plain comma-joined text, matching the notebook's own + // ConverterParametersTable (ConverterBox.jsx) — no chips here, since + // that's how Notebooks shows an already-applied converter's scope. + key: t("datasets:label.scopeColumns"), + value: step.scope + .map((ref) => { + const key = refToKey(ref); + return optionLabels[key] || key; + }) + .join(", "), + }, + { + // Usually one chip; more than one when this step's scope mixed + // column types (e.g. SimpleImputer preserving both a categorical and + // a numeric column), so each declared slot gets its own chip. + key: t("datasets:label.converterOutput"), + value: ( + + {outputEntries.map(({ key, label }) => ( + + ))} + + ), + }, + ]; + + const table = useMaterialReactTable({ + columns: paramColumns, + data: paramRows, + muiTableBodyCellProps: { sx: { whiteSpace: "pre" } }, + localization, + initialState: { density: "compact" }, + enablePagination: false, + enableTopToolbar: false, + enableBottomToolbar: false, + enableColumnActions: false, + enableSorting: false, + enableColumnFilter: false, + muiTablePaperProps: { elevation: 0 }, + }); + + return ; +} + +SessionConverterParametersTable.propTypes = { + step: PropTypes.object.isRequired, + optionLabels: PropTypes.object.isRequired, + columnTypes: PropTypes.object.isRequired, + outputEntries: PropTypes.arrayOf( + PropTypes.shape({ + key: PropTypes.string.isRequired, + label: PropTypes.string.isRequired, + }), + ).isRequired, + t: PropTypes.func.isRequired, + localization: PropTypes.object.isRequired, +}; + +/** + * A single applied-converter card, styled after the notebook's own + * ConverterBox (icon + real component display name + description + + * parameters table), but built against the session's preprocessing step + * shape (`{converter, scope, outputSlots}`) instead of the notebook's + * (`{parameters: {scope, target}}`), and with a "Salida" row showing the + * converter's declared output type(s) — usually one chip, more than one + * when the step's scope mixed column types — since that group doesn't + * exist as a real column until the session is created. + */ +function SessionConverterCard({ + step, + index, + displayName, + description, + onDelete, + datasetTypes, + preprocessing, + stepDisplayNames, +}) { + const theme = useTheme(); + const { t } = useTranslation(["datasets", "models", "common"]); + const localization = useTableLocalization(); + + const { columnTypes } = buildColumnKeysAndTypes({ + datasetTypes, + preprocessing, + }); + // Every earlier step's group label(s) use its resolved display name (e.g. + // "Bag of Words: output"), matching this card's own output label(s) + // below — not the raw registry name buildColumnKeysAndTypes falls back + // to when it has no display-name lookup of its own. One label per + // declared slot, so a step whose scope mixed types (more than one slot) + // gets one distinguishable label per slot. + const optionLabels = {}; + preprocessing.forEach((s, i) => { + const name = stepDisplayNames[i]; + const slots = s.outputSlots?.length > 0 ? s.outputSlots : [{ slot: null }]; + slots.forEach(({ slot }) => { + const key = groupKey(i, slot); + optionLabels[key] = slot + ? `${name}: output (${slot})` + : `${name}: output`; + }); + }); + const ownSlots = + step.outputSlots?.length > 0 ? step.outputSlots : [{ slot: null }]; + const outputEntries = ownSlots.map(({ slot }) => { + const key = groupKey(index, slot); + return { key, label: optionLabels[key] || `${displayName}: output` }; + }); + + return ( + + + + + {displayName} + + + onDelete(index)} + aria-label={t("common:remove")} + > + + + + + + {description && ( + + {description} + + )} + + + + ); +} + +SessionConverterCard.propTypes = { + step: PropTypes.object.isRequired, + index: PropTypes.number.isRequired, + displayName: PropTypes.string.isRequired, + description: PropTypes.string, + onDelete: PropTypes.func.isRequired, + datasetTypes: PropTypes.object, + preprocessing: PropTypes.array, + stepDisplayNames: PropTypes.arrayOf(PropTypes.string).isRequired, +}; + +/** + * Cards for every converter already added to the session's preprocessing + * sequence, mirroring the notebook module's applied-tool cards. + * + * Deleting a converter cascades: any later converter whose scope + * references its output group would be left pointing at a step that no + * longer exists, so every converter configured after it is removed too. + * Confirmed first via the same `DeleteConfirmationModal` + + * `ItemsToDeleteList` the notebook's own converter deletion uses, so the + * user sees exactly what else is about to go before confirming. + */ +export default function AppliedConvertersView({ + newExp, + setNewExp, + datasetTypes, +}) { + const { t } = useTranslation(["experiments", "models", "datasets", "common"]); + const theme = useTheme(); + const { setPendingDropTool } = useExplorersAndConverters(); + const [convertersMeta, setConvertersMeta] = useState({}); + const [deleteIndex, setDeleteIndex] = useState(null); + const [isDragOver, setIsDragOver] = useState(false); + const [isDragging, setIsDragging] = useState(false); + const steps = newExp.preprocessing || []; + + // A converter is draggable from SessionConvertersRightBar's ToolList/ + // ToolGrid for free (drag-start lives in the shared ToolListItem/ + // ToolGridItem, not in those wrappers), so this view only needs to be a + // drop target — same "application/x-dashai-tool" payload and the same + // setPendingDropTool hand-off Notebooks' own NotebookView.jsx uses, which + // ToolList/ToolGrid already resolve through their normal click-to-add path. + useEffect(() => { + const onStart = (e) => { + if (e.dataTransfer.types.includes("application/x-dashai-tool")) { + setIsDragging(true); + } + }; + const onEnd = () => { + setIsDragging(false); + setIsDragOver(false); + }; + window.addEventListener("dragstart", onStart); + window.addEventListener("dragend", onEnd); + return () => { + window.removeEventListener("dragstart", onStart); + window.removeEventListener("dragend", onEnd); + }; + }, []); + + const handleDragOver = (e) => { + if (!e.dataTransfer.types.includes("application/x-dashai-tool")) return; + e.preventDefault(); + e.dataTransfer.dropEffect = "copy"; + }; + + const handleDragEnter = (e) => { + if (!e.dataTransfer.types.includes("application/x-dashai-tool")) return; + e.preventDefault(); + setIsDragOver(true); + }; + + const handleDragLeave = (e) => { + const related = e.relatedTarget; + if (!related || !e.currentTarget.contains(related)) { + setIsDragOver(false); + } + }; + + const handleDrop = (e) => { + e.preventDefault(); + setIsDragOver(false); + try { + const tool = JSON.parse( + e.dataTransfer.getData("application/x-dashai-tool"), + ); + if (tool?.name) setPendingDropTool(tool); + } catch { + // ignore invalid drops + } + }; + + useEffect(() => { + let cancelled = false; + getComponents({ selectTypes: ["Converter"] }) + .then((data) => { + if (cancelled) return; + const byName = Object.fromEntries( + (data || []).map((component) => [component.name, component]), + ); + setConvertersMeta(byName); + }) + .catch((error) => + console.error("Failed to fetch converter metadata:", error), + ); + return () => { + cancelled = true; + }; + }, []); + + const stepDisplayNames = steps.map( + (step) => convertersMeta[step.converter]?.display_name || step.converter, + ); + + const itemsToDelete = useMemo(() => { + if (deleteIndex === null) return []; + return steps.slice(deleteIndex).map((step, i) => ({ + id: deleteIndex + i, + type: "converter", + converter: stepDisplayNames[deleteIndex + i], + })); + }, [steps, deleteIndex, stepDisplayNames]); + + const handleConfirmDelete = () => { + setNewExp({ + ...newExp, + preprocessing: steps.slice(0, deleteIndex), + }); + setDeleteIndex(null); + }; + + return ( + + {isDragging && ( + + + {t("datasets:label.dropToolHere")} + + + )} + + {steps.length === 0 ? ( + + {t("experiments:label.noConverterAdded")} + + ) : ( + steps.map((step, index) => { + const meta = convertersMeta[step.converter]; + return ( + setDeleteIndex(i)} + datasetTypes={datasetTypes} + preprocessing={steps} + stepDisplayNames={stepDisplayNames} + /> + ); + }) + )} + + setDeleteIndex(null)} + onConfirm={handleConfirmDelete} + content={ + + + {t("datasets:label.deleteConverterConfirmation", { + converter: stepDisplayNames[deleteIndex], + })} + + + + } + /> + + ); +} + +AppliedConvertersView.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + datasetTypes: PropTypes.object, +}; diff --git a/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx new file mode 100644 index 000000000..a1865f03c --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/AppliedConvertersView.test.jsx @@ -0,0 +1,211 @@ +import React from "react"; +import { screen, fireEvent } from "@testing-library/react"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import { getComponents } from "../../../api/component"; +import { useExplorersAndConverters } from "../../notebooks/context/ExplorersAndConvertersContext"; + +jest.mock("../../../api/component", () => ({ + getComponents: jest.fn(), +})); + +jest.mock("../../notebooks/context/ExplorersAndConvertersContext", () => ({ + useExplorersAndConverters: jest.fn(), +})); + +import AppliedConvertersView from "./AppliedConvertersView"; + +const renderView = (props) => + renderWithProviders(); + +const makeDataTransfer = (payload) => ({ + types: payload ? ["application/x-dashai-tool"] : [], + getData: () => (payload ? JSON.stringify(payload) : ""), + dropEffect: "none", +}); + +const AVAILABLE_CONVERTERS = [ + { + name: "Binarizer", + display_name: "Binarizador", + description: "Binariza datos.", + metadata: { output_type: "Integer" }, + }, + { + name: "BagOfWordsConverter", + display_name: "Bag of Words", + description: "Bolsa de palabras.", + metadata: { output_type: "Integer" }, + }, +]; + +const datasetTypes = { age: { type: "Integer" }, text: { type: "Text" } }; + +describe("AppliedConvertersView", () => { + let setPendingDropTool; + + beforeEach(() => { + getComponents.mockResolvedValue(AVAILABLE_CONVERTERS); + setPendingDropTool = jest.fn(); + useExplorersAndConverters.mockReturnValue({ setPendingDropTool }); + }); + + it("shows a message when there are no converters", () => { + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + datasetTypes, + }); + + expect(screen.getByText("No converter was added.")).toBeInTheDocument(); + }); + + it("shows a converter's real display name, scope and output type chip", async () => { + const newExp = { + preprocessing: [ + { + converter: "Binarizer", + params: { threshold: 0.5 }, + scope: [{ kind: "raw", name: "age" }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, datasetTypes }); + + expect(await screen.findByText("Binarizador")).toBeInTheDocument(); + expect(screen.getByText("age")).toBeInTheDocument(); + expect(screen.getByText("Binarizador: output")).toBeInTheDocument(); + expect(screen.getAllByText("Integer").length).toBeGreaterThan(0); + }); + + it("shows a chained converter's scope as the earlier converter's output group", async () => { + const newExp = { + preprocessing: [ + { + converter: "BagOfWordsConverter", + params: {}, + scope: [{ kind: "raw", name: "text" }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "group", step: 0 }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + ], + }; + + renderView({ newExp, setNewExp: () => {}, datasetTypes }); + + await screen.findByText("Bag of Words"); + // The second card's scope shows the first converter's output label. + expect(screen.getAllByText("Bag of Words: output").length).toBe(2); + }); + + it("shows one chip per declared slot when a step's scope mixed column types", async () => { + const newExp = { + preprocessing: [ + { + converter: "SimpleImputer", + params: { strategy: "most_frequent" }, + scope: [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "text" }, + ], + outputSlots: [ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Categorical", type: "Categorical", dtype: null }, + ], + }, + ], + }; + + renderView({ + newExp, + setNewExp: () => {}, + datasetTypes: { + ...datasetTypes, + // "SimpleImputer" isn't in AVAILABLE_CONVERTERS, so its display + // name falls back to the raw converter name — fine here, this + // test only cares about the output chips. + }, + }); + + expect( + await screen.findByText("SimpleImputer: output (Integer)"), + ).toBeInTheDocument(); + expect( + screen.getByText("SimpleImputer: output (Categorical)"), + ).toBeInTheDocument(); + }); + + it("cascades deletion to every converter configured after the deleted one", async () => { + const setNewExp = jest.fn(); + const newExp = { + preprocessing: [ + { + converter: "BagOfWordsConverter", + params: {}, + scope: [{ kind: "raw", name: "text" }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + { + converter: "Binarizer", + params: {}, + scope: [{ kind: "group", step: 0 }], + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + ], + }; + + renderView({ newExp, setNewExp, datasetTypes }); + + await screen.findByText("Bag of Words"); + const deleteButtons = screen.getAllByRole("button", { name: "Remove" }); + fireEvent.click(deleteButtons[0]); + + // Deleting the first converter cascades to the second (its scope + // references the first one's output group), so the confirmation modal + // must warn about both before anything is actually removed. + expect( + await screen.findByText("The following items will be deleted:"), + ).toBeInTheDocument(); + expect(setNewExp).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "Delete" })); + + expect(setNewExp).toHaveBeenCalledWith({ + ...newExp, + preprocessing: [], + }); + }); + + it("hands a dropped tool off to setPendingDropTool, just like Notebooks' drop target", () => { + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + datasetTypes, + }); + + const dropZone = screen.getByText("No converter was added.").parentElement; + const tool = { name: "Binarizer", display_name: "Binarizador" }; + fireEvent.drop(dropZone, { dataTransfer: makeDataTransfer(tool) }); + + expect(setPendingDropTool).toHaveBeenCalledWith(tool); + }); + + it("ignores a drop with no recognizable tool payload", () => { + renderView({ + newExp: { preprocessing: [] }, + setNewExp: () => {}, + datasetTypes, + }); + + const dropZone = screen.getByText("No converter was added.").parentElement; + fireEvent.drop(dropZone, { dataTransfer: makeDataTransfer(null) }); + + expect(setPendingDropTool).not.toHaveBeenCalled(); + }); +}); diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx index 20351a932..0b8c35d11 100644 --- a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.jsx @@ -15,6 +15,8 @@ import { useTranslation } from "react-i18next"; function DivideDatasetColumns({ allColumnNames, columnTypes = {}, + inputOptionNames, + optionLabels = {}, selectedInputColumnNames, onInputColumnNamesChange, selectedOutputColumnNames, @@ -28,6 +30,8 @@ function DivideDatasetColumns({ const { t } = useTranslation(["experiments", "common"]); const theme = useTheme(); + const inputOptions = inputOptionNames || allColumnNames; + const handleInputAutocompleteChange = (event, newValue) => { onInputColumnNamesChange(newValue); }; @@ -36,13 +40,12 @@ function DivideDatasetColumns({ onOutputColumnNamesChange(newValue); }; - const getColumnLabel = (columnName) => { - const columnType = columnTypes[columnName]; - if (columnType && columnType.type) { - return `${columnName} (${columnType.type})`; - } - return columnName; - }; + // Each option's identity (used for selection/value tracking) doesn't have + // to be its display text — `optionLabels` lets a caller show a readable + // label (e.g. a converter's output group label) for an option whose real + // identity is an internal synthetic key. Defaults to the identity itself, + // so a plain raw column name (with no entry in optionLabels) is unaffected. + const getOptionLabel = (option) => optionLabels[option] || option; const renderColumnOption = (props, option) => { const { key, ...otherProps } = props; @@ -58,7 +61,7 @@ function DivideDatasetColumns({ {...otherProps} sx={{ display: "flex", alignItems: "center", gap: 2 }} > - {option} + {getOptionLabel(option)} {columnType && columnType.type && ( - {option} + {getOptionLabel(option)} ) : ( - option + getOptionLabel(option) ); return ; @@ -129,10 +132,10 @@ function DivideDatasetColumns({ data-tour="dataset-input-columns-autocomplete" multiple id="dataset-input-columns-autocomplete" - options={allColumnNames} + options={inputOptions} value={selectedInputColumnNames} onChange={handleInputAutocompleteChange} - getOptionLabel={(option) => option} + getOptionLabel={getOptionLabel} renderOption={renderColumnOption} renderTags={renderTags} filterSelectedOptions @@ -146,14 +149,14 @@ function DivideDatasetColumns({ error={inputError} helperText={inputHelperText} placeholder={ - allColumnNames.length > 0 + inputOptions.length > 0 ? t("common:selectColumns") : t("common:loadingColumns") } /> )} sx={{ mb: 8 }} - disabled={disabled || allColumnNames.length === 0} + disabled={disabled || inputOptions.length === 0} /> option} + getOptionLabel={getOptionLabel} renderOption={renderColumnOption} renderTags={renderTags} filterSelectedOptions @@ -192,6 +195,11 @@ function DivideDatasetColumns({ DivideDatasetColumns.propTypes = { allColumnNames: PropTypes.arrayOf(PropTypes.string).isRequired, columnTypes: PropTypes.object, + // Input options can include synthetic keys (a converter's output group) + // beyond the raw dataset columns; output stays raw-only (see spec: every + // output ColumnRef must be raw). Defaults to allColumnNames. + inputOptionNames: PropTypes.arrayOf(PropTypes.string), + optionLabels: PropTypes.object, selectedInputColumnNames: PropTypes.arrayOf(PropTypes.string).isRequired, onInputColumnNamesChange: PropTypes.func.isRequired, selectedOutputColumnNames: PropTypes.arrayOf(PropTypes.string).isRequired, diff --git a/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx new file mode 100644 index 000000000..53a5d657e --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/DivideDatasetColumns.test.jsx @@ -0,0 +1,70 @@ +import React from "react"; +import { screen, fireEvent } from "@testing-library/react"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import DivideDatasetColumns from "./DivideDatasetColumns"; + +describe("DivideDatasetColumns", () => { + it("offers a converter's output group as an input column option, with its label and type chip", async () => { + const onInputColumnNamesChange = jest.fn(); + renderWithProviders( + {}} + />, + ); + + const input = screen.getByLabelText(/Input Columns/i); + fireEvent.mouseDown(input); + fireEvent.click(await screen.findByText("Binarizador: output")); + + expect(onInputColumnNamesChange).toHaveBeenCalledWith(["__group__0"]); + }); + + it("does not offer converter groups as output column options", () => { + renderWithProviders( + {}} + selectedOutputColumnNames={[]} + onOutputColumnNamesChange={() => {}} + />, + ); + + const output = screen.getByLabelText(/Output Columns/i); + fireEvent.mouseDown(output); + expect(screen.queryByText("Binarizador: output")).not.toBeInTheDocument(); + }); + + it("falls back to allColumnNames for input options when inputOptionNames is not passed", () => { + renderWithProviders( + {}} + selectedOutputColumnNames={[]} + onOutputColumnNamesChange={() => {}} + />, + ); + + const input = screen.getByLabelText(/Input Columns/i); + fireEvent.mouseDown(input); + expect(screen.getAllByText("age").length).toBeGreaterThan(0); + expect(screen.getAllByText("name").length).toBeGreaterThan(0); + }); +}); diff --git a/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx new file mode 100644 index 000000000..1ff3be744 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/FormSessionConverterSection.jsx @@ -0,0 +1,103 @@ +import React, { useState } from "react"; +import PropTypes from "prop-types"; +import { Box } from "@mui/material"; +import ScopeStepSessionConverter from "./ScopeStepSessionConverter"; +import ParameterStepConverter from "../../notebooks/converterCreation/ParameterStepConverter"; +import { resolveDeclaredOutputSlots } from "./sessionColumnRefs"; + +/** + * "Add a converter" form for the session wizard's preprocessing step — + * mirrors the notebook's FormConverterSection (same two-step scope/params + * flow, same ParameterStepConverter for the second step), but a session + * converter is never fit immediately: saving here only appends + * `{converter, params, scope}` to the session's local `preprocessing` list. + * The actual fit happens once, later, when the session is created (see the + * backend's PreprocessingJob) — never from this form. + */ +export default function FormSessionConverterSection({ + step, + setStep, + handleClose, + tool, + newExp, + setNewExp, + datasetTypes, + filePath, + hideButtons = false, +}) { + const [scope, setScope] = useState([]); + + const handleSaveConverter = async (params) => { + const outputSlots = resolveDeclaredOutputSlots({ + tool, + params, + scope, + datasetTypes, + preprocessing: newExp.preprocessing, + }); + const newStep = { + converter: tool.name, + params: params || {}, + scope, + outputSlots, + }; + setNewExp({ + ...newExp, + preprocessing: [...(newExp.preprocessing || []), newStep], + }); + handleClose(); + }; + + return ( + + {step === 0 && ( + 0 + ? () => setStep((s) => s + 1) + : () => handleSaveConverter({}) + } + /> + )} + + {step === 1 && ( + + )} + + ); +} + +FormSessionConverterSection.propTypes = { + step: PropTypes.number.isRequired, + setStep: PropTypes.func.isRequired, + handleClose: PropTypes.func.isRequired, + tool: PropTypes.object.isRequired, + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + datasetTypes: PropTypes.object.isRequired, + filePath: PropTypes.string, + hideButtons: PropTypes.bool, +}; diff --git a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx index 85859df9b..889e51d9d 100644 --- a/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx +++ b/DashAI/front/src/components/models/modelSession/PrepareDatasetStep.jsx @@ -1,26 +1,17 @@ -import React, { useEffect, useLayoutEffect, useState } from "react"; +import React, { useEffect, useState } from "react"; import PropTypes from "prop-types"; import { Grid, - CircularProgress, Box, Alert, AlertTitle, - Chip, + FormControlLabel, + Switch, + Typography, } from "@mui/material"; -import DivideDatasetColumns from "./DivideDatasetColumns"; import SplitDatasetRows from "./SplitDatasetRows"; -import { - getDatasetInfo as getDatasetInfoRequest, - getDatasetTypes as getDatasetTypesRequest, -} from "../../../api/datasets"; -import { getComponents as getComponentsRequest } from "../../../api/component"; -import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; -import { useSnackbar } from "notistack"; -import { getColorByColumnType } from "../../../utils"; import { useTranslation } from "react-i18next"; -import { Trans } from "react-i18next"; import { useModels } from "../ModelsContext"; import { buildSplitsPayload, @@ -29,13 +20,17 @@ import { SPLIT_TYPES, } from "../../../utils/splitsPayload"; /** - * Step of the experiment modal: Set the input and output columns to use for clasification - * and the splits for training, validation and testing. - * @param {object} newExp object that contains the Experiment Modal state - * @param {function} setNewExp updates the Eperimento Modal state (newExp) - * @param {function} setNextEnabled function to enable or disable the "Next" button in the modal - * @param {string} evaluationStrategy the evaluation strategy selected for the experiment, either holdout or cross-validation - * @param {function} setEvaluationStrategy function to update the evaluation strategy in the parent component (CreateSessionSteps) + * Step of the session wizard: configure the evaluation strategy, partitions, + * and whether preprocessing converters should be applied before column + * selection. Column selection itself lives in SelectColumnsStep. + * @param {object} newExp object that contains the Session wizard state + * @param {function} setNewExp updates the session wizard state (newExp) + * @param {function} setNextEnabled function to enable or disable the "Next" button + * @param {string} evaluationStrategy the evaluation strategy selected for the session + * @param {function} setEvaluationStrategy function to update the evaluation strategy + * @param {object} dataset the selected dataset + * @param {object} datasetInfo dataset metadata fetched by the parent step + * @param {boolean} infoLoading whether datasetInfo is still being fetched */ function PrepareDatasetStep({ newExp, @@ -44,35 +39,15 @@ function PrepareDatasetStep({ dataset, evaluationStrategy, setEvaluationStrategy, + datasetInfo, + infoLoading, }) { const { setSessionRightContent } = useModels(); - const [datasetInfo, setDatasetInfo] = useState({}); - const [datasetTypes, setDatasetTypes] = useState({}); - const { enqueueSnackbar } = useSnackbar(); - const [infoLoading, setInfoLoading] = useState(true); - const { t } = useTranslation(["experiments", "common"]); - - // null means "not fetched yet" — distinct from the empty-but-loaded shape - // getTaskRequirements falls back to when the task genuinely isn't found. - // The banner below only renders once this is non-null, otherwise it briefly - // interpolates its message with blank task name/types/cardinality. - const [taskRequirements, setTaskRequirements] = useState(null); + const { t } = useTranslation(["experiments", "models", "common"]); - const [inputColumnNames, setInputColumnNames] = useState( - newExp.input_columns, + const [applyPreprocessing, setApplyPreprocessing] = useState( + Boolean(newExp.applyPreprocessing), ); - const [outputColumnNames, setOutputColumnNames] = useState( - newExp.output_columns, - ); - - const columnsReady = - inputColumnNames.length >= 1 && outputColumnNames.length >= 1; - const [columnsAreValid, setColumnsAreValid] = useState(false); - // True until the current column selection has actually been checked against - // the backend at least once — distinct from columnsAreValid=false, so the - // banner doesn't flash red while columns are still being auto-selected or a - // check is in flight, only once a real valid/invalid result is known. - const [validationPending, setValidationPending] = useState(true); // Values submitted by the schema generated splitter form, and whether that // form currently reports a validation error. @@ -104,158 +79,26 @@ function PrepareDatasetStep({ const [splitsReady, setSplitsReady] = useState(false); - const getDatasetInfo = async () => { - if (!dataset?.id) return; - setInfoLoading(true); - setInputColumnNames([]); - setOutputColumnNames([]); - try { - const [fetchedDatasetInfo, fetchedDatasetTypes] = await Promise.all([ - getDatasetInfoRequest(dataset.id), - getDatasetTypesRequest(dataset.id), - ]); - setDatasetInfo(fetchedDatasetInfo); - setDatasetTypes(fetchedDatasetTypes); - - if (fetchedDatasetInfo) { - setDatasetPartitionsIndex({ - train: fetchedDatasetInfo.train_indices || [], - validation: fetchedDatasetInfo.val_indices || [], - test: fetchedDatasetInfo.test_indices || [], - }); - } - - if ( - fetchedDatasetInfo && - fetchedDatasetInfo.column_names && - fetchedDatasetInfo.column_names.length > 0 - ) { - const allNames = fetchedDatasetInfo.column_names; - if ( - inputColumnNames.length === 0 && - (!newExp.input_columns || newExp.input_columns.length === 0) - ) { - if (allNames.length > 1) { - setInputColumnNames(allNames.slice(0, -1)); - } else if (allNames.length === 1) { - setInputColumnNames([allNames[0]]); - } - } - - if ( - outputColumnNames.length === 0 && - (!newExp.output_columns || newExp.output_columns.length === 0) - ) { - if (allNames.length > 0) { - setOutputColumnNames([allNames[allNames.length - 1]]); - } - } - } - } catch (error) { - enqueueSnackbar(t("experiments:error.errorFetchingDatasetInfo")); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } finally { - setInfoLoading(false); - } - }; - - const getTaskRequirements = async () => { - try { - const taskComponents = await getComponentsRequest({ - selectTypes: ["Task"], - }); - - const currentTask = taskComponents.find( - (task) => task.name === newExp.task_name, - ); - if (currentTask) { - setTaskRequirements(currentTask); - } else { - enqueueSnackbar( - t("experiments:error.taskRequirementsNotFound", { - taskName: newExp.task_name, - }), - ); - setTaskRequirements({ - name: newExp.task_name, - metadata: { - inputs_types: [], - inputs_cardinality: "", - outputs_types: [], - outputs_cardinality: "", - }, - }); - } - } catch (error) { - enqueueSnackbar(t("experiments:error.errorFetchingTaskRequirements")); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - } - }; - - const validateColumns = async () => { - try { - if ( - !datasetInfo || - !datasetInfo.column_names || - datasetInfo.column_names.length === 0 - ) { - setColumnsAreValid(false); - return; - } - - if (inputColumnNames.length === 0 || outputColumnNames.length === 0) { - setColumnsAreValid(false); - return; - } - - const validation = await validateColumnsRequest( - newExp.task_name, - dataset.id, - inputColumnNames, - outputColumnNames, - ); - setColumnsAreValid(validation.dataset_status === "valid"); - } catch (error) { - enqueueSnackbar(t("experiments:error.errorFetchingColumnsValidation")); - if (error.response) { - console.error("Response error:", error.message); - } else if (error.request) { - console.error("Request error", error.request); - } else { - console.error("Unknown Error", error.message); - } - setColumnsAreValid(false); - } finally { - setValidationPending(false); - } - }; - - const updateExperiment = () => { + useEffect(() => { if ( - !datasetInfo || - !datasetInfo.column_names || - datasetInfo.column_names.length === 0 + datasetInfo && + (datasetInfo.train_indices || + datasetInfo.val_indices || + datasetInfo.test_indices) ) { - return; + setDatasetPartitionsIndex({ + train: datasetInfo.train_indices || [], + validation: datasetInfo.val_indices || [], + test: datasetInfo.test_indices || [], + }); } + }, [datasetInfo]); + const updateExperiment = () => { const updatedExpData = { ...newExp, - input_columns: inputColumnNames, - output_columns: outputColumnNames, evaluation_strategy: evaluationStrategy, + applyPreprocessing: applyPreprocessing, }; const splitterName = resolveSplitterName(strategyKind, cvType, holdoutType); @@ -282,87 +125,31 @@ function PrepareDatasetStep({ setNewExp(updatedExpData); }; - // Column validity depends on the columns, the dataset and the task, never on - // the split configuration. Gating it on the splits being ready made every - // split change re-check the columns over HTTP and blank the requirements - // banner in the meantime. - // - // When columnsReady is false we already know the selection is invalid - // (input or output is empty) without asking the backend, so - // validationPending goes straight to false — otherwise the requirements - // banner below stayed hidden forever after the user cleared a column, - // instead of showing why the selection is invalid. - // - // This runs as a layout effect (not a regular effect) so that when - // columns go from empty back to a ready selection — e.g. the dataset - // finishes loading and auto-selects defaults — validationPending flips - // back to true synchronously before the browser paints. A regular effect - // runs one commit too late: React would paint a frame with the stale - // "already validated" state (an incorrect red banner) against the new, - // not-yet-checked columns before the effect corrected it. - useLayoutEffect(() => { - if (!columnsReady) { - setColumnsAreValid(false); - setValidationPending(false); - return; - } - if ( - datasetInfo && - datasetInfo.column_names && - datasetInfo.column_names.length > 0 - ) { - setValidationPending(true); - validateColumns(); - } - }, [columnsReady, inputColumnNames, outputColumnNames, datasetInfo]); - useEffect(() => { - if (columnsAreValid && splitsReady && columnsReady) { + if (splitsReady) { updateExperiment(); setNextEnabled(true); } else { setNextEnabled(false); } }, [ - columnsReady, splitsReady, - columnsAreValid, splitType, splitterParams, - inputColumnNames, - outputColumnNames, cvType, holdoutType, strategyKind, groupColumn, evaluationStrategy, + applyPreprocessing, rowsPartitionsIndex, datasetPartitionsIndex, ]); - useEffect(() => { - getDatasetInfo(); - }, [dataset?.id]); - - useEffect(() => { - getTaskRequirements(); - }, []); - // Push SplitDatasetRows (or loading spinner) into the right bar useEffect(() => { if (infoLoading) { - setSessionRightContent( - - - , - ); + setSessionRightContent(null); return () => setSessionRightContent(null); } setSessionRightContent( @@ -388,7 +175,7 @@ function PrepareDatasetStep({ setStrategyKind={setStrategyKind} groupColumn={groupColumn} setGroupColumn={setGroupColumn} - inputColumnNames={inputColumnNames} + inputColumnNames={datasetInfo.column_names || []} taskName={newExp.task_name} />, ); @@ -405,71 +192,8 @@ function PrepareDatasetStep({ holdoutType, strategyKind, groupColumn, - inputColumnNames, ]); - const columnGroupsOf = (side) => { - const metadata = taskRequirements?.metadata ?? {}; - if (Array.isArray(metadata[side]) && metadata[side].length > 0) { - return metadata[side]; - } - const cardinality = metadata[`${side}_cardinality`]; - return [ - { - types: metadata[`${side}_types`] ?? [], - min: cardinality === "n" ? 0 : cardinality, - max: cardinality, - }, - ]; - }; - - const describeCardinality = ({ min, max }) => { - if (max === "n") { - return min ? t("experiments:label.cardinalityAtLeast", { min }) : "n"; - } - if (min === max) { - return String(max); - } - return t("experiments:label.cardinalityBetween", { min, max }); - }; - - const renderTypesAsChips = (typesList) => { - if (!typesList || typesList.length === 0) { - return {t("common:any")}; - } - - return ( - - {typesList.map((type, index) => ( - - - {index < typesList.length - 1 && ( - {t("common:or")} - )} - - ))} - - ); - }; - return ( {!infoLoading && datasetInfo.nan ? ( @@ -499,90 +223,29 @@ function PrepareDatasetStep({ ) : null ) : null} - {taskRequirements && !infoLoading && !validationPending && ( - - `${theme.palette[columnsAreValid ? "success" : "error"].main}40`, - border: (theme) => - `1px solid ${ - theme.palette[columnsAreValid ? "success" : "error"].main - }`, - }} - data-tour="models-validation-alert" - > - - {t( - columnsAreValid - ? "experiments:label.columnsValidRequirements" - : "experiments:label.columnsInvalidRequirements", - { taskName: taskRequirements.display_name }, - )} - - - {["inputs", "outputs"].map((side) => - columnGroupsOf(side).map((group, index) => ( - - - - The columns must be of the types - {renderTypesAsChips(group.types)} - , and they should have a cardinality of - - {{ cardinality: describeCardinality(group) }}. - - - - - )), - )} - - - )} - {!infoLoading ? ( - - - - ) : ( - - - - )} + + setApplyPreprocessing(event.target.checked)} + /> + } + label={t("models:label.applyPreprocessing")} + /> + + {t("models:label.applyPreprocessingDescription")} + + ); } @@ -600,9 +263,14 @@ PrepareDatasetStep.propTypes = { created: PropTypes.instanceOf(Date), last_modified: PropTypes.instanceOf(Date), runs: PropTypes.array, + applyPreprocessing: PropTypes.bool, }), setNewExp: PropTypes.func.isRequired, setNextEnabled: PropTypes.func.isRequired, dataset: PropTypes.object.isRequired, + evaluationStrategy: PropTypes.string, + setEvaluationStrategy: PropTypes.func, + datasetInfo: PropTypes.object, + infoLoading: PropTypes.bool, }; export default PrepareDatasetStep; diff --git a/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx new file mode 100644 index 000000000..41a6d2051 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/PreprocessingStep.jsx @@ -0,0 +1,61 @@ +import React, { useEffect } from "react"; +import PropTypes from "prop-types"; +import { useModels } from "../ModelsContext"; +import SessionConvertersRightBar from "./SessionConvertersRightBar"; +import AppliedConvertersView from "./AppliedConvertersView"; + +/** + * Optional preprocessing step of the session wizard: pick converters to fit + * on training data only (see the backend's PreprocessingJob), and configure + * each one's scope. A converter's scope may include the not-yet-materialized + * output group of an earlier converter in the same list, which is how + * chaining works before any real fit exists. + * + * Styled like the notebook module's own converter picker: the catalog + * (search + category list/grid) lives in the right bar + * (SessionConvertersRightBar), the already-added converters are shown as + * cards in the main content area (AppliedConvertersView). + */ +function PreprocessingStep({ + newExp, + setNewExp, + setNextEnabled, + dataset, + datasetTypes, +}) { + const { setSessionRightContent } = useModels(); + + useEffect(() => { + setNextEnabled(true); + }, [setNextEnabled]); + + useEffect(() => { + setSessionRightContent( + , + ); + return () => setSessionRightContent(null); + }, [newExp, setNewExp, dataset, datasetTypes]); + + return ( + + ); +} + +PreprocessingStep.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + dataset: PropTypes.object, + datasetTypes: PropTypes.object, +}; + +export default PreprocessingStep; diff --git a/DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx b/DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx new file mode 100644 index 000000000..fc979eb08 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/PreprocessingStep.test.jsx @@ -0,0 +1,50 @@ +import React from "react"; +import { screen } from "@testing-library/react"; +import { renderWithProviders } from "../../../test-utils/renderWithProviders"; +import { getComponents } from "../../../api/component"; + +// react-markdown ships ESM only and Create React App's jest does not transform +// node_modules, so importing PreprocessingStep (which transitively imports the +// converter parameter form, all the way down to FormTooltip) pulls in a parse +// error before a single assertion runs. Same stub used elsewhere in this repo +// (e.g. FormSchemaRenderFields.test.jsx) for the same reason. +jest.mock("react-markdown", () => ({ + __esModule: true, + default: ({ children }) => {children}, +})); + +jest.mock("../../../api/component", () => ({ + getComponents: jest.fn(), +})); + +jest.mock("../ModelsContext", () => ({ + useModels: () => ({ setSessionRightContent: () => {} }), +})); + +jest.mock("../../notebooks/context/ExplorersAndConvertersContext", () => ({ + useExplorersAndConverters: () => ({ setPendingDropTool: () => {} }), +})); + +import PreprocessingStep from "./PreprocessingStep"; + +describe("PreprocessingStep", () => { + beforeEach(() => { + getComponents.mockResolvedValue([]); + }); + + it("shows a message when no converter has been added yet", async () => { + renderWithProviders( + {}} + setNextEnabled={() => {}} + datasetInfo={{ column_names: [] }} + datasetTypes={{}} + />, + ); + + expect( + await screen.findByText("No converter was added."), + ).toBeInTheDocument(); + }); +}); diff --git a/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx new file mode 100644 index 000000000..7d1976e0a --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/ScopeStepSessionConverter.jsx @@ -0,0 +1,108 @@ +import { useMemo, useState } from "react"; +import PropTypes from "prop-types"; +import { Box, Typography } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import { useTranslation } from "react-i18next"; +import ColumnSelector from "../../notebooks/ColumnSelector"; +import FormSchemaButtonGroup from "../../shared/FormSchemaButtonGroup"; +import { buildColumnKeysAndTypes, keyToRef } from "./sessionColumnRefs"; + +/** + * Scope step for a converter being added to a session's preprocessing + * sequence: column selection only (there's no row-level scope — the + * train/test split doesn't exist yet at config time, and a session + * converter never sees rows outside its own fold's training partition at + * fit time regardless of what's picked here). + * + * The scope offered isn't just the dataset's raw columns: it's every raw + * column plus one synthetic key per output group of every converter + * already configured *before* this one in the sequence (chaining). The + * shared `ColumnSelector` only knows a plain `{name: {type, dtype}}` map, + * so groups are represented as synthetic keys built by `sessionColumnRefs` + * and translated back into real ColumnRef scope entries once the user + * picks a selection. + */ +export default function ScopeStepSessionConverter({ + tool, + datasetTypes, + preprocessing, + filePath, + scope, + setScope, + nextStep, +}) { + const theme = useTheme(); + const { t } = useTranslation(["common", "datasets", "models"]); + const [isColumnSelectionValid, setIsColumnSelectionValid] = useState(false); + + const allowedTypes = tool?.metadata?.allowed_types || []; + const allowedDtypes = tool?.metadata?.allowed_dtypes || []; + const nonAllowedDtypes = tool?.metadata?.non_allowed_dtypes || []; + const inputCardinality = tool?.metadata?.input_cardinality || {}; + + const { columnTypes: columnTypesForSelector } = useMemo( + () => buildColumnKeysAndTypes({ datasetTypes, preprocessing }), + [datasetTypes, preprocessing], + ); + + const handleSelectionChange = (selected) => { + setScope(selected.map((col) => keyToRef(col.columnName))); + }; + + const hasParams = Object.values(tool.schema.properties).length > 0; + + return ( + + + + {t("datasets:label.selectScopeDescriptionColumns")} + + + + + + + + + + ); +} + +ScopeStepSessionConverter.propTypes = { + tool: PropTypes.object.isRequired, + datasetTypes: PropTypes.object.isRequired, + preprocessing: PropTypes.array, + filePath: PropTypes.string, + scope: PropTypes.array.isRequired, + setScope: PropTypes.func.isRequired, + nextStep: PropTypes.func.isRequired, +}; diff --git a/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx new file mode 100644 index 000000000..1bf182567 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/SelectColumnsStep.jsx @@ -0,0 +1,368 @@ +import React, { useEffect, useLayoutEffect, useState } from "react"; +import PropTypes from "prop-types"; + +import { + Grid, + CircularProgress, + Box, + Alert, + AlertTitle, + Chip, +} from "@mui/material"; +import DivideDatasetColumns from "./DivideDatasetColumns"; +import { getComponents as getComponentsRequest } from "../../../api/component"; +import { validateColumns as validateColumnsRequest } from "../../../api/modelSession"; +import { useSnackbar } from "notistack"; +import { getColorByColumnType } from "../../../utils"; +import { useTranslation, Trans } from "react-i18next"; +import { + isGroupKey, + refToKey, + keyToRef, + groupKey, + buildColumnKeysAndTypes, +} from "./sessionColumnRefs"; + +/** + * Step of the session wizard: pick input/output columns. A column can be a + * raw dataset column, or (when preprocessing is configured) the whole output + * group of one of the session's converter steps, represented here as a + * synthetic key (e.g. "__group__0") so the underlying Autocomplete only + * ever deals with plain strings — the same shape a raw column name has. + */ +function SelectColumnsStep({ + newExp, + setNewExp, + setNextEnabled, + dataset, + datasetInfo, + datasetTypes, +}) { + const { enqueueSnackbar } = useSnackbar(); + const { t } = useTranslation(["experiments", "models", "common"]); + + const [taskRequirements, setTaskRequirements] = useState(null); + + const rawColumnNames = datasetInfo.column_names || []; + + const { + allKeys: inputOptionNames, + columnTypes: columnTypesForSelector, + optionLabels, + } = buildColumnKeysAndTypes({ + datasetTypes, + preprocessing: newExp.preprocessing, + }); + + const [inputSelection, setInputSelection] = useState(() => + newExp.input_column_refs && newExp.input_column_refs.length > 0 + ? newExp.input_column_refs.map(refToKey) + : newExp.input_columns || [], + ); + const [outputColumnNames, setOutputColumnNames] = useState( + newExp.output_columns || [], + ); + + const inputColumnRefs = inputSelection.map(keyToRef); + const rawInputNames = inputSelection.filter((key) => !isGroupKey(key)); + + const columnsReady = + inputSelection.length >= 1 && outputColumnNames.length >= 1; + const [columnsAreValid, setColumnsAreValid] = useState(false); + const [validationPending, setValidationPending] = useState(true); + + useEffect(() => { + // Auto-select sensible defaults the first time the dataset's columns load. + if (rawColumnNames.length === 0) return; + if ( + inputSelection.length === 0 && + (!newExp.input_columns || newExp.input_columns.length === 0) + ) { + const steps = newExp.preprocessing || []; + if (steps.length > 0) { + // With preprocessing configured, default to just the last step's + // output group(s) — the point of building a chain is usually to + // end up using its final result, not the raw columns it started + // from. Every raw column and every other group stays available to + // pick instead; this is only the starting default. + const lastIndex = steps.length - 1; + const slots = + steps[lastIndex].outputSlots?.length > 0 + ? steps[lastIndex].outputSlots + : [{ slot: null }]; + setInputSelection(slots.map(({ slot }) => groupKey(lastIndex, slot))); + } else { + setInputSelection( + rawColumnNames.length > 1 + ? rawColumnNames.slice(0, -1) + : [rawColumnNames[0]], + ); + } + } + if ( + outputColumnNames.length === 0 && + (!newExp.output_columns || newExp.output_columns.length === 0) + ) { + setOutputColumnNames([rawColumnNames[rawColumnNames.length - 1]]); + } + }, [rawColumnNames.join(",")]); + + const getTaskRequirements = async () => { + try { + const taskComponents = await getComponentsRequest({ + selectTypes: ["Task"], + }); + const currentTask = taskComponents.find( + (task) => task.name === newExp.task_name, + ); + if (currentTask) { + setTaskRequirements(currentTask); + } else { + setTaskRequirements({ + name: newExp.task_name, + metadata: { + inputs_types: [], + inputs_cardinality: "", + outputs_types: [], + outputs_cardinality: "", + }, + }); + } + } catch (error) { + enqueueSnackbar(t("experiments:error.errorFetchingTaskRequirements")); + console.error("Error fetching task requirements:", error); + } + }; + + const validateColumns = async () => { + try { + if ( + rawColumnNames.length === 0 || + inputSelection.length === 0 || + outputColumnNames.length === 0 + ) { + setColumnsAreValid(false); + return; + } + const hasGroupRef = inputSelection.some(isGroupKey); + // Keyed "{step}" for a step with one declared type (unslotted, the + // common case) or "{step}:{slot}" for one of several — matches the + // key scheme validate_columns looks declared_type up by backend-side + // (see model_sessions.py), which mirrors GroupColumnRef.slot. + const converterOutputTypes = {}; + (newExp.preprocessing || []).forEach((step, index) => { + const slots = + step.outputSlots?.length > 0 + ? step.outputSlots + : [{ slot: null, type: step.outputType || null }]; + slots.forEach(({ slot, type }) => { + const key = slot == null ? String(index) : `${index}:${slot}`; + converterOutputTypes[key] = type || null; + }); + }); + const validation = await validateColumnsRequest( + newExp.task_name, + dataset.id, + hasGroupRef ? rawInputNames : inputSelection, + outputColumnNames, + hasGroupRef ? inputColumnRefs : undefined, + hasGroupRef ? converterOutputTypes : undefined, + ); + setColumnsAreValid(validation.dataset_status === "valid"); + } catch (error) { + enqueueSnackbar(t("experiments:error.errorFetchingColumnsValidation")); + console.error("Error validating columns:", error); + setColumnsAreValid(false); + } finally { + setValidationPending(false); + } + }; + + useLayoutEffect(() => { + if (!columnsReady) { + setColumnsAreValid(false); + setValidationPending(false); + return; + } + if (rawColumnNames.length > 0) { + setValidationPending(true); + validateColumns(); + } + }, [columnsReady, inputSelection.join(","), outputColumnNames.join(",")]); + + useEffect(() => { + if (columnsAreValid && columnsReady) { + setNewExp({ + ...newExp, + input_columns: rawInputNames, + output_columns: outputColumnNames, + input_column_refs: inputColumnRefs, + }); + setNextEnabled(true); + } else { + setNextEnabled(false); + } + }, [ + columnsAreValid, + columnsReady, + inputSelection.join(","), + outputColumnNames.join(","), + ]); + + useEffect(() => { + getTaskRequirements(); + }, []); + + const columnGroupsOf = (side) => { + const metadata = taskRequirements?.metadata ?? {}; + if (Array.isArray(metadata[side]) && metadata[side].length > 0) { + return metadata[side]; + } + const cardinality = metadata[`${side}_cardinality`]; + return [ + { + types: metadata[`${side}_types`] ?? [], + min: cardinality === "n" ? 0 : cardinality, + max: cardinality, + }, + ]; + }; + + const describeCardinality = ({ min, max }) => { + if (max === "n") { + return min ? t("experiments:label.cardinalityAtLeast", { min }) : "n"; + } + if (min === max) { + return String(max); + } + return t("experiments:label.cardinalityBetween", { min, max }); + }; + + const renderTypesAsChips = (typesList) => { + if (!typesList || typesList.length === 0) { + return {t("common:any")}; + } + return ( + + {typesList.map((type, index) => ( + + + {index < typesList.length - 1 && ( + {t("common:or")} + )} + + ))} + + ); + }; + + return ( + + {taskRequirements && !validationPending && ( + + `${theme.palette[columnsAreValid ? "success" : "error"].main}40`, + border: (theme) => + `1px solid ${theme.palette[columnsAreValid ? "success" : "error"].main}`, + }} + data-tour="models-validation-alert" + > + + {t( + columnsAreValid + ? "experiments:label.columnsValidRequirements" + : "experiments:label.columnsInvalidRequirements", + { taskName: taskRequirements.display_name }, + )} + + + {["inputs", "outputs"].map((side) => + columnGroupsOf(side).map((group, index) => ( + + + + The columns must be of the types + {renderTypesAsChips(group.types)} + , and they should have a cardinality of + + {{ cardinality: describeCardinality(group) }}. + + + + + )), + )} + + + )} + + + + + + ); +} + +SelectColumnsStep.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + setNextEnabled: PropTypes.func.isRequired, + dataset: PropTypes.object.isRequired, + datasetInfo: PropTypes.object, + datasetTypes: PropTypes.object, +}; + +export default SelectColumnsStep; diff --git a/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx new file mode 100644 index 000000000..c5128d2ff --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/SessionConvertersRightBar.jsx @@ -0,0 +1,243 @@ +import { useEffect, useMemo, useState } from "react"; +import PropTypes from "prop-types"; +import { + Box, + Typography, + ToggleButtonGroup, + ToggleButton, +} from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import { ViewList, ViewModule } from "@mui/icons-material"; +import { useSnackbar } from "notistack"; +import { useTranslation } from "react-i18next"; +import SearchBar from "../../threeSectionLayout/SearchBar"; +import ToolList from "../../notebooks/tool/ToolList"; +import ToolGrid from "../../notebooks/tool/ToolGrid"; +import { getComponents } from "../../../api/component"; +import { evaluateColumnEligibility } from "../../../utils/columnEligibility"; +import FormSessionConverterSection from "./FormSessionConverterSection"; +import { buildColumnKeysAndTypes } from "./sessionColumnRefs"; + +/** + * Converters-only sidebar for the session wizard's preprocessing step, + * styled like the notebook module's own converters picker (same SearchBar, + * ToolList/ToolGrid, category accordion, list/grid toggle). Unlike the + * notebook flow, picking a converter here never fits anything or POSTs + * anywhere — see FormSessionConverterSection, which just appends the + * configured step to newExp.preprocessing. + */ +export default function SessionConvertersRightBar({ + newExp, + setNewExp, + dataset, + datasetTypes, +}) { + const theme = useTheme(); + const { t } = useTranslation(["models", "datasets", "common"]); + const { enqueueSnackbar } = useSnackbar(); + const [converters, setConverters] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [viewMode, setViewMode] = useState("list"); + + useEffect(() => { + let cancelled = false; + getComponents({ selectTypes: ["Converter"] }) + .then((data) => { + if (!cancelled) setConverters(Array.isArray(data) ? data : []); + }) + .catch((error) => { + console.error("Failed to fetch converters:", error); + enqueueSnackbar(t("datasets:error.fetchingConverters"), { + variant: "error", + }); + }); + return () => { + cancelled = true; + }; + }, [t]); + + // A new converter can be scoped on any raw dataset column OR the output + // group (any declared slot) of any converter already configured — it + // always lands at the end of the sequence, so every existing step is + // "before" it and fair game to chain off. Gating this on raw columns + // alone wrongly blocked, e.g., PCA on a text-only dataset (no raw + // Integer/Float columns) even after adding Bag of Words, whose Integer + // output group PCA could legitimately scope on. + const { columnTypes: allColumnTypes } = useMemo( + () => + buildColumnKeysAndTypes({ + datasetTypes, + preprocessing: newExp.preprocessing, + }), + [datasetTypes, newExp.preprocessing], + ); + + const datasetColumns = useMemo( + () => + Object.entries(allColumnTypes || {}).map( + ([columnName, typeInfo], idx) => ({ + id: idx, + columnName, + valueType: typeInfo.type || t("common:unknown"), + dataType: typeInfo.dtype || t("common:unknown"), + order: idx, + }), + ), + [allColumnTypes, t], + ); + + const validateConverter = (converter) => { + if (!datasetColumns.length) return { disabled: false, tooltip: "" }; + + const { validColumns, shortfall, restrictions, restricted } = + evaluateColumnEligibility(converter?.metadata, datasetColumns, { + unknownLabel: t("common:unknown"), + }); + + let disabled = shortfall !== null; + let tooltip = + converter.description || converter.metadata?.short_description || ""; + + if (shortfall !== null) { + const key = + shortfall.kind === "exact" + ? "datasets:error.requiresExactColumns" + : "datasets:error.requiresMinColumns"; + tooltip += `\n\n${t(key, { + required: shortfall.required, + available: shortfall.available, + count: shortfall.required, + })}`; + } + + if (validColumns.length === 0 && restricted && restrictions.length > 0) { + disabled = true; + tooltip += `\n\n${t("datasets:error.noValidColumnsWithDtypesMentioned", { + dtypes: restrictions.join(", "), + })}`; + } + + return { disabled, tooltip, validColumns }; + }; + + const validatedConverters = useMemo( + () => + converters.map((converter) => { + const validation = validateConverter(converter); + return { ...converter, ...validation }; + }), + [converters, datasetColumns, t], + ); + + const filteredConverters = useMemo(() => { + const query = searchQuery.trim().toLowerCase(); + if (!query) return validatedConverters; + const tokens = query.split(/\s+/).filter(Boolean); + return validatedConverters.filter((item) => { + const displayName = (item.display_name || item.name || "").toLowerCase(); + return tokens.every((token) => displayName.includes(token)); + }); + }, [searchQuery, validatedConverters]); + + // Stable identity across re-renders (e.g. every keystroke in the search + // box) so ConfigureToolModal never remounts FormSessionConverterSection + // — and loses in-progress scope/parameter input — while it's open. + const SessionFormSection = useMemo(() => { + function Wrapped(sectionProps) { + return ( + + ); + } + return Wrapped; + }, [newExp, setNewExp, datasetTypes, dataset]); + + return ( + + + setSearchQuery(e.target.value)} + onClear={() => setSearchQuery("")} + placeholder={t("datasets:label.searchConverters")} + /> + + + + {t("datasets:label.viewMode")} + + newMode && setViewMode(newMode)} + size="small" + > + + + + + + + + + {(() => { + const ListComponent = viewMode === "list" ? ToolList : ToolGrid; + const containerSx = + viewMode === "list" + ? { + flex: 1, + overflowY: "auto", + overflowX: "hidden", + p: 4, + minWidth: 0, + } + : { flex: 1, overflow: "auto", p: 4 }; + return ( + + + + ); + })()} + + ); +} + +SessionConvertersRightBar.propTypes = { + newExp: PropTypes.object.isRequired, + setNewExp: PropTypes.func.isRequired, + dataset: PropTypes.object, + datasetTypes: PropTypes.object, +}; diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js new file mode 100644 index 000000000..0c505ac1f --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.js @@ -0,0 +1,230 @@ +/** + * Synthetic-key representation of a session's ColumnRef (see the backend's + * DashAI.back.preprocessing.column_ref module). A raw dataset column is + * just its own name; a converter's not-yet-materialized output group is + * represented as a synthetic string key, so every UI piece that already + * knows how to work with a flat `{name: string}` column list (Autocomplete + * options, MaterialReactTable rows, columnTypes maps) can represent a group + * without knowing groups exist at all. + * + * A step's output isn't always one homogeneous type — e.g. SimpleImputer + * with "most_frequent"/"constant" just preserves each scope column's own + * type, so a scope mixing a categorical and a numeric column produces both + * kinds of columns. `slot` (a DashAI type's display_name(), e.g. + * "Categorical") picks out just the columns of one declared type from a + * step's output, mirroring the backend's GroupColumnRef.slot / + * SessionPreprocessor.resolved_slots. A step with only one declared type — + * the common case, and everything before slots existed — has no slot + * (`slot: null`), which means "the whole group," unchanged from before. + */ + +const GROUP_KEY_PREFIX = "__group__"; +const SLOT_SEPARATOR = "__slot__"; + +export const groupKey = (step, slot = null) => + slot == null + ? `${GROUP_KEY_PREFIX}${step}` + : `${GROUP_KEY_PREFIX}${step}${SLOT_SEPARATOR}${slot}`; + +export const isGroupKey = (key) => + typeof key === "string" && key.startsWith(GROUP_KEY_PREFIX); + +export const stepFromGroupKey = (key) => { + const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); + const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); + const stepPart = + separatorIndex === -1 + ? withoutPrefix + : withoutPrefix.slice(0, separatorIndex); + return Number(stepPart); +}; + +export const slotFromGroupKey = (key) => { + const withoutPrefix = key.slice(GROUP_KEY_PREFIX.length); + const separatorIndex = withoutPrefix.indexOf(SLOT_SEPARATOR); + return separatorIndex === -1 + ? null + : withoutPrefix.slice(separatorIndex + SLOT_SEPARATOR.length); +}; + +/** ColumnRef -> synthetic key */ +export const refToKey = (ref) => + ref.kind === "raw" ? ref.name : groupKey(ref.step, ref.slot ?? null); + +/** synthetic key -> ColumnRef */ +export const keyToRef = (key) => { + if (!isGroupKey(key)) return { kind: "raw", name: key }; + const step = stepFromGroupKey(key); + const slot = slotFromGroupKey(key); + return slot == null ? { kind: "group", step } : { kind: "group", step, slot }; +}; + +// A step's declared output, normalized to a list of slots — even a +// homogeneous step (the common case) is one "slot" with slot: null, so +// every caller iterates the same shape regardless of how many there are. +// Falls back defensively for a step that predates outputSlots. +function stepOutputSlots(step) { + if (Array.isArray(step?.outputSlots) && step.outputSlots.length > 0) { + return step.outputSlots; + } + return [ + { + slot: null, + type: step?.outputType ?? null, + dtype: step?.outputDtype ?? null, + }, + ]; +} + +/** + * Every column key a session's preprocessing sequence can be scoped over, + * up to (and not including) `uptoStep`: every raw dataset column, plus one + * group key per declared slot of every converter step before it (usually + * one key per step; more than one only when that step's scope mixed + * column types). Passing no `uptoStep` includes every configured step + * (used once a sequence is final and being displayed, e.g. in + * SelectColumnsStep, where every step is already "before" the + * column-selection step that comes after all of them). + */ +export function buildColumnKeysAndTypes({ + datasetTypes, + preprocessing, + uptoStep, +}) { + const steps = preprocessing || []; + const limit = uptoStep === undefined ? steps.length : uptoStep; + + const columnTypes = { ...datasetTypes }; + const optionLabels = {}; + const allKeys = Object.keys(datasetTypes || {}); + + for (let index = 0; index < limit; index += 1) { + const step = steps[index]; + stepOutputSlots(step).forEach(({ slot, type, dtype }) => { + const key = groupKey(index, slot); + columnTypes[key] = { type: type || null, dtype: dtype || null }; + optionLabels[key] = slot + ? `${step.converter}: output (${slot})` + : `${step.converter}: output`; + allKeys.push(key); + }); + } + + return { allKeys, columnTypes, optionLabels }; +} + +// SimpleImputer only preserves its input column's type for the +// "most_frequent"/"constant" strategies (no arithmetic performed); "mean"/ +// "median" always produce something numeric, but which exact numeric type +// depends on a statistic computed from real data, so there's nothing to +// resolve ahead of a fit — tool.metadata.output_type ("Float") already +// covers that case correctly, since every allowed_types list in this +// codebase that includes Integer also includes Float (they're always +// interchangeable for compatibility checks). +function simpleImputerPreservesType(toolName, params) { + if (toolName !== "SimpleImputer") return false; + const strategy = params?.strategy; + return strategy === "most_frequent" || strategy === "constant"; +} + +/** + * The output slot(s) to record for a converter step once its scope is + * known — always an array of {slot, type, dtype}, even when there's just + * one (slot: null, meaning "the whole group", exactly like before slots + * existed). + * + * Most converters' declared output_type/output_dtype (metadata computed + * backend-side from a bare, never-fitted instance) is the best available + * answer regardless of scope, and stays a single slot. But a converter + * that only keeps or drops whole columns unchanged (backend + * PRESERVES_INPUT_TYPE, e.g. feature selection, VarianceThreshold, or + * SimpleImputer with "most_frequent"/"constant") has an output type + * that's actually already knowable — it's just whatever the chosen + * scope's own column type already is. When that scope mixes distinct + * types, this splits into one slot per distinct type instead of falling + * back to a single guess, matching exactly what SessionPreprocessor. + * _classify_by_type computes for real once the session is created. + */ +export function resolveDeclaredOutputSlots({ + tool, + params, + scope, + datasetTypes, + preprocessing, +}) { + const fallback = [ + { + slot: null, + type: tool?.metadata?.output_type || null, + dtype: tool?.metadata?.output_dtype || null, + }, + ]; + + const preservesInputType = + Boolean(tool?.metadata?.preserves_input_type) || + simpleImputerPreservesType(tool?.name, params); + + if (!preservesInputType || !scope?.length) return fallback; + + const { columnTypes } = buildColumnKeysAndTypes({ + datasetTypes, + preprocessing, + }); + const scopeEntries = scope.map((ref) => columnTypes[refToKey(ref)]); + if (scopeEntries.some((entry) => !entry?.type)) return fallback; + + const distinctTypes = [...new Set(scopeEntries.map((entry) => entry.type))]; + + const dtypeFor = (type) => { + const dtypes = [ + ...new Set( + scopeEntries + .filter((entry) => entry.type === type) + .map((entry) => entry.dtype) + .filter(Boolean), + ), + ]; + return dtypes.length === 1 ? dtypes[0] : null; + }; + + if (distinctTypes.length === 1) { + return [ + { slot: null, type: distinctTypes[0], dtype: dtypeFor(distinctTypes[0]) }, + ]; + } + + return distinctTypes.map((type) => ({ + slot: type, + type, + dtype: dtypeFor(type), + })); +} + +/** + * Every RAW dataset column name needed to compute a list of ColumnRef, + * walking group refs back to their step's own scope recursively (a group + * ref's step may itself reference an earlier group, chained arbitrarily + * deep). This is what a caller must actually supply values for — e.g. + * manual prediction, where the backend only ever accepts real dataset + * columns as input (see BaseTask.process_manual_input), never a + * converter's resolved output name like "pca_1": it runs the raw values + * through the session's persisted preprocessor itself before predicting. + */ +export function rawColumnsNeededFor(refs, steps) { + const needed = []; + const seen = new Set(); + const visit = (ref) => { + if (ref.kind === "raw") { + if (!seen.has(ref.name)) { + seen.add(ref.name); + needed.push(ref.name); + } + return; + } + const step = (steps || [])[ref.step]; + if (!step) return; + (step.scope || []).forEach(visit); + }; + (refs || []).forEach(visit); + return needed; +} diff --git a/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js new file mode 100644 index 000000000..234524de1 --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/sessionColumnRefs.test.js @@ -0,0 +1,328 @@ +import { + groupKey, + isGroupKey, + stepFromGroupKey, + slotFromGroupKey, + refToKey, + keyToRef, + buildColumnKeysAndTypes, + resolveDeclaredOutputSlots, + rawColumnsNeededFor, +} from "./sessionColumnRefs"; + +describe("sessionColumnRefs", () => { + it("round-trips a raw ColumnRef through a key", () => { + const ref = { kind: "raw", name: "age" }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + + it("round-trips a group ColumnRef through a key", () => { + const ref = { kind: "group", step: 2 }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + + it("round-trips a slotted group ColumnRef through a key", () => { + const ref = { kind: "group", step: 0, slot: "Categorical" }; + expect(keyToRef(refToKey(ref))).toEqual(ref); + }); + + it("recognizes group keys and extracts their step", () => { + expect(isGroupKey(groupKey(3))).toBe(true); + expect(isGroupKey("age")).toBe(false); + expect(stepFromGroupKey(groupKey(3))).toBe(3); + }); + + it("extracts both step and slot from a slotted group key", () => { + const key = groupKey(2, "Categorical"); + expect(stepFromGroupKey(key)).toBe(2); + expect(slotFromGroupKey(key)).toBe("Categorical"); + expect(slotFromGroupKey(groupKey(2))).toBeNull(); + }); + + it("builds keys/types/labels for raw columns plus every prior step", () => { + const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ + datasetTypes: { age: { type: "Integer" }, text: { type: "Text" } }, + preprocessing: [ + { + converter: "BagOfWordsConverter", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + { + converter: "Binarizer", + outputSlots: [{ slot: null, type: "Integer", dtype: null }], + }, + ], + uptoStep: 1, + }); + + expect(allKeys).toEqual(["age", "text", "__group__0"]); + expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); + expect(optionLabels.__group__0).toBe("BagOfWordsConverter: output"); + // step 1 (Binarizer) is excluded: uptoStep=1 only includes steps before it + expect(allKeys).not.toContain(groupKey(1)); + }); + + it("includes every step when uptoStep is omitted", () => { + const { allKeys } = buildColumnKeysAndTypes({ + datasetTypes: { age: { type: "Integer" } }, + preprocessing: [ + { + converter: "BagOfWordsConverter", + outputSlots: [{ slot: null, type: "Integer" }], + }, + { + converter: "Binarizer", + outputSlots: [{ slot: null, type: "Integer" }], + }, + ], + }); + + expect(allKeys).toEqual(["age", "__group__0", "__group__1"]); + }); + + it("offers one key per declared slot for a step with a heterogeneous scope", () => { + const { allKeys, columnTypes, optionLabels } = buildColumnKeysAndTypes({ + datasetTypes: {}, + preprocessing: [ + { + converter: "SimpleImputer", + outputSlots: [ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Categorical", type: "Categorical", dtype: null }, + ], + }, + ], + }); + + const integerKey = groupKey(0, "Integer"); + const categoricalKey = groupKey(0, "Categorical"); + expect(allKeys).toEqual([integerKey, categoricalKey]); + expect(columnTypes[integerKey]).toEqual({ + type: "Integer", + dtype: "int64", + }); + expect(columnTypes[categoricalKey]).toEqual({ + type: "Categorical", + dtype: null, + }); + expect(optionLabels[integerKey]).toBe("SimpleImputer: output (Integer)"); + expect(optionLabels[categoricalKey]).toBe( + "SimpleImputer: output (Categorical)", + ); + }); + + it("falls back to a single unslotted key for a step predating outputSlots", () => { + const { allKeys, columnTypes } = buildColumnKeysAndTypes({ + datasetTypes: {}, + preprocessing: [ + { converter: "Binarizer", outputType: "Integer", outputDtype: "int64" }, + ], + }); + + expect(allKeys).toEqual(["__group__0"]); + expect(columnTypes.__group__0).toEqual({ type: "Integer", dtype: "int64" }); + }); + + describe("resolveDeclaredOutputSlots", () => { + const datasetTypes = { + age: { type: "Integer", dtype: "int64" }, + score: { type: "Float", dtype: "float64" }, + name: { type: "Text", dtype: "string" }, + }; + + it("falls back to the converter's declared metadata by default", () => { + const tool = { + name: "Binarizer", + metadata: { output_type: "Integer", output_dtype: "int64" }, + }; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "raw", name: "age" }], + datasetTypes, + preprocessing: [], + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + + it("uses the scope's real type when the converter preserves input type", () => { + const tool = { + name: "SelectKBest", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "raw", name: "age" }], + datasetTypes, + preprocessing: [], + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + + it("splits into one slot per distinct type when a preserving converter's scope mixes types", () => { + const tool = { + name: "VarianceThreshold", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "name" }, + ], + datasetTypes, + preprocessing: [], + }); + expect(result).toEqual( + expect.arrayContaining([ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Text", type: "Text", dtype: "string" }, + ]), + ); + expect(result).toHaveLength(2); + }); + + it("treats SimpleImputer as type-preserving only for most_frequent/constant", () => { + const tool = { + name: "SimpleImputer", + metadata: { output_type: "Float", output_dtype: null }, + }; + const scope = [{ kind: "raw", name: "name" }]; + + const mostFrequent = resolveDeclaredOutputSlots({ + tool, + params: { strategy: "most_frequent" }, + scope, + datasetTypes, + preprocessing: [], + }); + expect(mostFrequent).toEqual([ + { slot: null, type: "Text", dtype: "string" }, + ]); + + const mean = resolveDeclaredOutputSlots({ + tool, + params: { strategy: "mean" }, + scope, + datasetTypes, + preprocessing: [], + }); + expect(mean).toEqual([{ slot: null, type: "Float", dtype: null }]); + }); + + it("resolves a preserving converter's scope through a chained group ref", () => { + const tool = { + name: "SelectKBest", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const preprocessing = [ + { + converter: "BagOfWordsConverter", + outputSlots: [{ slot: null, type: "Integer", dtype: "int64" }], + }, + ]; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "group", step: 0 }], + datasetTypes, + preprocessing, + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + + it("resolves a preserving converter's scope through one specific slot of a chained group ref", () => { + const tool = { + name: "SelectKBest", + metadata: { output_type: "Float", preserves_input_type: true }, + }; + const preprocessing = [ + { + converter: "SimpleImputer", + outputSlots: [ + { slot: "Integer", type: "Integer", dtype: "int64" }, + { slot: "Categorical", type: "Categorical", dtype: null }, + ], + }, + ]; + const result = resolveDeclaredOutputSlots({ + tool, + params: {}, + scope: [{ kind: "group", step: 0, slot: "Integer" }], + datasetTypes, + preprocessing, + }); + expect(result).toEqual([{ slot: null, type: "Integer", dtype: "int64" }]); + }); + }); + + describe("rawColumnsNeededFor", () => { + it("returns raw ref names as-is", () => { + const refs = [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "score" }, + ]; + expect(rawColumnsNeededFor(refs, [])).toEqual(["age", "score"]); + }); + + it("resolves a group ref to its step's own raw scope", () => { + const refs = [{ kind: "group", step: 0 }]; + const steps = [ + { + converter: "BagOfWordsConverter", + scope: [{ kind: "raw", name: "text" }], + }, + ]; + expect(rawColumnsNeededFor(refs, steps)).toEqual(["text"]); + }); + + it("resolves a chained group ref recursively through an earlier step", () => { + const refs = [{ kind: "group", step: 1 }]; + const steps = [ + { + converter: "BagOfWordsConverter", + scope: [{ kind: "raw", name: "text" }], + }, + { converter: "PCA", scope: [{ kind: "group", step: 0 }] }, + ]; + expect(rawColumnsNeededFor(refs, steps)).toEqual(["text"]); + }); + + it("ignores slot when walking a group ref back to raw columns", () => { + const refs = [{ kind: "group", step: 0, slot: "Integer" }]; + const steps = [ + { + converter: "SimpleImputer", + scope: [ + { kind: "raw", name: "age" }, + { kind: "raw", name: "city" }, + ], + }, + ]; + // The slot only narrows which OUTPUT columns you get; fitting the + // step still needs every raw column in its scope, regardless. + expect(rawColumnsNeededFor(refs, steps)).toEqual(["age", "city"]); + }); + + it("de-duplicates a raw column needed by more than one ref", () => { + const refs = [ + { kind: "raw", name: "age" }, + { kind: "group", step: 0 }, + ]; + const steps = [ + { converter: "Doubler", scope: [{ kind: "raw", name: "age" }] }, + ]; + expect(rawColumnsNeededFor(refs, steps)).toEqual(["age"]); + }); + + it("skips a group ref pointing at a step that doesn't exist", () => { + const refs = [ + { kind: "raw", name: "age" }, + { kind: "group", step: 5 }, + ]; + expect(rawColumnsNeededFor(refs, [])).toEqual(["age"]); + }); + }); +}); diff --git a/DashAI/front/src/pages/models/ModelsContent.jsx b/DashAI/front/src/pages/models/ModelsContent.jsx index b51d1d993..80fda32a9 100644 --- a/DashAI/front/src/pages/models/ModelsContent.jsx +++ b/DashAI/front/src/pages/models/ModelsContent.jsx @@ -15,6 +15,7 @@ import { useThreePanelLayout } from "../../hooks/useThreePanelsLayout"; import { ThreePanelLayoutContext } from "../../components/threeSectionLayout/panels/ThreePanelLayoutContext"; import { useModels } from "../../components/models/ModelsContext"; import { getDatasetInfo } from "../../api/datasets"; +import { ExplorersAndConvertersProvider } from "../../components/notebooks/context/ExplorersAndConvertersContext"; export default function ModelsContent() { const location = useLocation(); @@ -134,40 +135,57 @@ export default function ModelsContent() { }, [selectedSessionId, sessions]); return ( - - - - {/* Left Panel */} - - - - {selectedSessionId ? ( - - - - - - - - - ) : ( - <> - - - + // ExplorersAndConvertersProvider is shared with the notebooks module: + // the session wizard's preprocessing step reuses the notebook's + // converter sidebar (ToolList/ToolGrid), which reads this context — + // wrapping the whole page here, so the center panel and the right bar + // share one instance of it, the same as the notebooks module does. + + + + + {/* Left Panel */} + + + + {selectedSessionId ? ( + + + + + + + + + ) : ( + <> + + + - {/* Right Panel */} - - - - - )} - - - + {/* Right Panel */} + + + + + )} + + + + ); } diff --git a/DashAI/front/src/types/modelSession.ts b/DashAI/front/src/types/modelSession.ts index d674dcaa8..0048d86b2 100644 --- a/DashAI/front/src/types/modelSession.ts +++ b/DashAI/front/src/types/modelSession.ts @@ -1,6 +1,26 @@ import type { IDataset } from "./dataset"; import type { IRun } from "./run"; +export interface IRawColumnRef { + kind: "raw"; + name: string; +} + +export interface IGroupColumnRef { + kind: "group"; + step: number; +} + +export type IColumnRef = IRawColumnRef | IGroupColumnRef; + +export interface IConverterStep { + converter: string; + params: Record; + scope: IColumnRef[]; +} + +export type IPreprocessingStatus = "ready" | "pending" | "failed"; + export interface IModelSession { id: string; dataset: IDataset; @@ -12,4 +32,9 @@ export interface IModelSession { created: Date; last_modified: Date; runs: IRun[]; + preprocessing?: IConverterStep[]; + input_column_refs?: IColumnRef[]; + preprocessing_status?: IPreprocessingStatus; + preprocessing_error?: string | null; + preprocessing_job_id?: string | null; } diff --git a/DashAI/front/src/utils/i18n/locales/de/datasets.json b/DashAI/front/src/utils/i18n/locales/de/datasets.json index 6005aa137..aae529bdc 100644 --- a/DashAI/front/src/utils/i18n/locales/de/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/de/datasets.json @@ -52,6 +52,7 @@ "uploadFile": "Datei hochladen" }, "error": { + "cannotSaveEmptyDataset": "Dataset kann nicht gespeichert werden: Alle Spalten wurden entfernt.", "converterFailed": "Beim Verarbeiten des Konverters ist ein Fehler aufgetreten.", "converterFailedWithInfo": "Fehler beim Verarbeiten des Konverters: {{error}}", "createConverterError": "Konverter konnte nicht erstellt werden", @@ -68,68 +69,52 @@ "failedToCreateDataset": "Der Datensatz konnte nicht erstellt werden. Bitte pruefe deine Datei und Konfiguration und versuche es erneut.", "failedToCreateDatasetFromNotebook": "Datensatz konnte nicht aus Notizbuch erstellt werden", "failedToCreateExplorer": "Explorer konnte nicht erstellt werden", + "failedToCreateFolder": "Ordner konnte nicht erstellt werden", "failedToDeleteDataset": "Datensatz konnte nicht gelöscht werden", "failedToDeleteDatasets": "Die ausgewählten Datensätze konnten nicht gelöscht werden", - "failedToCreateFolder": "Ordner konnte nicht erstellt werden", - "failedToUpdateFolder": "Ordner konnte nicht aktualisiert werden", "failedToDeleteFolder": "Ordner konnte nicht gelöscht werden", - "folderNameExists": "Ein Ordner mit diesem Namen existiert bereits", - "failedToMoveDataset": "Datensatz konnte nicht verschoben werden", "failedToDeleteNotebook": "Notizbuch konnte nicht gelöscht werden", "failedToDeleteNotebooks": "Die ausgewählten Notizbücher konnten nicht gelöscht werden", "failedToFetchNotebooks": "Notizbücher konnten nicht abgerufen werden", "failedToLoadDatasetInfo": "Datensatzinformationen konnten nicht abgerufen werden", + "failedToMoveDataset": "Datensatz konnte nicht verschoben werden", + "failedToResetExplorerResults": "Ursprüngliches Diagramm konnte nicht wiederhergestellt werden", "failedToUpdateDataset": "Datensatz konnte nicht aktualisiert werden", "failedToUpdateExplorerResults": "Explorer-Ergebnisse konnten nicht aktualisiert werden", - "failedToResetExplorerResults": "Ursprüngliches Diagramm konnte nicht wiederhergestellt werden", + "failedToUpdateFolder": "Ordner konnte nicht aktualisiert werden", "failedToUpdateNotebook": "Notizbuch konnte nicht aktualisiert werden", + "fetchingConverters": "Fehler beim Abrufen der verfügbaren Converter.", "fetchingDataloaders": "Fehler beim Abrufen kompatibler Datenlader.", "fetchingDatasetColumns": "Fehler beim Abrufen der Datensatzspalten.", "fetchingExplorersConverters": "Explorer/Konverter konnten nicht abgerufen werden", "fileTypeNotAllowed": "Dateityp für den ausgewählten Datenlader nicht zulässig.", + "folderNameExists": "Ein Ordner mit diesem Namen existiert bereits", "loadingDatasetPreview": "Fehler beim Laden der Datensatzvorschau", "noDatasetDataAvailable": "Keine Datensatzdaten verfügbar", "noDatasetFileAvailable": "Keine Datensatzdatei verfügbar", - "notebookNameEmpty": "Notizbuchname darf nicht leer sein", "noValidColumnsForExplorer": "Keine gültigen Spalten für diesen Explorer verfügbar.", "noValidColumnsWithDtypesMentioned": "Dieser Datensatz hat keine Spalten mit den erforderlichen Typen ({{dtypes}}).", + "notebookNameEmpty": "Notizbuchname darf nicht leer sein", "processConverterError": "Konverter konnte nicht verarbeitet werden", "requiredFieldsMissing": "Pflichtfelder fehlen", "requiresExactColumns_one": "Erfordert genau {{required}} gültige Spalte, aber {{available}} verfügbar.", "requiresExactColumns_other": "Erfordert genau {{required}} gültige Spalten, aber {{available}} verfügbar.", "requiresMinColumns_one": "Erfordert mindestens {{required}} gültige Spalte, aber nur {{available}} verfügbar.", "requiresMinColumns_other": "Erfordert mindestens {{required}} gültige Spalten, aber nur {{available}} verfügbar.", - "zipContentsNotCompatible": "ZIP enthält keine mit dem ausgewählten Datenlader kompatiblen Dateien", - "cannotSaveEmptyDataset": "Dataset kann nicht gespeichert werden: Alle Spalten wurden entfernt." + "zipContentsNotCompatible": "ZIP enthält keine mit dem ausgewählten Datenlader kompatiblen Dateien" }, "label": { - "task": "Aufgabe", + "aboutTool": "Über {{toolName}}", "all": "alle", + "allTypeChangesValid": "Alle Typänderungen sind gültig und können sicher angewendet werden.", "allowedDataTypes": "Zulässige Datentypen: <1><0>", "allowedValueTypes": "Zulässige Wertetypen: <1><0>", - "excludedDataTypes": "Ausgeschlossene Datentypen: {{dtypes}}", - "allTypeChangesValid": "Alle Typänderungen sind gültig und können sicher angewendet werden.", "analysisTools": "Analysewerkzeuge", "appearance": "Erscheinungsbild", "appliedTransformations": "Angewendete Transformationen:", "associatedDataset": "Verknüpfter Datensatz", "atLeastTwoColorStopsRequired": "Keine Farbstopps definiert. Bitte fügen Sie mindestens zwei Farbstopps hinzu.", "availableDatasets": "Verfügbare Datensätze", - "confirmDeleteFolder": "Ordner löschen", - "confirmDeleteFolderContent": "Möchten Sie den Ordner \"{{name}}\" wirklich löschen? Datensätze darin werden in keinen Ordner verschoben.", - "noFolder": "Kein Ordner", - "newFolder": "Neuer Ordner", - "folderName": "Ordnername", - "confirmDeleteDataset": "Sind Sie sicher, dass Sie den Datensatz \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmDeleteDatasetLinkedWarning": "Alle mit diesem Datensatz verknüpften Notizbücher und Sitzungen werden ebenfalls gelöscht.", - "confirmBulkDeleteDatasets_one": "Sind Sie sicher, dass Sie den ausgewählten Datensatz löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteDatasets_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Datensätze löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "selectDatasetsToDelete": "Datensätze zum Löschen auswählen", - "confirmDeleteNotebook": "Sind Sie sicher, dass Sie das Notizbuch \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteNotebooks": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteNotebooks_one": "Sind Sie sicher, dass Sie das ausgewählte Notizbuch löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "confirmBulkDeleteNotebooks_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "selectNotebooksToDelete": "Notizbücher zum Löschen auswählen", "avg": "Durchschn.", "avgLength": "Durchschnittliche Länge", "avgWordCount": "Durchschnittliche Wortanzahl", @@ -154,6 +139,7 @@ "classTargetColumn": "Klassen-/Zielspalte", "clickToUpload": "Zum Hochladen klicken", "color": "Farbe", + "colorStops": "Farbstopps", "colorbarBorderColor": "Farbleisten-Rahmenfarbe", "colorbarBorderWidth": "Farbleisten-Rahmenbreite", "colorbarTickFontColor": "Farbleisten-Skalierungsschriftfarbe", @@ -161,7 +147,7 @@ "colors": "Farben", "colorscale": "Farbskala", "colorscaleMode": "Farbskalenmodus", - "colorStops": "Farbstopps", + "columnInsights": "Spalteneinblicke", "columnName": "Spaltenname", "columnTypesDistribution": "Spaltentypenverteilung", "configureAndUpload": "Konfigurieren & Hochladen", @@ -170,10 +156,20 @@ "configureParametersStep": "Schritt {{step}}: Parameter konfigurieren", "configureScope": "Bereich konfigurieren", "configureToolTitle": "{{toolType}} konfigurieren: {{toolName}}", - "columnInsights": "Spalteneinblicke", + "confirmBulkDeleteDatasets_one": "Sind Sie sicher, dass Sie den ausgewählten Datensatz löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteDatasets_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Datensätze löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteNotebooks": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteNotebooks_one": "Sind Sie sicher, dass Sie das ausgewählte Notizbuch löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmBulkDeleteNotebooks_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Notizbücher löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmDeleteDataset": "Sind Sie sicher, dass Sie den Datensatz \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmDeleteDatasetLinkedWarning": "Alle mit diesem Datensatz verknüpften Notizbücher und Sitzungen werden ebenfalls gelöscht.", + "confirmDeleteFolder": "Ordner löschen", + "confirmDeleteFolderContent": "Möchten Sie den Ordner \"{{name}}\" wirklich löschen? Datensätze darin werden in keinen Ordner verschoben.", + "confirmDeleteNotebook": "Sind Sie sicher, dass Sie das Notizbuch \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", "constantColumns": "Konstante Spalten", "convert": "Konvertieren", "converter": "Konverter", + "converterOutput": "Ausgabe", "correlation": "Korrelation", "correlationAnalysis": "Korrelationsanalyse", "correlations": "Korrelationen", @@ -181,43 +177,46 @@ "createNewNotebook": "Notizbuch", "createNewNotebookDescription": "Starten Sie eine neue Analysesitzung mit einem vorhandenen Datensatz.", "customArray": "Benutzerdefiniertes Array", - "dataloaderConfiguration": "Datenlader-Konfiguration", "dataQuality": "Datenqualität", "dataQualityScoreTooltip": "Der Datenqualitätswert wird auf Basis verschiedener Faktoren berechnet, einschließlich fehlender Werte, doppelter Zeilen und Datenkonsistenz. Ein höherer Wert zeigt bessere Datenqualität an.", "dataQualitySummary": "Datenqualitätszusammenfassung", + "dataType": "Datentyp", + "dataloaderConfiguration": "Datenlader-Konfiguration", "datasetDescription": "{{rows}} Zeilen, {{columns}} Spalten", "datasetLoading": "Datensatz wird geladen...", "datasetModule": "Datensatz-Modul", "datasetModuleSubtitle": "Laden Sie Ihre Datensätze hoch: Erkunden, analysieren und transformieren Sie Ihre Daten mit erweiterten explorativen Analysewerkzeugen. Erstellen Sie interaktive Notizbücher, generieren Sie Visualisierungen und wenden Sie Datentransformationen intuitiv an.", "datasetName": "Datensatzname", "datasetPreview": "Datensatzvorschau", - "aboutTool": "Über {{toolName}}", "datasetPreviewFor": "Notizbuch: {{name}} Vorschau", - "dataType": "Datentyp", + "decreasingColor": "Farbe für Rückgang", "deleteConverterConfirmation": "Sind Sie sicher, dass Sie den Konverter \"{{converter}}\" löschen möchten? Das Löschen dieses Konverters entfernt auch alle nachfolgenden Konverter und Explorer. Diese Aktion kann nicht rückgängig gemacht werden.", "deleteExplorerConfirmation": "Sind Sie sicher, dass Sie den Explorer \"{{explorer}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", "detailsForExplorer": "Details für Explorer: {{name}}", "dimensionIdx": "Dimension {{idx}}: {{label}}", - "dimensionsLabels": "Dimensionsbeschriftungen", "dimensionTitle": "Dimensionstitel", + "dimensionsLabels": "Dimensionsbeschriftungen", "distributionMetrics": "Verteilungsmetriken", "dragAndDropFileHere": "Datei hier ablegen", "dragToResize": "Ziehen zum Vergrößern und Anzeigen weiterer Inhalte", + "dropToolHere": "Hier ablegen zum Hinzufügen", "duplicatedRows": "Doppelte Zeilen", "editPlotLayout": "Plot-Layout bearbeiten", "endIndex": "Endindex", + "excludedDataTypes": "Ausgeschlossene Datentypen: {{dtypes}}", "explorationPath": "Explorationspfad", "explorationType": "Explorationstyp", "explore": "Erkunden", "explorer": "Explorer", + "exportCSV": "Als CSV herunterladen", "exportCardImage": "Karte als Bild", "exportChartImage": "Diagramm als Bild", - "exportCSV": "Als CSV herunterladen", "exportImage": "Herunterladen", "exportJSON": "Als JSON herunterladen", "exportMetrics": "Metriken als JSON exportieren", "fileSizeMB": "Dateigröße (MB)", "firstLastStopsAtExtremes": "Farbstopps müssen Positionen 0 am Anfang und 1 am Ende enthalten.", + "folderName": "Ordnername", "fontFamily": "Schriftfamilie", "foundDuplicateRows_one": "{{count}} doppelte Zeile im Datensatz gefunden", "foundDuplicateRows_other": "{{count}} doppelte Zeilen im Datensatz gefunden", @@ -228,17 +227,20 @@ "highCardinality": "Hohe Kardinalität", "highCardinalityDetected": "Hohe Kardinalität erkannt in: {{columns}}", "ifYourDatasetHaveSplits": "Wenn Ihr Datensatz Aufteilungen hat, laden Sie ihn als ZIP-Datei hoch", + "importFromHub": "Datei-Hub", + "importFromHubDescription": "Datensätze aus externen Quellen wie HuggingFace und OpenML durchsuchen und herunterladen.", + "increasingColor": "Farbe für Anstieg", "indices": "Indizes (kommagetrennt oder 'all' eingeben)", "inferenceRows": "Inferenzzeilen", "inferenceRowsDescription": "Anzahl der Zeilen für Vorschau/Typinferenz (Minimum 2).", "inferredConfiguration": "Typinferenz-Konfiguration", "insightConstantColumn": "Diese Spalte hat nur einen eindeutigen Wert. Sie liefert keine Informationen für die Analyse.", - "insightHighCardinality": "Diese kategorische Spalte hat mehr als 100 eindeutige Werte. Erwägen Sie Gruppierung oder Kodierung.", "insightEmptyMessage": "Keine Probleme erkannt. Spalteneinblicke erscheinen hier, wenn potenzielle Beobachtungen gefunden werden.", + "insightHighCardinality": "Diese kategorische Spalte hat mehr als 100 eindeutige Werte. Erwägen Sie Gruppierung oder Kodierung.", "insightHighNanRatio": "{{value}}% der Werte fehlen in dieser Spalte. Erwägen Sie Imputation oder Entfernung.", "insightLowUniqueness": "Niedrige Eindeutigkeitsrate ({{value}}%). Könnte eine kategorische Variable sein, die fälschlicherweise als Text klassifiziert wurde.", - "insightOutliers_one": "{{count}} potenzieller Ausreißerwert erkannt (IQR-Methode).", "insightOutliers_many": "{{count}} potenzielle Ausreißerwerte erkannt (IQR-Methode).", + "insightOutliers_one": "{{count}} potenzieller Ausreißerwert erkannt (IQR-Methode).", "insightOutliers_other": "{{count}} potenzielle Ausreißerwerte erkannt (IQR-Methode).", "insightPossibleId": "Alle Werte sind eindeutig. Diese Spalte könnte ein Bezeichner sein und für die Modellierung nicht nützlich.", "insightSkewed": "Rechtsschiefe Verteilung (Schiefe: {{value}}). Erwägen Sie eine Logarithmustransformation.", @@ -260,19 +262,15 @@ "legendYPosition": "Legenden-Y-Position", "lengthDistribution": "Längenverteilung", "lengthMetrics": "Längenmetriken", - "lowerBound": "Untergrenze", + "lineColor": "Linienfarbe", "lowUniquenessWarning": "Warnung: Diese Textspalte hat eine sehr niedrige Eindeutigkeitsrate. Dies könnte eine kategorische Variable sein, die fälschlicherweise als Text klassifiziert wurde.", + "lowerBound": "Untergrenze", "marginBottom": "Unterer Rand", "marginLeft": "Linker Rand", "marginRight": "Rechter Rand", - "margins": "Ränder", "marginTop": "Oberer Rand", + "margins": "Ränder", "markerColor": "Markierungsfarbe", - "lineColor": "Linienfarbe", - "increasingColor": "Farbe für Anstieg", - "decreasingColor": "Farbe für Rückgang", - "totalsColor": "Farbe für Summen", - "transparent": "Transparent", "max": "Max", "maxLength": "Maximale Länge", "mean": "Mittelwert", @@ -286,29 +284,28 @@ "missingValuesDetected": "Fehlende Werte in Spalten erkannt: {{columns}}", "missingValuesOverview": "Übersicht fehlender Werte", "mostFrequent": "Am häufigsten", - "nameYourNotebook": "Notizbuch benennen", "nComponentsColumnInfo": "Sie haben {{n}} Spalte(n) ausgewählt. n_components muss kleiner oder gleich {{n}} sein. Reduzieren Sie n_components oder wählen Sie mehr Spalten aus, um Fehler zu vermeiden.", + "nameYourNotebook": "Notizbuch benennen", "newDatasetCreatedWithTransformations": "Ein neuer Datensatz wird mit diesen Transformationen erstellt. Er kann mit anderen Modulen verwendet werden, ohne das Original zu beeinflussen.", + "newFolder": "Neuer Ordner", "noDataQualityIssuesDetected": "Keine Datenqualitätsprobleme erkannt", - "noDuplicateRows": "Keine doppelten Zeilen im Datensatz gefunden.", - "qualityIssuesFound_one": "{{count}} Problem erkannt", - "qualityIssuesFound_other": "{{count}} Probleme erkannt", "noDataset": "Kein Datensatz", "noDatasetsAvailable": "Keine Datensätze verfügbar", + "noDuplicateRows": "Keine doppelten Zeilen im Datensatz gefunden.", "noExplorersOrConverters": "Beginnen Sie mit der Exploration, indem Sie Ihren ersten Explorer oder Konverter hinzufügen!", + "noFolder": "Kein Ordner", "noMissingValues": "Keine fehlenden Werte im Datensatz gefunden.", - "none": "keine", "noStrongCorrelationsFound": "Keine starken Korrelationen erkannt", + "noToolsMatched": "Keine Werkzeuge gefunden, die Ihrer Suche entsprechen.", + "noTransformationsApplied": "Keine Transformationen angewendet.", + "noTransformationsAppliedYet": "Noch keine Transformationen angewendet.", + "none": "keine", "notebookCreationNote": "Eine Kopie des ausgewählten Datensatzes wird erstellt, um im Notizbuch zu arbeiten, ohne das Original zu verändern.", "notebookDescription": "Notizbuchbeschreibung", "notebookHistory": "Notizbuchverlauf: {{notebook}}", "notebookInformation": "Notizbuchinformationen", "notebookName": "Notizbuchname", "notebooks": "Notizbücher", - "noToolsMatched": "Keine Werkzeuge gefunden, die Ihrer Suche entsprechen.", - "dropToolHere": "Hier ablegen zum Hinzufügen", - "noTransformationsApplied": "Keine Transformationen angewendet.", - "noTransformationsAppliedYet": "Noch keine Transformationen angewendet.", "numericalAnalysis": "Numerische Analyse", "outliers": "Ausreißer", "overview": "Übersicht", @@ -316,11 +313,15 @@ "position": "Position", "possibleIDColumns": "Mögliche ID-Spalten", "presetScale": "Voreingestellte Skala", - "processingTitle": "Ihr Dataset wird verarbeitet...", + "previewRows": "Vorschauzeilen", + "previewRowsDescription": "Anzahl der Zeilen für die Vorschau (Minimum 2).", "processingMessage": "Dies kann je nach Größe Ihrer Daten einige Momente dauern.", + "processingTitle": "Ihr Dataset wird verarbeitet...", "proportion": "Anteil", "q1": "Q1", "q3": "Q3", + "qualityIssuesFound_one": "{{count}} Problem erkannt", + "qualityIssuesFound_other": "{{count}} Probleme erkannt", "qualityScore": "Qualitätswert: {{value}}{{unit}}", "range": "Bereich", "requiredColumns": "Erforderliche Spalten", @@ -328,6 +329,7 @@ "requiredColumns_min": "Erforderliche Spalten: mindestens {{min}}", "requiredColumns_range": "Erforderliche Spalten: zwischen {{min}} und {{max}}", "restrictedDataTypes": "Eingeschränkte Datentypen: <1><0>", + "reverseColorscale": "Farbskala umkehren", "rightSkewedWarning": "<0>Rechtsschiefe Verteilung: Erwägen Sie eine Logarithmustransformation.", "rowIndicesPlaceholder": "0,1,2,5,10 oder all", "rowsColumnsInfo": "Zeilen: {{totalRows}} | Spalten: {{totalColumns}}", @@ -336,6 +338,7 @@ "saveProcessedDataset": "Verarbeiteten Datensatz speichern", "scopeColumns": "Bereich - Spalten", "scopeRows": "Bereich - Zeilen", + "searchConverters": "Converter suchen", "searchDatasetsNotebooks": "Datensätze und Notizbücher suchen", "searchExplorersConverters": "Explorer/Konverter suchen", "selectColorscale": "Farbskala auswählen", @@ -345,6 +348,14 @@ "selectDataset": "Datensatz auswählen", "selectDatasetFirst": "Zuerst einen Datensatz auswählen", "selectDatasetForNotebook": "Datensatz für das Notizbuch auswählen", + "selectDatasetsToDelete": "Datensätze zum Löschen auswählen", + "selectNotebookToAccessAnalysisTools": "Wählen Sie ein Notizbuch aus, um auf Analysewerkzeuge zuzugreifen.", + "selectNotebooksToDelete": "Notizbücher zum Löschen auswählen", + "selectScopeDescriptionColumns": "Hier konfigurieren Sie, auf welche Spalten der Konverter angewendet werden soll.", + "selectScopeDescriptionRows": "Hier konfigurieren Sie, auf welche Zeilen der Konverter angewendet werden soll.", + "selectScopeStep": "Schritt {{step}}: Bereich auswählen", + "selectTargetColumnDescription": "Wählen Sie eine Spalte als Zielvariable für überwachtes Lernen aus.", + "selectUploadMethod": "Wählen Sie eine Methode zum Hochladen Ihrer Daten", "selectedColumns_one": "{{count}} Spalte ausgewählt", "selectedColumns_other": "{{count}} Spalten ausgewählt", "selectedDataloaderConfiguration": "{{dataloader}}-Konfiguration", @@ -352,27 +363,22 @@ "selectedOrder": "Ausgewählte Reihenfolge", "selectedRows": "Ausgewählte Zeilen: {{value}}", "selectionMode": "Auswahlmodus", - "selectNotebookToAccessAnalysisTools": "Wählen Sie ein Notizbuch aus, um auf Analysewerkzeuge zuzugreifen.", - "selectScopeDescriptionColumns": "Hier konfigurieren Sie, auf welche Spalten der Konverter angewendet werden soll.", - "selectScopeDescriptionRows": "Hier konfigurieren Sie, auf welche Zeilen der Konverter angewendet werden soll.", - "selectScopeStep": "Schritt {{step}}: Bereich auswählen", - "selectTargetColumnDescription": "Wählen Sie eine Spalte als Zielvariable für überwachtes Lernen aus.", - "selectUploadMethod": "Wählen Sie eine Methode zum Hochladen Ihrer Daten", "shapeIndicators": "Formindikatoren", "showGrid": "{{axis}}-Achse Gitter anzeigen", - "showingRowsInference": "{{sampleLength}} von {{previewRowCount}} Zeilen für die Typinferenz analysiert.", - "showingRowsPreview": "{{sampleLength}} von {{previewRowCount}} Zeilen werden angezeigt.", "showLegend": "Legende anzeigen", "showMore": "Mehr anzeigen ({{count}} verbleibend)", "showMore_one": "Mehr anzeigen ({{count}} verbleibend)", "showMore_other": "Mehr anzeigen ({{count}} verbleibend)", "showZeroLine": "{{axis}}-Achse Nulllinie anzeigen", + "showingRowsInference": "{{sampleLength}} von {{previewRowCount}} Zeilen für die Typinferenz analysiert.", + "showingRowsPreview": "{{sampleLength}} von {{previewRowCount}} Zeilen werden angezeigt.", "skewness": "Schiefe", "someTypeChangesCannotBeApplied": "Einige Typänderungen können nicht angewendet werden:", "startIndex": "Startindex", "stdDev": "Standardabweichung", "strongCorrelations": "Starke Korrelationen", "targetColumn": "Zielspalte", + "task": "Aufgabe", "text": "Text", "tickLabels": "Skalierungsbeschriftungen", "tickLabelsHelper": "Eine Beschriftung pro Zeile", @@ -387,36 +393,33 @@ "totalPercentage": "{{percentage}}% des Gesamten", "totalRows": "Gesamtzeilen", "totalRowsCount": "Gesamtzeilen: {{total}}", + "totalsColor": "Farbe für Summen", "tourDisabledMessage": "Zurück zur Startseite, um die Tour zu starten", "tourDisabledMessageNotebook": "Zurück zur Datensatzvisualisierung, um die Tour zu starten", "traceIdx": "Spur {{index}}: {{trace}}", + "transparent": "Transparent", "typeChangeWarnings": "Warnungen zu Typänderungen:", "typeToSearchDatasets": "Tippen Sie, um Datensätze zu suchen...", "unique": "Eindeutig", - "uniquenessFormula": "Eindeutigkeit = (Eindeutig ÷ Gesamt) x 100", "uniquePercentage": "{{percentage}}% Eindeutig", "uniqueValues": "Eindeutige Werte", + "uniquenessFormula": "Eindeutigkeit = (Eindeutig ÷ Gesamt) x 100", "unknownDataset": "Unbekannter Datensatz", "uploadAndConfigure": "Laden Sie Ihren Datensatz hoch und konfigurieren Sie Parameter", - "importFromHub": "Datei-Hub", - "importFromHubDescription": "Datensätze aus externen Quellen wie HuggingFace und OpenML durchsuchen und herunterladen.", "uploadDataset": "Datensatz", "uploadDatasetBeforeCreatingSession": "Sie müssen einen Datensatz hochladen, bevor Sie eine Sitzung erstellen. Gehen Sie zum Datensatz-Modul, um Ihre Daten hochzuladen.", "uploadDatasetDescription": "Importieren Sie Ihre Daten aus verschiedenen Quellen und Formaten", "uploadYourDataset": "Laden Sie Ihren Datensatz hoch", "upperBound": "Obergrenze", + "useNativeTypes": "Native Typen verwenden", + "useNativeTypesDescription": "Spaltentypen aus der Datei verwenden statt statistische Inferenz. Schneller und exakt für selbstbeschreibende Formate.", "validateTypeChanges": "Typänderungen validieren", "validatingTypeChanges": "Typänderungen werden validiert...", "valueDistribution": "Werteverteilung", "valueType": "Wertetyp", "viewMode": "Ansichtsmodus", "xAxis": "X-Achse", - "yAxis": "Y-Achse", - "previewRows": "Vorschauzeilen", - "previewRowsDescription": "Anzahl der Zeilen für die Vorschau (Minimum 2).", - "useNativeTypes": "Native Typen verwenden", - "useNativeTypesDescription": "Spaltentypen aus der Datei verwenden statt statistische Inferenz. Schneller und exakt für selbstbeschreibende Formate.", - "reverseColorscale": "Farbskala umkehren" + "yAxis": "Y-Achse" }, "message": { "columnTypesUpdated": "Spaltentypen erfolgreich aktualisiert", diff --git a/DashAI/front/src/utils/i18n/locales/de/experiments.json b/DashAI/front/src/utils/i18n/locales/de/experiments.json index 3f091f917..71a2580a8 100644 --- a/DashAI/front/src/utils/i18n/locales/de/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/de/experiments.json @@ -34,53 +34,57 @@ "label": { "addModelsToExperiment": "Modelle zum Experiment hinzufügen", "addOptimizersToExperiment": "Optimierer zum Experiment hinzufügen", + "cardinalityAtLeast": "mindestens {{min}}", + "cardinalityBetween": "{{min}} bis {{max}}", "columnsInvalidRequirements": "Aktuelle Ein- und Ausgabespalten erfüllen nicht die Anforderungen von {{taskName}}", "columnsValidRequirements": "Aktuelle Ein- und Ausgabespalten erfüllen die Anforderungen von {{taskName}}", "configureExperimentsSubtitle": "Experimente konfigurieren, um Modelle zu trainieren.", "configureModels": "Modelle konfigurieren", "configureOptimizer": "Hyperparameter-Optimierung konfigurieren", + "crossValidation": "Kreuzvalidierung", "currentExperiments": "Aktuelle Experimente", "currentOptimizerSettings": "Aktuelle Optimierereinstellungen {{optimizer}}", + "cvType": "Art der Kreuzvalidierung", "datasetInputColumnRequirements": "<0>Die Eingabespalten müssen folgende Typen haben<1><2>, und eine Kardinalität von <3>{{cardinality}}.", "datasetOutputColumnRequirements": "<0>Die Ausgabespalten müssen folgende Typen haben<1><2>, und eine Kardinalität von <3>{{cardinality}}.", - "cardinalityAtLeast": "mindestens {{min}}", - "cardinalityBetween": "{{min}} bis {{max}}", "duration": "Dauer", "endTime": "Endzeit", "experimentName": "Experimentname", "experimentsModuleTitle": "Experiment-Modul", + "groupColumn": "Spalte zum Gruppieren", + "groupColumnDescription": "Spalte, die zur Gruppierung verwendet wird. Stichproben derselben Gruppe bleiben im selben Fold zusammen, sodass eine Gruppe nicht gleichzeitig in Trainings- und Testmenge erscheint.", + "holdout": "Holdout", "manual": "Manuell", "metricToOptimize": "Zu optimierende Metrik", "missingValues": "fehlende Werte", "missingValuesDetected": "Der Datensatz enthält fehlende Werte (NaN) in den Spalten:", "modelName": "Modellname", "modelsInExperiment": "Aktuelle Modelle im Experiment", + "noConverterAdded": "Es wurde kein Converter hinzugefügt.", "noDatasetsAvailable": "Keine Datensätze verfügbar", "noDatasetsAvailableGoToDataTab": "Gehen Sie zum <1>Daten-Tab, um zuerst einen hochzuladen.", "noModelsAvailable": "Keine Modelle verfügbar", "noOptimizersNoMetric": "Keine Hyperparameter-Optimierung", + "numFolds": "Anzahl der Folds (k)", + "numFoldsDescription": "Anzahl der Folds bei der Kreuzvalidierung. Mehr Folds können die Stabilität der Bewertung verbessern, erhöhen aber die Verarbeitungszeit. Muss eine ganze Zahl größer als 1 sein.", + "numRepeats": "Anzahl der Wiederholungen", + "numRepeatsDescription": "Anzahl der Wiederholungen des Kreuzvalidierungsprozesses. Ein höherer Wert liefert präzisere Werte, erfordert aber mehr Trainingszeit. Muss eine ganze Zahl größer als 1 sein.", "optimizer": "Optimierer", "optimizerMetric": "Optimierungsmetrik", "parameterModification": "Parametermodifikation", - "prepareDataset": "Datensatz vorbereiten", "predefined": "Vordefiniert", + "prepareDataset": "Datensatz vorbereiten", "random": "Zufällig", "recommendPreprocessMissingValues": "Es wird empfohlen, den Datensatz vorzuverarbeiten, um diese fehlenden Werte vor dem Training eines Modells zu behandeln.", "rowIndexes": "Zeilenindizes", "rowIndexesDescription": "Geben Sie Zeilenbereiche für jede Aufteilung mit kommaseparierten Werten oder Bereichen an (z.B. 0-100, 200).", "seed": "Seed", + "selectAColumn": "Eine Spalte auswählen", + "selectDataset": "Datensatz auswählen", "selectDatasetColumns": "Geben Sie an, welche Spalten des Datensatzes als Ein- und Ausgabe verwendet werden.", "selectDatasetTitle": "Datensatz für die ausgewählte Aufgabe auswählen", - "selectHowToDivideDataset": "Wählen Sie, wie der Datensatz in Trainings-, Validierungs- und Testmengen aufgeteilt werden soll.", "selectEvaluationStrategy": "Bewertungsstrategie", - "holdout": "Holdout", - "crossValidation": "Kreuzvalidierung", - "cvType": "Art der Kreuzvalidierung", - "numFolds": "Anzahl der Folds (k)", - "numFoldsDescription": "Anzahl der Folds bei der Kreuzvalidierung. Mehr Folds können die Stabilität der Bewertung verbessern, erhöhen aber die Verarbeitungszeit. Muss eine ganze Zahl größer als 1 sein.", - "numRepeats": "Anzahl der Wiederholungen", - "numRepeatsDescription": "Anzahl der Wiederholungen des Kreuzvalidierungsprozesses. Ein höherer Wert liefert präzisere Werte, erfordert aber mehr Trainingszeit. Muss eine ganze Zahl größer als 1 sein.", - "groupColumn": "Spalte zum Gruppieren", + "selectHowToDivideDataset": "Wählen Sie, wie der Datensatz in Trainings-, Validierungs- und Testmengen aufgeteilt werden soll.", "selectInputOutputColumnsDescription": "Spaltenbezeichnungen aus den Listen auswählen.", "selectMetrics": "Metriken auswählen", "selectModelFirst": "Zuerst ein Modell auswählen", @@ -88,20 +92,17 @@ "setNameAndTask": "Name und Aufgabe festlegen", "shuffle": "Mischen", "shuffleDescription": "Bestimmt, ob die Daten beim Definieren der Mengen gemischt werden. Muss wahr sein, um die Daten zu mischen, andernfalls falsch.", + "splitType": "Aufteilungstyp", "splits": "Aufteilungen", "splitsDescription": "Anteil der jedem Teilbereich zugewiesenen Daten. Werte müssen zwischen 0 und 1 liegen und 1 ergeben.", - "splitType": "Aufteilungstyp", "startTime": "Startzeit", "stratify": "Stratifizieren", "stratifyDescription": "Legt fest, ob die Daten proportional gemäß der Klassenverteilung in jeder Menge aufgeteilt werden. Mischen muss wahr sein, um die Daten zu stratifizieren.", + "stratifyRequiresShuffle": "Erfordert aktiviertes Mischen", "useManualSplittingBySpecifyingRowIndexes": "Manuelle Aufteilung durch Angabe der Zeilenindizes jeder Teilmenge", "usePredefinedSplitsFromDataset": "Vordefinierte Aufteilungen aus dem Datensatz verwenden", "usePredefinedSplitsFromDatasetNotAvailable": "Vordefinierte Aufteilungen aus dem Datensatz verwenden (nicht verfügbar)", - "useRandomRowsBySpecifyingPortion": "Zufällige Zeilen durch Angabe des Anteils des Datensatzes für jede Teilmenge", - "selectAColumn": "Eine Spalte auswählen", - "groupColumnDescription": "Spalte, die zur Gruppierung verwendet wird. Stichproben derselben Gruppe bleiben im selben Fold zusammen, sodass eine Gruppe nicht gleichzeitig in Trainings- und Testmenge erscheint.", - "stratifyRequiresShuffle": "Erfordert aktiviertes Mischen", - "selectDataset": "Datensatz auswählen" + "useRandomRowsBySpecifyingPortion": "Zufällige Zeilen durch Angabe des Anteils des Datensatzes für jede Teilmenge" }, "message": { "confirmDeleteRun": "Sind Sie sicher, dass Sie diesen Durchlauf löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", diff --git a/DashAI/front/src/utils/i18n/locales/de/models.json b/DashAI/front/src/utils/i18n/locales/de/models.json index 47bab16f2..501e34c7e 100644 --- a/DashAI/front/src/utils/i18n/locales/de/models.json +++ b/DashAI/front/src/utils/i18n/locales/de/models.json @@ -1,22 +1,22 @@ { "button": { + "addConverter": "Converter hinzufügen", + "addNewPrediction": "Neue Vorhersage hinzufügen", "createExplainer": "Erklärungsmodell erstellen", "createGlobalExplainer": "Neues globales Erklärungsmodell", "createLocalExplainer": "Neues lokales Erklärungsmodell", "createPrediction": "Vorhersage erstellen", - "addNewPrediction": "Neue Vorhersage hinzufügen", - "uploadNewDataset": "Neuen Datensatz hochladen", "createSession": "Sitzung erstellen", "deleteAndRetrain": "Löschen und neu trainieren", "deleteRun": "Durchlauf löschen", "hideOperations": "Operationen ausblenden", "hideParameters": "Parameter ausblenden", "modelsHub": "Modell-Zentrale", - "newSession": "Neue Sitzung", "modifyParameters": "Parameter bearbeiten", - "newPrediction": "Neue Vorhersage", "newDatasetPrediction": "Neue Datensatz-Vorhersage", "newManualPrediction": "Neue manuelle Vorhersage", + "newPrediction": "Neue Vorhersage", + "newSession": "Neue Sitzung", "retrain": "Neu trainieren", "runAll": "Alle ausführen", "runAllModels": "Alle Modelle ausführen", @@ -24,11 +24,13 @@ "saveAndRunModel": "Modell speichern und ausführen", "showOperations": "Operationen anzeigen", "showParameters": "Parameter anzeigen", - "updateAndRetrain": "Aktualisieren und neu trainieren" + "updateAndRetrain": "Aktualisieren und neu trainieren", + "uploadNewDataset": "Neuen Datensatz hochladen" }, "error": { "completeRequiredFields": "Bitte füllen Sie alle Pflichtfelder aus", "createRun": "Fehler beim Erstellen eines neuen Durchlaufs: {{name}}", + "createRunReason": "Fehler beim Erstellen eines neuen Durchlaufs \"{{name}}\": {{reason}}", "createSession": "Fehler beim Erstellen einer neuen Sitzung", "datasetRequired": "Datensatz ist erforderlich", "enterModelName": "Bitte geben Sie einen Namen für das Modell ein", @@ -70,104 +72,33 @@ }, "label": { "addModelToSession": "Modell zur Sitzung hinzufügen", + "allRepetitions": "Alle Wiederholungen", + "alternativeHypothesis": "Alternativhypothese", + "applyPreprocessing": "Vorverarbeitung anwenden", + "applyPreprocessingDescription": "Die ausgewählten Converter werden nur anhand der Trainingsdaten angepasst und auf den Rest angewendet, um Datenlecks zu vermeiden. Wird dies nicht angewendet, trainiert das Modell mit den Daten wie sie sind.", "availableExplainers": "Verfügbare Erklärungsmodelle", "availableModels": "Verfügbare Modelle", - "noSavedTests": "Noch keine gespeicherten Tests.", - "name": "Name", - "metric": "Metrik", - "metricSplit": "Aufteilung", - "significantsCount_one": "{{count}} / {{total}} signifikant", - "significantsCount_other": "{{count}} / {{total}} signifikanten", - "bestModel": "Bestes", - "score": "Bewertung", - "scoreProfile": "Bewertungsprofil", - "scoreHeaderTooltip": "Gewichtete Bewertung von 0–100 basierend auf dem ausgewählten Profil. Fehlermetriken ohne Obergrenze (MAE, RMSE, TER, …) werden relativ zum schlechtesten Modell in diesem Vergleich normalisiert. Bewegen Sie den Mauszeiger über eine Zelle, um die Aufschlüsselung anzuzeigen.", - "profile_balanced": "Ausgewogen", - "profile_detectPositives": "Positive Erkennen", - "profile_avoidFalseAlarms": "Falschalarme Vermeiden", - "profile_probabilityQuality": "Qualität der Wahrscheinlichkeiten", - "profile_regression_fit": "Modellanpassung", - "profile_regression_error": "Ausgewogener Fehler", - "profile_translation_quality": "Übersetzungsqualität", - "profile_translation_balanced": "Ausgewogene Übersetzung", - "profile_text_balanced": "Ausgewogen", - "profile_text_detectPositives": "Positive Erkennen", - "profile_text_avoidFalseAlarms": "Falschalarme Vermeiden", - "profile_text_probabilityQuality": "Qualität der Wahrscheinlichkeiten", - "nestedCrossValidation": "Verschachtelte Kreuzvalidierung", - "outerSplitterInherited": "Die ursprüngliche Splitter-Konfiguration der Sitzung wird für die äußere Schleife der verschachtelten Kreuzvalidierung übernommen", - "innerLoopConfiguration": "Konfigurieren Sie die Datenaufteilung für die innere Schleife der verschachtelten Kreuzvalidierung", - "outerSplitter": "Splitter für die äußere Schleife", - "outerFolds": "Anzahl der Folds", - "innerSplitter": "Splitter für die innere Schleife", - "innerFolds": "Anzahl der Folds", - "statisticalTests": "Statistische Tests", - "modelsToCompare": "Zu vergleichende Modelle", - "significanceLevel": "Signifikanzniveau", - "noFinishedRuns": "Keine abgeschlossenen Durchläufe verfügbar", - "result": "Ergebnis", - "statistic": "Statistik", - "significant": "Signifikant", - "notSignificant": "Nicht Signifikant", - "technicalDetails": "Technische Details", - "model1": "Modell 1", - "model2": "Modell 2", - "nemenyiPairwiseComparisons": "Nemenyi-Post-hoc-Vergleiche nach dem Friedman-Test", - "tukeyPairwiseComparisons": "Tukey-Post-hoc-Vergleiche nach dem ANOVA-Test", - "wilcoxonPairwiseComparisons": "Paarweise Wilcoxon-Tests mit Holm-Korrektur", - "alternativeHypothesis": "Alternativhypothese", - "correctionMethod": "Korrekturmethode", - "repetition": "Wiederholung", - "allRepetitions": "Alle Wiederholungen", "averaged": "Gemittelt", - "lines": "Linien", - "histogramPlot": "Histogramm", - "foldNumber": "Fold-Nummer", - "theoricalQuantiles": "Theoretische Quantile", - "sampleQuantiles": "Stichprobenquantile", - "metricValue": "Metrikwert", - "frequency": "Häufigkeit", - "graphs": "Grafiken", - "helperTests": "Voraussetzungstests", - "selectAtLeastRuns": "Wählen Sie mindestens {{min}} Modelle aus", - "selectExactlyRuns_one": "Wählen Sie genau {{count}} Modell aus", - "selectExactlyRuns_other": "Wählen Sie genau {{count}} Modelle aus", - "normalityByRunSummary": "{{normal}} von {{total}} Durchläufen scheinen einer Normalverteilung zu folgen (α = {{alpha}}).", - "notNormal": "Nicht normal", - "normal": "Normal", - "selectBetweenRuns": "Wählen Sie zwischen {{min}} und {{max}} Modellen aus", - "saveDetails": "Details zum Speichern", - "testName": "Testname (optional)", - "testNameHelp": "Wenn Sie dieses Feld leer lassen, wird der Testname verwendet.", - "testDescription": "Beschreibung (optional)", - "resultSaved": "Gespeichert", - "saveResult": "Ergebnis speichern", - "withHpo": "Mit HPO", - "withoutHpo": "Ohne HPO", - "nestedCv": "Verschachtelte Kreuzvalidierung", + "bar": "Balken", + "bestModel": "Bestes", + "chartType": "Diagrammtyp", "chooseTaskForSessionWithDataset": "Wählen Sie die ML-Aufgabe für Ihre Sitzung mit dem Datensatz \"{{datasetName}}\".", "configuration": "Konfiguration", "configureModel": "Modell konfigurieren", "configureOptimizer": "Optimierer konfigurieren", "configureSession": "Sitzung konfigurieren", "configureTasksTrainCompareModels": "Aufgaben konfigurieren, Modelle trainieren und vergleichen. Wählen Sie eine Aufgabe, um Ihren Modellierungs-Workflow zu beginnen.", - "confirmDeleteSession": "Sind Sie sicher, dass Sie die Sitzung \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "configureTest": "Test konfigurieren", "confirmBulkDeleteSessions_one": "Sind Sie sicher, dass Sie die ausgewählte Sitzung löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", "confirmBulkDeleteSessions_other": "Sind Sie sicher, dass Sie die {{count}} ausgewählten Sitzungen löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", - "selectSessionsToDelete": "Sitzungen zum Löschen auswählen", + "confirmDeleteSession": "Sind Sie sicher, dass Sie die Sitzung \"{{name}}\" löschen möchten? Diese Aktion kann nicht rückgängig gemacht werden.", + "confirmParameterUpdate": "Parameteraktualisierung bestätigen", + "correctionMethod": "Korrekturmethode", "customMetrics": "Benutzerdefinierte Metriken", "datasetPredictions": "Datensatz-Vorhersagen", + "divideColumnsAndSplits": "Spalten aufteilen und Datensatz-Splits konfigurieren", "dropModelHere": "Hier ablegen zum Hinzufügen", "editRun": "Durchlauf bearbeiten", - "chartType": "Diagrammtyp", - "savedTests": "Gespeicherte Tests", - "foldGraphs": "Fold-Grafiken", - "nestedCvResults": "Verschachtelte Ergebnisse", - "outerFoldFetch": "Äußere Folds", - "finalFoldFetch": "Finale Folds", - "twoSided": "Beidseitig (Modell 1 ≠ Modell 2)", - "greater": "Größer (Modell 1 > Modell 2)", - "less": "Kleiner (Modell 1 < Modell 2)", "editRunParameters": "Parameter bearbeiten und Modell neu ausführen", "epoch": "Epoche", "exitModelDetailToAddModels": "Kehren Sie zur Sitzungsübersicht zurück, um weitere Modelle hinzuzufügen.", @@ -175,39 +106,64 @@ "explainability": "Erklärbarkeit", "explainersCount_one": "• <1>{{count}} Erklärungsmodell", "explainersCount_other": "• <1>{{count}} Erklärungsmodelle", + "finalFoldFetch": "Finale Folds", + "foldGraphs": "Fold-Grafiken", + "foldNumber": "Fold-Nummer", + "frequency": "Häufigkeit", "generalMetrics": "Allgemeine Metriken", "globalExplainer": "Globales Erklärungsmodell", "globalExplainers": "Globale Erklärungsmodelle", "goalMetric": "Zielmetrik", + "graphs": "Grafiken", + "greater": "Größer (Modell 1 > Modell 2)", "heatmap": "Heatmap", + "helperTests": "Voraussetzungstests", "hideResults": "Ergebnisse ausblenden", "higherIsBetter": "Höher ist besser", + "histogramPlot": "Histogramm", "hyperparameterOptimizationPlots": "Hyperparameter-Optimierungsplots", "hyperparameterOptimizerConfiguration": "Hyperparameter-Optimierer-Konfiguration", "hyperparameters": "Hyperparameter", + "innerFolds": "Anzahl der Folds", + "innerLoopConfiguration": "Konfigurieren Sie die Datenaufteilung für die innere Schleife der verschachtelten Kreuzvalidierung", + "innerSplitter": "Splitter für die innere Schleife", "inputColumns": "Eingabespalten", + "less": "Kleiner (Modell 1 < Modell 2)", + "lines": "Linien", "liveMetrics": "Live-Metriken", "localExplainer": "Lokales Erklärungsmodell", "localExplainers": "Lokale Erklärungsmodelle", "lowerIsBetter": "Niedriger ist besser", - "manualPredictions": "Manuelle Vorhersagen", "manual": "Manuell", + "manualPredictions": "Manuelle Vorhersagen", + "metric": "Metrik", + "metricSplit": "Aufteilung", + "metricToOptimize": "Zu optimierende Metrik", + "metricValue": "Metrikwert", "metrics": "Metriken", "metricsEmptyForDisplaySet": "Die Ergebnismetriken für {{set}} sind leer.", - "metricToOptimize": "Zu optimierende Metrik", + "minTwoRunsRequired": "Wählen Sie mindestens 2 abgeschlossene Durchläufe aus, um statistische Tests durchzuführen", + "model1": "Modell 1", + "model2": "Modell 2", "modelComparison": "Modellvergleich", "modelConfiguration": "Modellkonfiguration", "modelCount_one": "{{count}} Modell", "modelCount_other": "{{count}} Modelle", - "bar": "Balken", "modelsModule": "Modell-Modul", + "modelsToCompare": "Zu vergleichende Modelle", + "name": "Name", "nameYourSession": "Sitzung benennen", + "nemenyiPairwiseComparisons": "Nemenyi-Post-hoc-Vergleiche nach dem Friedman-Test", + "nestedCrossValidation": "Verschachtelte Kreuzvalidierung", + "nestedCv": "Verschachtelte Kreuzvalidierung", + "nestedCvResults": "Verschachtelte Ergebnisse", "noCompatibleExplainersFound": "Keine kompatiblen Erklärungsmodelle gefunden", "noCompatibleModelsFound": "Keine kompatiblen Modelle gefunden", "noCompletedRuns": "Noch keine abgeschlossenen Durchläufe", "noConfigurationAvailable": "Keine Konfiguration für diesen Durchlauf verfügbar", "noDatasetPredictionsYet": "Noch keine Datensatz-Vorhersagen", "noExplainersMatchSearch": "Keine Erklärungsmodelle entsprechen Ihrer Suche", + "noFinishedRuns": "Keine abgeschlossenen Durchläufe verfügbar", "noGlobalExplainersYet": "Noch keine globalen Erklärungsmodelle", "noHyperparameterPlotsAvailable": "Keine Hyperparameter-Plots verfügbar. Nur Durchläufe mit mindestens einem optimierbaren Parameter erzeugen Plots.", "noLocalExplainersYet": "Noch keine lokalen Erklärungsmodelle", @@ -217,19 +173,50 @@ "noModelsMatchSearch": "Keine Modelle entsprechen Ihrer Suche", "noPredictionsYet": "Noch keine Vorhersagen", "noRunsYet": "Noch keine Durchläufe. Fügen Sie Modelle aus dem rechten Panel hinzu.", + "noSavedTests": "Noch keine gespeicherten Tests.", "noSessionSelected": "Keine Sitzung ausgewählt", + "noTestsMatch": "Kein Test entspricht Ihrer Suche", + "nonParametricTests": "Nichtparametrische Tests", + "normal": "Normal", + "normalityByRunSummary": "{{normal}} von {{total}} Durchläufen scheinen einer Normalverteilung zu folgen (α = {{alpha}}).", + "notNormal": "Nicht normal", + "notSignificant": "Nicht Signifikant", "operations": "Vorgänge", "operationsWillBeDeletedWarning": "Diese Operationen werden dauerhaft gelöscht und können nicht wiederhergestellt werden. Sind Sie sicher?", + "optimizer": "Optimierer", "optimizerConfiguration": "Einstellungen des Hyperparameter-Optimierers konfigurieren", "optimizerParameters": "Optimiererparameter", - "optimizer": "Optimierer", + "outerFoldFetch": "Äußere Folds", + "outerFolds": "Anzahl der Folds", + "outerSplitter": "Splitter für die äußere Schleife", + "outerSplitterInherited": "Die ursprüngliche Splitter-Konfiguration der Sitzung wird für die äußere Schleife der verschachtelten Kreuzvalidierung übernommen", "outputColumns": "Ausgabespalten", + "parametricTests": "Parametrische Tests", "pleaseSelectMetricToOptimize": "Bitte wählen Sie eine Metrik zur Optimierung aus.", "predictions": "Vorhersagen", "predictionsCount_one": "• <1>{{count}} Vorhersage", "predictionsCount_other": "• <1>{{count}} Vorhersagen", - "divideColumnsAndSplits": "Spalten aufteilen und Datensatz-Splits konfigurieren", "prepareDataset": "Datensatz vorbereiten", + "preprocessingFailed": "Vorverarbeitung fehlgeschlagen", + "preprocessingInProgress": "Verarbeitung läuft...", + "preprocessingOptional": "Vorverarbeitung (optional)", + "preprocessingOptionalDescription": "Die ausgewählten Converter werden nur anhand der Trainingsdaten angepasst und auf den Rest angewendet, um Datenlecks zu vermeiden.", + "profile_avoidFalseAlarms": "Falschalarme Vermeiden", + "profile_balanced": "Ausgewogen", + "profile_detectPositives": "Positive Erkennen", + "profile_probabilityQuality": "Qualität der Wahrscheinlichkeiten", + "profile_regression_error": "Ausgewogener Fehler", + "profile_regression_fit": "Modellanpassung", + "profile_text_avoidFalseAlarms": "Falschalarme Vermeiden", + "profile_text_balanced": "Ausgewogen", + "profile_text_detectPositives": "Positive Erkennen", + "profile_text_probabilityQuality": "Qualität der Wahrscheinlichkeiten", + "profile_translation_balanced": "Ausgewogene Übersetzung", + "profile_translation_quality": "Übersetzungsqualität", + "repetition": "Wiederholung", + "reports": "Berichte", + "result": "Ergebnis", + "resultSaved": "Gespeichert", "retrainConfirmDetails": "Sind Sie sicher, dass Sie den Durchlauf \"<1>{{runName}}\" neu trainieren möchten?", "retrainModel": "Modell neu trainieren?", "retrainWillDeleteOperations": "Dieser Durchlauf hat bestehende Operationen, die gelöscht werden", @@ -239,47 +226,70 @@ "runFailedNoHyperparameterPlots": "Durchlauf fehlgeschlagen. Keine Hyperparameter-Plots verfügbar.", "runInProgressCannotEdit": "Der Durchlauf wird gerade ausgeführt und kann nicht bearbeitet werden.", "runName": "Durchlaufname", - "confirmParameterUpdate": "Parameteraktualisierung bestätigen", "runNotFound": "Durchlauf nicht gefunden", "runNotStartedNoHyperparameterPlots": "Durchlauf nicht gestartet. Keine Hyperparameter-Plots verfügbar.", + "runsSelected": "ausgewählte Durchläufe", + "sampleQuantiles": "Stichprobenquantile", "saveConfirmDetails": "Das Speichern von \"<1>{{runName}}\" setzt seinen Status auf 'Nicht gestartet' zurück und löscht seine aktuellen Metriken und Ergebnisse. Möchten Sie wirklich fortfahren?", + "saveDetails": "Details zum Speichern", "saveParameterChanges": "Parameteränderungen speichern?", + "saveResult": "Ergebnis speichern", "saveWillDeleteOperationsDetails": "Das Speichern von \"<1>{{runName}}\" setzt den Durchlauf zurück. Folgendes wird beim erneuten Training gelöscht:", + "savedTests": "Gespeicherte Tests", + "score": "Bewertung", + "scoreHeaderTooltip": "Gewichtete Bewertung von 0–100 basierend auf dem ausgewählten Profil. Fehlermetriken ohne Obergrenze (MAE, RMSE, TER, …) werden relativ zum schlechtesten Modell in diesem Vergleich normalisiert. Bewegen Sie den Mauszeiger über eine Zelle, um die Aufschlüsselung anzuzeigen.", + "scoreProfile": "Bewertungsprofil", "searchDatasetsSessions": "Datensätze und Sitzungen suchen", "searchMetric": "Metrik suchen...", "searchModels": "Modelle suchen...", "searchTests": "Tests suchen...", - "parametricTests": "Parametrische Tests", - "nonParametricTests": "Nichtparametrische Tests", - "minTwoRunsRequired": "Wählen Sie mindestens 2 abgeschlossene Durchläufe aus, um statistische Tests durchzuführen", - "noTestsMatch": "Kein Test entspricht Ihrer Suche", - "runsSelected": "ausgewählte Durchläufe", - "configureTest": "Test konfigurieren", + "selectAtLeastRuns": "Wählen Sie mindestens {{min}} Modelle aus", + "selectBetweenRuns": "Wählen Sie zwischen {{min}} und {{max}} Modellen aus", + "selectColumnsDescription": "Wählen Sie aus, welche Spalten Eingaben sind und welche die Ausgabe für diese Sitzung ist.", + "selectColumnsTitle": "Spalten auswählen", "selectDataset": "Datensatz auswählen", - "selectDatasetForSession": "Datensatz für Ihre Sitzung auswählen", "selectDatasetAndPrepare": "Benennen Sie Ihre Sitzung, wählen Sie einen Datensatz aus und konfigurieren Sie Spalten und Aufteilungen.", "selectDatasetFirst": "Wählen Sie einen Datensatz aus, um Spalten und Zeilenaufteilungen zu konfigurieren.", + "selectDatasetForSession": "Datensatz für Ihre Sitzung auswählen", + "selectExactlyRuns_one": "Wählen Sie genau {{count}} Modell aus", + "selectExactlyRuns_other": "Wählen Sie genau {{count}} Modelle aus", "selectSessionToViewModels": "Wählen Sie eine Sitzung, um verfügbare Modelle anzuzeigen.", + "selectSessionsToDelete": "Sitzungen zum Löschen auswählen", "selectTask": "Aufgabe auswählen", "selectTaskForSession": "Eine Aufgabe für Ihre Sitzung auswählen", "sessionConfiguration": "Sitzungskonfiguration", "sessionName": "Sitzungsname", "showResults": "Ergebnisse anzeigen", + "significanceLevel": "Signifikanzniveau", + "significant": "Signifikant", + "significantsCount_one": "{{count}} / {{total}} signifikant", + "significantsCount_other": "{{count}} / {{total}} signifikanten", + "statistic": "Statistik", + "statisticalTests": "Statistische Tests", "step": "Schritt", + "technicalDetails": "Technische Details", "test": "Test", + "testDescription": "Beschreibung (optional)", "testMetrics": "Testmetriken", + "testName": "Testname (optional)", + "testNameHelp": "Wenn Sie dieses Feld leer lassen, wird der Testname verwendet.", "testSet": "Testmenge", + "theoricalQuantiles": "Theoretische Quantile", "thereAreNoMetricsForThisRun": "Es gibt keine Metriken für {{set}} in diesem Durchlauf", "tourDisabledMessage": "Zurück zur Startseite, um die Tour zu starten", "train": "Training", - "trainingMetrics": "Trainingsmetriken", "trainSet": "Trainingsmenge", + "trainingMetrics": "Trainingsmetriken", "trial": "Versuch", + "tukeyPairwiseComparisons": "Tukey-Post-hoc-Vergleiche nach dem ANOVA-Test", + "twoSided": "Beidseitig (Modell 1 ≠ Modell 2)", "validation": "Validierung", "validationMetrics": "Validierungsmetriken", "validationSet": "Validierungsmenge", "viewResultsAs": "Ergebnisse als Spalten oder Graphen anzeigen", - "reports": "Berichte" + "wilcoxonPairwiseComparisons": "Paarweise Wilcoxon-Tests mit Holm-Korrektur", + "withHpo": "Mit HPO", + "withoutHpo": "Ohne HPO" }, "message": { "allRunsCompleted": "{{experiment}} hat alle Durchläufe abgeschlossen.", diff --git a/DashAI/front/src/utils/i18n/locales/en/datasets.json b/DashAI/front/src/utils/i18n/locales/en/datasets.json index 8145624e7..964efdc85 100644 --- a/DashAI/front/src/utils/i18n/locales/en/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/en/datasets.json @@ -52,6 +52,7 @@ "uploadFile": "Upload a file" }, "error": { + "fetchingConverters": "Error fetching the available converters.", "converterFailed": "An error occurred during processing the converter.", "converterFailedWithInfo": "Error processing converter: {{error}}", "createConverterError": "Failed to create converter", @@ -174,6 +175,7 @@ "constantColumns": "Constant Columns", "convert": "Convert", "converter": "Converter", + "converterOutput": "Output", "correlation": "Correlation", "correlationAnalysis": "Correlation Analysis", "correlations": "Correlations", @@ -340,6 +342,7 @@ "scopeRows": "Scope - Rows", "searchDatasetsNotebooks": "Search Datasets and Notebooks", "searchExplorersConverters": "Search explorers/converters", + "searchConverters": "Search converters", "selectColorscale": "Select Colorscale", "selectColumnsForExplorerScope": "Select the columns to be used by the explorer.", "selectDataloader": "Select Dataloader", diff --git a/DashAI/front/src/utils/i18n/locales/en/experiments.json b/DashAI/front/src/utils/i18n/locales/en/experiments.json index d166e2a37..f02175efb 100644 --- a/DashAI/front/src/utils/i18n/locales/en/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/en/experiments.json @@ -56,6 +56,7 @@ "modelName": "Model Name", "modelsInExperiment": "Current Models in the Experiment", "noDatasetsAvailable": "There are no Datasets Available", + "noConverterAdded": "No converter was added.", "noDatasetsAvailableGoToDataTab": "Go to <1>data tab to upload one first.", "noModelsAvailable": "No Models Available", "noOptimizersNoMetric": "No hyperparameter optimization", diff --git a/DashAI/front/src/utils/i18n/locales/en/models.json b/DashAI/front/src/utils/i18n/locales/en/models.json index f52b952bf..8bcbd08d3 100644 --- a/DashAI/front/src/utils/i18n/locales/en/models.json +++ b/DashAI/front/src/utils/i18n/locales/en/models.json @@ -1,5 +1,6 @@ { "button": { + "addConverter": "Add Converter", "createExplainer": "Create Explainer", "createGlobalExplainer": "New Global Explainer", "createLocalExplainer": "New Local Explainer", @@ -29,6 +30,7 @@ "error": { "completeRequiredFields": "Please complete all required fields", "createRun": "Error while trying to create a new run: {{name}}", + "createRunReason": "Error while trying to create a new run \"{{name}}\": {{reason}}", "createSession": "Error while trying to create a new session", "datasetRequired": "Dataset is required", "enterModelName": "Please enter a name for the model", @@ -230,6 +232,14 @@ "predictionsCount_other": "• <1>{{count}} predictions", "divideColumnsAndSplits": "Divide columns and configure dataset splits", "prepareDataset": "Prepare Dataset", + "applyPreprocessing": "Apply preprocessing", + "applyPreprocessingDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage. If not applied, the model trains on the data as-is.", + "preprocessingOptional": "Preprocessing (optional)", + "preprocessingOptionalDescription": "Selected converters are fit only on training data and applied to the rest, to avoid data leakage.", + "preprocessingInProgress": "Preprocessing in progress...", + "preprocessingFailed": "Preprocessing failed", + "selectColumnsTitle": "Select Columns", + "selectColumnsDescription": "Choose which columns are inputs and which one is the output for this session.", "retrainConfirmDetails": "Are you sure you want to re-train run \"<1>{{runName}}\"?", "retrainModel": "Re-train Model?", "retrainWillDeleteOperations": "This run has existing operations that will be deleted", diff --git a/DashAI/front/src/utils/i18n/locales/es/datasets.json b/DashAI/front/src/utils/i18n/locales/es/datasets.json index a37e405ac..b5f50b56d 100644 --- a/DashAI/front/src/utils/i18n/locales/es/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/es/datasets.json @@ -54,6 +54,7 @@ "uploadFile": "Subir un archivo" }, "error": { + "fetchingConverters": "Error al obtener los converters disponibles.", "converterFailed": "Ocurrió un error durante el procesamiento del convertidor.", "converterFailedWithInfo": "Error al procesar convertidor: {{error}}", "createConverterError": "Fallo al crear convertidor", @@ -180,6 +181,7 @@ "constantColumns": "Columnas Constantes", "convert": "Convertir", "converter": "Convertidor", + "converterOutput": "Salida", "correlation": "Correlación", "correlationAnalysis": "Análisis de Correlación", "correlations": "Correlaciones", @@ -348,6 +350,7 @@ "scopeRows": "Alcance - Filas", "searchDatasetsNotebooks": "Buscar Datasets y Cuadernos", "searchExplorersConverters": "Buscar exploradores/convertidores", + "searchConverters": "Buscar converters", "selectColorscale": "Seleccionar Escala de Colores", "selectColumnsForExplorerScope": "Selecciona las columnas a ser utilizadas por el explorador.", "selectDataloader": "Seleccionar Dataloader", diff --git a/DashAI/front/src/utils/i18n/locales/es/experiments.json b/DashAI/front/src/utils/i18n/locales/es/experiments.json index 53cdeb71b..cd6efcbd4 100644 --- a/DashAI/front/src/utils/i18n/locales/es/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/es/experiments.json @@ -56,6 +56,7 @@ "modelName": "Nombre del Modelo", "modelsInExperiment": "Modelos Actuales en el Experimento", "noDatasetsAvailable": "No hay Datasets Disponibles", + "noConverterAdded": "No se agregó ningún converter.", "noDatasetsAvailableGoToDataTab": "Vaya a <1>pestaña de datos para subir uno primero.", "noModelsAvailable": "No hay Modelos Disponibles", "noOptimizersNoMetric": "Sin optimización de hiperparámetros", diff --git a/DashAI/front/src/utils/i18n/locales/es/models.json b/DashAI/front/src/utils/i18n/locales/es/models.json index 38bbac05e..fd98076b8 100644 --- a/DashAI/front/src/utils/i18n/locales/es/models.json +++ b/DashAI/front/src/utils/i18n/locales/es/models.json @@ -1,5 +1,6 @@ { "button": { + "addConverter": "Agregar Converter", "createExplainer": "Crear Explicador", "createGlobalExplainer": "Nuevo Explicador Global", "createLocalExplainer": "Nuevo Explicador Local", @@ -29,6 +30,7 @@ "error": { "completeRequiredFields": "Por favor complete todos los campos requeridos", "createRun": "Error al intentar crear una nueva ejecución: {{name}}", + "createRunReason": "Error al intentar crear una nueva ejecución \"{{name}}\": {{reason}}", "createSession": "Error al intentar crear una nueva sesión", "datasetRequired": "Se requiere un dataset", "enterModelName": "Por favor ingrese un nombre para el modelo", @@ -236,6 +238,14 @@ "predictionsCount_other": "• <1>{{count}} predicciones", "divideColumnsAndSplits": "Divide columnas y configura las particiones del dataset", "prepareDataset": "Preparar Dataset", + "applyPreprocessing": "Aplicar preprocesamiento", + "applyPreprocessingDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos. Si no se aplica, el modelo se entrena con los datos tal como están.", + "preprocessingOptional": "Preprocesamiento (opcional)", + "preprocessingOptionalDescription": "Los converters seleccionados se ajustan solo sobre los datos de entrenamiento y se aplican al resto, para evitar fuga de datos.", + "preprocessingInProgress": "Procesando...", + "preprocessingFailed": "El preprocesamiento falló", + "selectColumnsTitle": "Seleccionar Columnas", + "selectColumnsDescription": "Elige qué columnas son de entrada y cuál es la de salida para esta sesión.", "retrainConfirmDetails": "¿Está seguro de que desea re-entrenar la ejecución \"<1>{{runName}}\"?", "retrainModel": "¿Re-entrenar Modelo?", "retrainWillDeleteOperations": "Esta ejecución tiene operaciones existentes que serán eliminadas", diff --git a/DashAI/front/src/utils/i18n/locales/pt/datasets.json b/DashAI/front/src/utils/i18n/locales/pt/datasets.json index e38b9ee3a..5ecccc5ec 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/pt/datasets.json @@ -54,6 +54,7 @@ "uploadFile": "Enviar um arquivo" }, "error": { + "cannotSaveEmptyDataset": "Não é possível salvar o dataset: todas as colunas foram removidas.", "converterFailed": "Ocorreu um erro durante o processamento do conversor.", "converterFailedWithInfo": "Erro ao processar conversor: {{error}}", "createConverterError": "Falha ao criar conversor", @@ -70,72 +71,54 @@ "failedToCreateDataset": "Nao foi possivel criar o conjunto de dados. Verifique seu arquivo e a configuracao e tente novamente.", "failedToCreateDatasetFromNotebook": "Falha ao criar conjunto de dados a partir do caderno", "failedToCreateExplorer": "Falha ao criar explorador", + "failedToCreateFolder": "Falha ao criar pasta", "failedToDeleteDataset": "Falha ao excluir conjunto de dados", "failedToDeleteDatasets": "Falha ao excluir os conjuntos de dados selecionados", - "failedToCreateFolder": "Falha ao criar pasta", - "failedToUpdateFolder": "Falha ao atualizar pasta", "failedToDeleteFolder": "Falha ao excluir pasta", - "folderNameExists": "Já existe uma pasta com este nome", - "failedToMoveDataset": "Falha ao mover conjunto de dados", "failedToDeleteNotebook": "Falha ao excluir caderno", "failedToDeleteNotebooks": "Falha ao excluir os cadernos selecionados", "failedToFetchNotebooks": "Falha ao obter cadernos", "failedToLoadDatasetInfo": "Falha ao obter informações do conjunto de dados", + "failedToMoveDataset": "Falha ao mover conjunto de dados", + "failedToResetExplorerResults": "Falha ao restaurar o gráfico original", "failedToUpdateDataset": "Falha ao atualizar conjunto de dados", "failedToUpdateExplorerResults": "Falha ao atualizar resultados do explorador", - "failedToResetExplorerResults": "Falha ao restaurar o gráfico original", + "failedToUpdateFolder": "Falha ao atualizar pasta", "failedToUpdateNotebook": "Falha ao atualizar caderno", + "fetchingConverters": "Erro ao obter os conversores disponíveis.", "fetchingDataloaders": "Erro ao tentar obter dataloaders compatíveis.", "fetchingDatasetColumns": "Erro ao tentar obter as colunas do conjunto de dados.", "fetchingExplorersConverters": "Falha ao obter exploradores/conversores", "fileTypeNotAllowed": "Tipo de arquivo não permitido para o dataloader selecionado.", + "folderNameExists": "Já existe uma pasta com este nome", "loadingDatasetPreview": "Erro ao carregar visualização prévia do conjunto de dados", "noDatasetDataAvailable": "Não há dados do conjunto de dados disponíveis", "noDatasetFileAvailable": "Não há arquivo do conjunto de dados disponível", - "notebookNameEmpty": "O nome do caderno não pode estar vazio", "noValidColumnsForExplorer": "Não há colunas válidas disponíveis para este explorador.", "noValidColumnsWithDtypesMentioned": "Este conjunto de dados não possui colunas com os tipos necessários ({{dtypes}}).", + "notebookNameEmpty": "O nome do caderno não pode estar vazio", "processConverterError": "Falha ao processar conversor", "requiredFieldsMissing": "Campos obrigatórios ausentes", - "requiresExactColumns_one": "Requer exatamente {{required}} coluna válida, mas {{available}} disponível.", "requiresExactColumns_many": "Requer exatamente {{required}} colunas válidas, mas {{available}} disponíveis.", + "requiresExactColumns_one": "Requer exatamente {{required}} coluna válida, mas {{available}} disponível.", "requiresExactColumns_other": "Requer exatamente {{required}} colunas válidas, mas {{available}} disponíveis.", - "requiresMinColumns_one": "Requer pelo menos {{required}} coluna válida, mas apenas {{available}} disponível.", "requiresMinColumns_many": "Requer pelo menos {{required}} colunas válidas, mas apenas {{available}} disponíveis.", + "requiresMinColumns_one": "Requer pelo menos {{required}} coluna válida, mas apenas {{available}} disponível.", "requiresMinColumns_other": "Requer pelo menos {{required}} colunas válidas, mas apenas {{available}} disponíveis.", - "zipContentsNotCompatible": "O arquivo ZIP não contém arquivos compatíveis com o dataloader selecionado", - "cannotSaveEmptyDataset": "Não é possível salvar o dataset: todas as colunas foram removidas." + "zipContentsNotCompatible": "O arquivo ZIP não contém arquivos compatíveis com o dataloader selecionado" }, "label": { - "task": "Tarefa", + "aboutTool": "Sobre {{toolName}}", "all": "todos", + "allTypeChangesValid": "Todas as alterações de tipo são válidas e podem ser aplicadas com segurança.", "allowedDataTypes": "Tipos de dados permitidos: <1><0>", "allowedValueTypes": "Tipos de valores permitidos: <1><0>", - "excludedDataTypes": "Tipos de dados excluídos: {{dtypes}}", - "allTypeChangesValid": "Todas as alterações de tipo são válidas e podem ser aplicadas com segurança.", "analysisTools": "Ferramentas de Análise", "appearance": "Aparência", "appliedTransformations": "Transformações Aplicadas:", "associatedDataset": "Conjunto de Dados Associado", "atLeastTwoColorStopsRequired": "Nenhum ponto de cor definido. Por favor, adicione pelo menos dois pontos de cor.", "availableDatasets": "Conjuntos de Dados Disponíveis", - "confirmDeleteFolder": "Excluir pasta", - "confirmDeleteFolderContent": "Tem certeza que deseja excluir a pasta \"{{name}}\"? Os conjuntos de dados dentro serão movidos para sem pasta.", - "noFolder": "Sem pasta", - "newFolder": "Nova pasta", - "folderName": "Nome da pasta", - "confirmDeleteDataset": "Tem certeza de que deseja excluir o conjunto de dados \"{{name}}\"? Esta ação não pode ser desfeita.", - "confirmDeleteDatasetLinkedWarning": "Todos os cadernos e sessões associados a este conjunto de dados também serão excluídos.", - "confirmBulkDeleteDatasets_one": "Tem certeza de que deseja excluir o conjunto de dados selecionado? Esta ação não pode ser desfeita.", - "confirmBulkDeleteDatasets_many": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", - "confirmBulkDeleteDatasets_other": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", - "selectDatasetsToDelete": "Selecionar conjuntos de dados para excluir", - "confirmDeleteNotebook": "Tem certeza de que deseja excluir o caderno \"{{name}}\"? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks_one": "Tem certeza de que deseja excluir o caderno selecionado? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks_many": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", - "confirmBulkDeleteNotebooks_other": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", - "selectNotebooksToDelete": "Selecionar cadernos para excluir", "avg": "Média", "avgLength": "Comprimento Médio", "avgWordCount": "Contagem Média de Palavras", @@ -160,6 +143,7 @@ "classTargetColumn": "Coluna de Classe/Alvo", "clickToUpload": "Clique para enviar", "color": "Cor", + "colorStops": "Pontos de Cor", "colorbarBorderColor": "Cor da Borda da Barra de Cores", "colorbarBorderWidth": "Largura da Borda da Barra de Cores", "colorbarTickFontColor": "Cor da Fonte dos Números da Barra de Cores", @@ -167,7 +151,7 @@ "colors": "Cores", "colorscale": "Escala de Cores", "colorscaleMode": "Modo da Escala de Cores", - "colorStops": "Pontos de Cor", + "columnInsights": "Observações por Coluna", "columnName": "Nome da Coluna", "columnTypesDistribution": "Distribuição dos Tipos de Coluna", "configureAndUpload": "Configurar e Enviar", @@ -176,10 +160,22 @@ "configureParametersStep": "Etapa {{step}}: Configurar Parâmetros", "configureScope": "Configurar Escopo", "configureToolTitle": "Configurar {{toolType}}: {{toolName}}", - "columnInsights": "Observações por Coluna", + "confirmBulkDeleteDatasets_many": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteDatasets_one": "Tem certeza de que deseja excluir o conjunto de dados selecionado? Esta ação não pode ser desfeita.", + "confirmBulkDeleteDatasets_other": "Tem certeza de que deseja excluir os {{count}} conjuntos de dados selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks_many": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks_one": "Tem certeza de que deseja excluir o caderno selecionado? Esta ação não pode ser desfeita.", + "confirmBulkDeleteNotebooks_other": "Tem certeza de que deseja excluir os {{count}} cadernos selecionados? Esta ação não pode ser desfeita.", + "confirmDeleteDataset": "Tem certeza de que deseja excluir o conjunto de dados \"{{name}}\"? Esta ação não pode ser desfeita.", + "confirmDeleteDatasetLinkedWarning": "Todos os cadernos e sessões associados a este conjunto de dados também serão excluídos.", + "confirmDeleteFolder": "Excluir pasta", + "confirmDeleteFolderContent": "Tem certeza que deseja excluir a pasta \"{{name}}\"? Os conjuntos de dados dentro serão movidos para sem pasta.", + "confirmDeleteNotebook": "Tem certeza de que deseja excluir o caderno \"{{name}}\"? Esta ação não pode ser desfeita.", "constantColumns": "Colunas Constantes", "convert": "Converter", "converter": "Conversor", + "converterOutput": "Saída", "correlation": "Correlação", "correlationAnalysis": "Análise de Correlação", "correlations": "Correlações", @@ -187,46 +183,49 @@ "createNewNotebook": "Caderno", "createNewNotebookDescription": "Inicia uma nova sessão de análise com um conjunto de dados existente.", "customArray": "Array Personalizado", - "dataloaderConfiguration": "Configuração do Dataloader", "dataQuality": "Qualidade dos Dados", "dataQualityScoreTooltip": "A qualidade dos dados é calculada com base na presença de valores ausentes, duplicatas e diversidade de tipos de dados nas colunas.", "dataQualitySummary": "Resumo da Qualidade dos Dados", + "dataType": "Tipo de Dado", + "dataloaderConfiguration": "Configuração do Dataloader", "datasetDescription": "{{rows}} Linhas, {{columns}} Colunas", "datasetLoading": "Carregando conjunto de dados...", "datasetModule": "Módulo de Conjuntos de Dados", "datasetModuleSubtitle": "Envie seus conjuntos de dados: Explore, analise e transforme seus dados com ferramentas avançadas de análise exploratória. Crie cadernos interativos, gere visualizações e aplique transformações de dados de forma intuitiva.", "datasetName": "Nome do Conjunto de Dados", "datasetPreview": "Visualização Prévia do Conjunto de Dados", - "aboutTool": "Sobre {{toolName}}", "datasetPreviewFor": "Caderno: Visualização Prévia de {{name}}", - "dataType": "Tipo de Dado", + "decreasingColor": "Cor de Diminuição", "deleteConverterConfirmation": "Tem certeza de que deseja excluir o conversor \"{{converter}}\"? Excluir este conversor também excluirá todos os conversores e exploradores posteriores aplicados em seguida. Esta ação não pode ser desfeita.", "deleteExplorerConfirmation": "Tem certeza de que deseja excluir o explorador \"{{explorer}}\"? Esta ação não pode ser desfeita.", "detailsForExplorer": "Detalhes do Explorador: {{name}}", "dimensionIdx": "Dimensão {{idx}}: {{label}}", - "dimensionsLabels": "Rótulos de Dimensões", "dimensionTitle": "Título da Dimensão", + "dimensionsLabels": "Rótulos de Dimensões", "distributionMetrics": "Métricas de Distribuição", "dragAndDropFileHere": "Arraste e solte o arquivo aqui", "dragToResize": "Arraste para redimensionar e ver mais conteúdo", + "dropToolHere": "Solte aqui para adicionar", "duplicatedRows": "Linhas Duplicadas", "editPlotLayout": "Editar Layout do Gráfico", "endIndex": "Índice Final", + "excludedDataTypes": "Tipos de dados excluídos: {{dtypes}}", "explorationPath": "Caminho de Exploração", "explorationType": "Tipo de Exploração", "explore": "Explorar", "explorer": "Explorador", + "exportCSV": "Baixar como CSV", "exportCardImage": "Cartão como imagem", "exportChartImage": "Gráfico como imagem", - "exportCSV": "Baixar como CSV", "exportImage": "Baixar", "exportJSON": "Baixar como JSON", "exportMetrics": "Exportar métricas como JSON", "fileSizeMB": "Tamanho do Arquivo (MB)", "firstLastStopsAtExtremes": "Os pontos de cor devem incluir as posições 0 no início e 1 no final.", + "folderName": "Nome da pasta", "fontFamily": "Família de Fonte", - "foundDuplicateRows_one": "{{count}} linha duplicada encontrada no conjunto de dados", "foundDuplicateRows_many": "{{count}} linhas duplicadas encontradas no conjunto de dados", + "foundDuplicateRows_one": "{{count}} linha duplicada encontrada no conjunto de dados", "foundDuplicateRows_other": "{{count}} linhas duplicadas encontradas no conjunto de dados", "fromDataset": "do conjunto de dados {{datasetName}}", "generalSettings": "Configurações Gerais", @@ -235,19 +234,20 @@ "highCardinality": "Alta Cardinalidade", "highCardinalityDetected": "Alta cardinalidade detectada em: {{columns}}", "ifYourDatasetHaveSplits": "Se seu conjunto de dados tiver divisões, envie-o como arquivo zip", + "importFromHub": "Central de Arquivos", + "importFromHubDescription": "Navegue e baixe conjuntos de dados de fontes externas como HuggingFace e OpenML.", + "increasingColor": "Cor de Aumento", "indices": "Índices (separados por vírgula, ou digite 'all')", "inferenceRows": "Linhas de Inferência", "inferenceRowsDescription": "Número de linhas utilizadas para visualização prévia/inferência de tipo (mínimo 2).", - "previewRows": "Linhas de Pré-visualização", - "previewRowsDescription": "Número de linhas a mostrar na pré-visualização (mínimo 2).", "inferredConfiguration": "Configuração de Inferência de Tipo", "insightConstantColumn": "Esta coluna possui apenas um valor único. Não contribui com informação para a análise.", - "insightHighCardinality": "Esta coluna categórica tem mais de 100 valores únicos. Considere agrupar ou codificar.", "insightEmptyMessage": "Nenhum problema detectado. As observações por coluna aparecerão aqui quando forem encontradas possíveis observações nos dados.", + "insightHighCardinality": "Esta coluna categórica tem mais de 100 valores únicos. Considere agrupar ou codificar.", "insightHighNanRatio": "{{value}}% dos valores estão ausentes nesta coluna. Considere imputação ou remoção.", "insightLowUniqueness": "Baixa unicidade ({{value}}%). Pode ser uma variável categórica classificada incorretamente como texto.", - "insightOutliers_one": "{{count}} valor atípico detectado (método IQR).", "insightOutliers_many": "{{count}} valores atípicos detectados (método IQR).", + "insightOutliers_one": "{{count}} valor atípico detectado (método IQR).", "insightOutliers_other": "{{count}} valores atípicos detectados (método IQR).", "insightPossibleId": "Todos os valores são únicos. Esta coluna pode ser um identificador e não é útil para modelagem.", "insightSkewed": "Distribuição assimétrica à direita (assimetria: {{value}}). Considere aplicar uma transformação logarítmica.", @@ -269,19 +269,15 @@ "legendYPosition": "Posição Y da Legenda", "lengthDistribution": "Distribuição de Comprimento", "lengthMetrics": "Métricas de Comprimento", - "lowerBound": "Limite Inferior", + "lineColor": "Cor da Linha", "lowUniquenessWarning": "Aviso: Esta coluna de texto tem uma proporção de unicidade muito baixa. Pode ser uma variável categórica classificada incorretamente como texto, o que pode causar problemas de análise.", + "lowerBound": "Limite Inferior", "marginBottom": "Margem Inferior", "marginLeft": "Margem Esquerda", "marginRight": "Margem Direita", - "margins": "Margens", "marginTop": "Margem Superior", + "margins": "Margens", "markerColor": "Cor do Marcador", - "lineColor": "Cor da Linha", - "increasingColor": "Cor de Aumento", - "decreasingColor": "Cor de Diminuição", - "totalsColor": "Cor dos Totais", - "transparent": "Transparente", "max": "Máximo", "maxLength": "Comprimento Máximo", "mean": "Média", @@ -295,30 +291,28 @@ "missingValuesDetected": "Valores ausentes detectados nas colunas: {{columns}}", "missingValuesOverview": "Resumo de Valores Ausentes", "mostFrequent": "Mais Frequente", - "nameYourNotebook": "Nomeie seu Caderno", "nComponentsColumnInfo": "Você selecionou {{n}} coluna(s). n_components deve ser menor ou igual a {{n}}. Reduza n_components ou adicione mais colunas para evitar erros.", + "nameYourNotebook": "Nomeie seu Caderno", "newDatasetCreatedWithTransformations": "Um novo conjunto de dados será criado com estas transformações. Pode ser usado com outros módulos sem afetar o original.", + "newFolder": "Nova pasta", "noDataQualityIssuesDetected": "Nenhum problema de qualidade de dados detectado", - "noDuplicateRows": "Nenhuma linha duplicada detectada no conjunto de dados.", - "qualityIssuesFound_one": "{{count}} problema detectado", - "qualityIssuesFound_many": "{{count}} problemas detectados", - "qualityIssuesFound_other": "{{count}} problemas detectados", "noDataset": "Sem conjunto de dados", "noDatasetsAvailable": "Nenhum Conjunto de Dados Disponível", + "noDuplicateRows": "Nenhuma linha duplicada detectada no conjunto de dados.", "noExplorersOrConverters": "Comece a explorar adicionando seu primeiro explorador ou conversor!", + "noFolder": "Sem pasta", "noMissingValues": "Nenhum valor ausente detectado no conjunto de dados.", - "none": "nenhum", "noStrongCorrelationsFound": "Nenhuma correlação forte detectada", + "noToolsMatched": "Nenhuma ferramenta encontrada que corresponda à sua busca.", + "noTransformationsApplied": "Nenhuma transformação aplicada.", + "noTransformationsAppliedYet": "Nenhuma transformação foi aplicada ainda.", + "none": "nenhum", "notebookCreationNote": "Uma cópia do conjunto de dados selecionado será criada para trabalhar no caderno sem alterar o original.", "notebookDescription": "Descrição do Caderno", "notebookHistory": "Histórico do Caderno: {{notebook}}", "notebookInformation": "Informações do Caderno", "notebookName": "Nome do Caderno", "notebooks": "Cadernos", - "noToolsMatched": "Nenhuma ferramenta encontrada que corresponda à sua busca.", - "dropToolHere": "Solte aqui para adicionar", - "noTransformationsApplied": "Nenhuma transformação aplicada.", - "noTransformationsAppliedYet": "Nenhuma transformação foi aplicada ainda.", "numericalAnalysis": "Análise Numérica", "outliers": "Valores Atípicos", "overview": "Visão Geral", @@ -326,11 +320,16 @@ "position": "Posição", "possibleIDColumns": "Possíveis Colunas de ID", "presetScale": "Escala Predefinida", - "processingTitle": "Processando seu dataset...", + "previewRows": "Linhas de Pré-visualização", + "previewRowsDescription": "Número de linhas a mostrar na pré-visualização (mínimo 2).", "processingMessage": "Isso pode levar alguns minutos dependendo do tamanho do conjunto de dados.", + "processingTitle": "Processando seu dataset...", "proportion": "Proporção", "q1": "Q1", "q3": "Q3", + "qualityIssuesFound_many": "{{count}} problemas detectados", + "qualityIssuesFound_one": "{{count}} problema detectado", + "qualityIssuesFound_other": "{{count}} problemas detectados", "qualityScore": "Pontuação de Qualidade: {{value}}{{unit}}", "range": "Intervalo", "requiredColumns": "Colunas necessárias", @@ -346,6 +345,7 @@ "saveProcessedDataset": "Salvar Conjunto de Dados Processado", "scopeColumns": "Escopo - Colunas", "scopeRows": "Escopo - Linhas", + "searchConverters": "Buscar conversores", "searchDatasetsNotebooks": "Buscar Conjuntos de Dados e Cadernos", "searchExplorersConverters": "Buscar exploradores/conversores", "selectColorscale": "Selecionar Escala de Cores", @@ -355,36 +355,39 @@ "selectDataset": "Selecionar um conjunto de dados", "selectDatasetFirst": "Selecione primeiro um conjunto de dados", "selectDatasetForNotebook": "Selecione um conjunto de dados para o caderno", - "selectedColumns_one": "{{count}} coluna selecionada", + "selectDatasetsToDelete": "Selecionar conjuntos de dados para excluir", + "selectNotebookToAccessAnalysisTools": "Selecione um caderno para acessar as ferramentas de análise.", + "selectNotebooksToDelete": "Selecionar cadernos para excluir", + "selectScopeDescriptionColumns": "Aqui você configurará em quais colunas aplicar o conversor.", + "selectScopeDescriptionRows": "Aqui você configurará em quais linhas aplicar o conversor.", + "selectScopeStep": "Etapa {{step}}: Selecionar Escopo", + "selectTargetColumnDescription": "Selecione uma coluna para ser usada como variável alvo para aprendizado supervisionado.", + "selectUploadMethod": "Selecione uma forma de enviar seus dados", "selectedColumns_many": "{{count}} colunas selecionadas", + "selectedColumns_one": "{{count}} coluna selecionada", "selectedColumns_other": "{{count}} colunas selecionadas", "selectedDataloaderConfiguration": "Configuração de {{dataloader}}", "selectedDataset": "Conjunto de dados selecionado", "selectedOrder": "Ordem Selecionada", "selectedRows": "Linhas selecionadas: {{value}}", "selectionMode": "Modo de Seleção", - "selectNotebookToAccessAnalysisTools": "Selecione um caderno para acessar as ferramentas de análise.", - "selectScopeDescriptionColumns": "Aqui você configurará em quais colunas aplicar o conversor.", - "selectScopeDescriptionRows": "Aqui você configurará em quais linhas aplicar o conversor.", - "selectScopeStep": "Etapa {{step}}: Selecionar Escopo", - "selectTargetColumnDescription": "Selecione uma coluna para ser usada como variável alvo para aprendizado supervisionado.", - "selectUploadMethod": "Selecione uma forma de enviar seus dados", "shapeIndicators": "Indicadores de Forma", "showGrid": "Mostrar Grade do Eixo {{axis}}", - "showingRowsInference": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas analisadas para inferência de tipo.", "showLegend": "Mostrar Legenda", "showMore": "Mostrar mais ({{count}} restantes)", - "showMore_one": "Mostrar mais ({{count}} restante)", "showMore_many": "Mostrar mais ({{count}} restantes)", + "showMore_one": "Mostrar mais ({{count}} restante)", "showMore_other": "Mostrar mais ({{count}} restantes)", - "showingRowsPreview": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas.", "showZeroLine": "Mostrar Linha Zero do Eixo {{axis}}", + "showingRowsInference": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas analisadas para inferência de tipo.", + "showingRowsPreview": "Mostrando {{sampleLength}} de {{previewRowCount}} linhas.", "skewness": "Assimetria", "someTypeChangesCannotBeApplied": "Algumas alterações de tipo não podem ser aplicadas:", "startIndex": "Índice Inicial", "stdDev": "Desvio Padrão", "strongCorrelations": "Correlações Fortes", "targetColumn": "Coluna Alvo", + "task": "Tarefa", "text": "Texto", "tickLabels": "Rótulos das Marcas", "tickLabelsHelper": "Um rótulo por linha", @@ -392,27 +395,27 @@ "title": "Título", "titleColor": "Cor do Título", "titleFontSize": "Tamanho de Fonte do Título", - "toolsCount_one": "{{count}} ferramenta", "toolsCount_many": "{{count}} ferramentas", + "toolsCount_one": "{{count}} ferramenta", "toolsCount_other": "{{count}} ferramentas", "topValueCount": "Contagem dos Valores Principais", "totalColumns": "Colunas Totais", "totalPercentage": "{{percentage}}% do total", "totalRows": "Linhas Totais", "totalRowsCount": "Linhas totais: {{total}}", + "totalsColor": "Cor dos Totais", "tourDisabledMessage": "Volte ao início para começar o tour", "tourDisabledMessageNotebook": "Volte à visualização do conjunto de dados para começar o tour", "traceIdx": "Rastreamento {{index}}: {{trace}}", + "transparent": "Transparente", "typeChangeWarnings": "Avisos sobre alterações de tipo:", "typeToSearchDatasets": "Digite para buscar conjuntos de dados...", "unique": "Único", - "uniquenessFormula": "Unicidade = (Único ÷ Total) x 100", "uniquePercentage": "{{percentage}}% Único", "uniqueValues": "Valores Únicos", + "uniquenessFormula": "Unicidade = (Único ÷ Total) x 100", "unknownDataset": "Conjunto de dados desconhecido", "uploadAndConfigure": "Envie seu conjunto de dados e configure os parâmetros", - "importFromHub": "Central de Arquivos", - "importFromHubDescription": "Navegue e baixe conjuntos de dados de fontes externas como HuggingFace e OpenML.", "uploadDataset": "Conjunto de Dados", "uploadDatasetBeforeCreatingSession": "Você precisa enviar um conjunto de dados antes de criar uma sessão. Por favor, vá ao Módulo de Conjuntos de Dados para enviar seus dados.", "uploadDatasetDescription": "Importe seus dados de várias fontes e formatos", diff --git a/DashAI/front/src/utils/i18n/locales/pt/experiments.json b/DashAI/front/src/utils/i18n/locales/pt/experiments.json index 5e68b14b8..86ef9313a 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/pt/experiments.json @@ -34,53 +34,57 @@ "label": { "addModelsToExperiment": "Adicionar Modelos ao seu Experimento", "addOptimizersToExperiment": "Adicionar Otimizadores ao seu Experimento", + "cardinalityAtLeast": "pelo menos {{min}}", + "cardinalityBetween": "{{min}} a {{max}}", "columnsInvalidRequirements": "As colunas de Entrada e Saída atuais não correspondem aos requisitos de {{taskName}}", "columnsValidRequirements": "As colunas de Entrada e Saída atuais correspondem aos requisitos de {{taskName}}", "configureExperimentsSubtitle": "Configure experimentos para treinar modelos.", "configureModels": "Configurar modelos", "configureOptimizer": "Configurar otimização de hiperparâmetros", + "crossValidation": "Validação Cruzada", "currentExperiments": "Experimentos Atuais", "currentOptimizerSettings": "Configuração Atual do Otimizador {{optimizer}}", + "cvType": "Tipo de Validação Cruzada", "datasetInputColumnRequirements": "<0>As colunas de entrada devem ser dos tipos<1><2>, e devem ter uma cardinalidade de <3>{{cardinality}}.", "datasetOutputColumnRequirements": "<0>As colunas de saída devem ser dos tipos<1><2>, e devem ter uma cardinalidade de <3>{{cardinality}}.", - "cardinalityAtLeast": "pelo menos {{min}}", - "cardinalityBetween": "{{min}} a {{max}}", "duration": "Duração", "endTime": "Hora de Conclusão", "experimentName": "Nome do Experimento", "experimentsModuleTitle": "Módulo de Experimentos", + "groupColumn": "Coluna para Agrupamento", + "groupColumnDescription": "Coluna a ser usada para o agrupamento. As amostras pertencentes ao mesmo grupo permanecerão juntas na mesma dobra, evitando que um grupo apareça simultaneamente no treinamento e no teste.", + "holdout": "Holdout", "manual": "Manual", "metricToOptimize": "Métrica a Otimizar", "missingValues": "valores ausentes", "missingValuesDetected": "O conjunto de dados contém valores ausentes (NaN) nas colunas:", "modelName": "Nome do Modelo", "modelsInExperiment": "Modelos Atuais no Experimento", + "noConverterAdded": "Nenhum conversor foi adicionado.", "noDatasetsAvailable": "Nenhum Conjunto de Dados Disponível", "noDatasetsAvailableGoToDataTab": "Vá para a <1>aba de dados para enviar um primeiro.", "noModelsAvailable": "Nenhum Modelo Disponível", "noOptimizersNoMetric": "Sem otimização de hiperparâmetros", + "numFolds": "Número de Dobras (k)", + "numFoldsDescription": "Número de dobras da validação cruzada. Mais dobras podem melhorar a estabilidade da avaliação, mas aumentam o tempo de processamento. Deve ser um número inteiro maior que 1.", + "numRepeats": "Número de Repetições", + "numRepeatsDescription": "Número de vezes que o processo de validação cruzada será repetido. Um valor mais alto gera valores mais precisos, mas requer mais tempo de treinamento. Deve ser um número inteiro maior que 1.", "optimizer": "Otimizador", "optimizerMetric": "Métrica de Otimização", "parameterModification": "Modificação de Parâmetros", - "prepareDataset": "Preparar conjunto de dados", "predefined": "Predefinido", + "prepareDataset": "Preparar conjunto de dados", "random": "Aleatório", "recommendPreprocessMissingValues": "Recomenda-se pré-processar o conjunto de dados para tratar esses valores ausentes antes de treinar um modelo.", "rowIndexes": "Índices de linhas", "rowIndexesDescription": "Especifica intervalos de linhas para cada partição usando valores separados por vírgulas ou intervalos (ex.: 0-100, 200).", "seed": "Semente", + "selectAColumn": "Selecione uma coluna", + "selectDataset": "Selecionar conjunto de dados", "selectDatasetColumns": "Indique quais colunas do conjunto de dados serão usadas como entrada e saída.", "selectDatasetTitle": "Selecione um conjunto de dados para a tarefa selecionada", - "selectHowToDivideDataset": "Selecione como dividir o conjunto de dados em subconjuntos de treinamento, validação e teste.", "selectEvaluationStrategy": "Estratégia de avaliação", - "holdout": "Holdout", - "crossValidation": "Validação Cruzada", - "cvType": "Tipo de Validação Cruzada", - "numFolds": "Número de Dobras (k)", - "numFoldsDescription": "Número de dobras da validação cruzada. Mais dobras podem melhorar a estabilidade da avaliação, mas aumentam o tempo de processamento. Deve ser um número inteiro maior que 1.", - "numRepeats": "Número de Repetições", - "numRepeatsDescription": "Número de vezes que o processo de validação cruzada será repetido. Um valor mais alto gera valores mais precisos, mas requer mais tempo de treinamento. Deve ser um número inteiro maior que 1.", - "groupColumn": "Coluna para Agrupamento", + "selectHowToDivideDataset": "Selecione como dividir o conjunto de dados em subconjuntos de treinamento, validação e teste.", "selectInputOutputColumnsDescription": "Selecione nomes de colunas das listas.", "selectMetrics": "Selecionar métricas", "selectModelFirst": "Selecione um Modelo Primeiro", @@ -88,20 +92,17 @@ "setNameAndTask": "Definir nome e tarefa", "shuffle": "Embaralhar", "shuffleDescription": "Determina se os dados serão misturados ao definir os conjuntos ou não. Deve ser verdadeiro para misturar os dados, caso contrário falso.", + "splitType": "Tipo de partição", "splits": "Partições", "splitsDescription": "Proporção de dados atribuída a cada subconjunto. Os valores devem estar entre 0 e 1 e somar 1.", - "splitType": "Tipo de partição", "startTime": "Hora de Início", "stratify": "Estratificar", "stratifyDescription": "Define se os dados serão separados proporcionalmente conforme a distribuição de classes em cada conjunto. Embaralhar deve ser verdadeiro para estratificar os dados.", + "stratifyRequiresShuffle": "Requer que embaralhamento esteja ativado", "useManualSplittingBySpecifyingRowIndexes": "Usar divisão manual especificando os índices de linha de cada subconjunto", "usePredefinedSplitsFromDataset": "Usar divisões predefinidas do conjunto de dados", "usePredefinedSplitsFromDatasetNotAvailable": "Usar divisões predefinidas do conjunto de dados (não disponível)", - "useRandomRowsBySpecifyingPortion": "Usar linhas aleatórias especificando qual porção do conjunto de dados deseja usar para cada subconjunto", - "selectAColumn": "Selecione uma coluna", - "groupColumnDescription": "Coluna a ser usada para o agrupamento. As amostras pertencentes ao mesmo grupo permanecerão juntas na mesma dobra, evitando que um grupo apareça simultaneamente no treinamento e no teste.", - "stratifyRequiresShuffle": "Requer que embaralhamento esteja ativado", - "selectDataset": "Selecionar conjunto de dados" + "useRandomRowsBySpecifyingPortion": "Usar linhas aleatórias especificando qual porção do conjunto de dados deseja usar para cada subconjunto" }, "message": { "confirmDeleteRun": "Tem certeza de que deseja excluir esta execução? Esta ação não pode ser desfeita.", diff --git a/DashAI/front/src/utils/i18n/locales/pt/models.json b/DashAI/front/src/utils/i18n/locales/pt/models.json index 65c474522..4d9d71dbb 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/models.json +++ b/DashAI/front/src/utils/i18n/locales/pt/models.json @@ -1,22 +1,22 @@ { "button": { + "addConverter": "Adicionar Conversor", + "addNewPrediction": "Adicionar Nova Previsão", "createExplainer": "Criar Explicador", "createGlobalExplainer": "Novo Explicador Global", "createLocalExplainer": "Novo Explicador Local", "createPrediction": "Criar Previsão", - "addNewPrediction": "Adicionar Nova Previsão", - "uploadNewDataset": "Enviar Novo Conjunto de Dados", "createSession": "Criar Sessão", "deleteAndRetrain": "Excluir e Retreinar", "deleteRun": "Excluir Execução", "hideOperations": "Ocultar Operações", "hideParameters": "Ocultar Parâmetros", "modelsHub": "Central de Modelos", - "newSession": "Nova Sessão", "modifyParameters": "Modificar Parâmetros", - "newPrediction": "Nova Previsão", "newDatasetPrediction": "Nova Previsão de Conjunto de Dados", "newManualPrediction": "Nova Previsão Manual", + "newPrediction": "Nova Previsão", + "newSession": "Nova Sessão", "retrain": "Retreinar", "runAll": "Executar Tudo", "runAllModels": "Executar Todos os Modelos", @@ -24,11 +24,13 @@ "saveAndRunModel": "Salvar e Executar Modelo", "showOperations": "Mostrar Operações", "showParameters": "Mostrar Parâmetros", - "updateAndRetrain": "Atualizar e Retreinar" + "updateAndRetrain": "Atualizar e Retreinar", + "uploadNewDataset": "Enviar Novo Conjunto de Dados" }, "error": { "completeRequiredFields": "Por favor, preencha todos os campos obrigatórios", "createRun": "Erro ao tentar criar uma nova execução: {{name}}", + "createRunReason": "Erro ao tentar criar uma nova execução \"{{name}}\": {{reason}}", "createSession": "Erro ao tentar criar uma nova sessão", "datasetRequired": "É necessário um conjunto de dados", "enterModelName": "Por favor, insira um nome para o modelo", @@ -70,149 +72,101 @@ }, "label": { "addModelToSession": "Adicionar Modelo à Sessão", + "allRepetitions": "Todas as Repetições", + "alternativeHypothesis": "Hipótese Alternativa", + "applyPreprocessing": "Aplicar pré-processamento", + "applyPreprocessingDescription": "Os conversores selecionados são ajustados apenas com os dados de treinamento e aplicados ao restante, para evitar vazamento de dados. Se não for aplicado, o modelo treina com os dados como estão.", "availableExplainers": "Explicadores Disponíveis", "availableModels": "Modelos Disponíveis", - "noSavedTests": "Ainda não há testes salvos.", - "name": "Nome", - "metric": "Métrica", - "metricSplit": "Divisão", - "significantsCount_one": "{{count}} / {{total}} Significativo", - "significantsCount_many": "{{count}} / {{total}} Significativos", - "significantsCount_other": "{{count}} / {{total}} Significativos", - "bestModel": "Melhor", - "score": "Pontuação", - "scoreProfile": "Perfil de Pontuação", - "scoreHeaderTooltip": "Pontuação ponderada 0–100 conforme o perfil selecionado. As métricas de erro sem limite superior (MAE, RMSE, TER, …) são normalizadas em relação ao pior modelo nesta comparação. Passe o cursor sobre uma célula para ver o detalhamento.", - "profile_balanced": "Equilíbrio", - "profile_detectPositives": "Detectar Positivos", - "profile_avoidFalseAlarms": "Evitar Falsos Alarmes", - "profile_probabilityQuality": "Qualidade de Probabilidades", - "profile_regression_fit": "Ajuste do Modelo", - "profile_regression_error": "Erro Balanceado", - "profile_translation_quality": "Qualidade de Tradução", - "profile_translation_balanced": "Tradução Balanceada", - "profile_text_balanced": "Equilíbrio", - "profile_text_detectPositives": "Detectar Positivos", - "profile_text_avoidFalseAlarms": "Evitar Falsos Alarmes", - "profile_text_probabilityQuality": "Qualidade de Probabilidades", - "optimizer": "Otimizador", - "nestedCrossValidation": "Validação Cruzada Aninhada", - "outerSplitterInherited": "Configuração do divisor inicial da sessão herdada para o loop externo de validação cruzada aninhada", - "innerLoopConfiguration": "Configure a partição de dados para o loop interno de validação cruzada aninhada", - "outerSplitter": "Divisor para o loop externo", - "outerFolds": "Número de folds", - "innerSplitter": "Divisor para o loop interno", - "innerFolds": "Número de folds", - "modelsToCompare": "Modelos a comparar", - "significanceLevel": "Nível de significância", - "noFinishedRuns": "Não há execuções concluídas disponíveis", - "model1": "Modelo 1", - "model2": "Modelo 2", - "nemenyiPairwiseComparisons": "Comparações post-hoc de Nemenyi após o teste de Friedman", - "tukeyPairwiseComparisons": "Comparações post-hoc de Tukey após o teste ANOVA", - "wilcoxonPairwiseComparisons": "Testes de Wilcoxon aos pares com correção de Holmes", - "alternativeHypothesis": "Hipótese Alternativa", - "correctionMethod": "Método de correção", - "result": "Resultado", - "statistic": "Estatística", - "significant": "Significativo", - "notSignificant": "Não Significativo", - "technicalDetails": "Detalhes Técnicos", - "repetition": "Repetição", - "allRepetitions": "Todas as Repetições", "averaged": "Média", - "lines": "Linhas", - "histogramPlot": "Histograma", - "foldNumber": "Número do Fold", - "theoricalQuantiles": "Quantis Teóricos", - "sampleQuantiles": "Quantis da Amostra", - "metricValue": "Valor da Métrica", - "frequency": "Frequência", - "graphs": "Gráficos", - "helperTests": "Testes de Pressupostos", - "selectAtLeastRuns": "Selecione pelo menos {{min}} modelos", - "selectExactlyRuns_one": "Selecione exatamente {{count}} modelo", - "selectExactlyRuns_many": "Selecione exatamente {{count}} modelos", - "selectExactlyRuns_other": "Selecione exatamente {{count}} modelos", - "normalityByRunSummary": "{{normal}} de {{total}} execuções parecem seguir uma distribuição normal (α = {{alpha}}).", - "notNormal": "Não normal", - "normal": "Normal", - "selectBetweenRuns": "Selecione entre {{min}} e {{max}} modelos", - "saveDetails": "Detalhes para salvar", - "testName": "Nome do Teste (opcional)", - "testNameHelp": "Se deixar em branco, será usado o nome do teste.", - "testDescription": "Descrição (opcional)", - "resultSaved": "Salvo", - "saveResult": "Salvar resultado", - "withHpo": "Com HPO", - "withoutHpo": "Sem HPO", - "nestedCv": "Validação Cruzada Aninhada", + "bar": "Barra", + "bestModel": "Melhor", + "chartType": "Tipo de gráfico", "chooseTaskForSessionWithDataset": "Escolha a tarefa de aprendizado de máquina para sua sessão com o conjunto de dados \"{{datasetName}}\".", "configuration": "Configuração", "configureModel": "Configurar Modelo", "configureOptimizer": "Configurar Otimizador", "configureSession": "Configurar Sessão", "configureTasksTrainCompareModels": "Configure tarefas, treine e compare modelos em sessões organizadas. Selecione uma tarefa para começar seu fluxo de trabalho de modelagem.", - "confirmDeleteSession": "Tem certeza de que deseja excluir a sessão \"{{name}}\"? Esta ação não pode ser desfeita.", - "confirmBulkDeleteSessions_one": "Tem certeza de que deseja excluir a sessão selecionada? Esta ação não pode ser desfeita.", + "configureTest": "Configurar Teste", "confirmBulkDeleteSessions_many": "Tem certeza de que deseja excluir as {{count}} sessões selecionadas? Esta ação não pode ser desfeita.", + "confirmBulkDeleteSessions_one": "Tem certeza de que deseja excluir a sessão selecionada? Esta ação não pode ser desfeita.", "confirmBulkDeleteSessions_other": "Tem certeza de que deseja excluir as {{count}} sessões selecionadas? Esta ação não pode ser desfeita.", - "selectSessionsToDelete": "Selecionar sessões para excluir", + "confirmDeleteSession": "Tem certeza de que deseja excluir a sessão \"{{name}}\"? Esta ação não pode ser desfeita.", + "confirmParameterUpdate": "Confirmar Atualização de Parâmetros", + "correctionMethod": "Método de correção", "customMetrics": "Métricas Personalizadas", "datasetPredictions": "Previsões de Conjunto de Dados", + "divideColumnsAndSplits": "Divida colunas e configure as partições do conjunto de dados", "dropModelHere": "Solte aqui para adicionar", "editRun": "Editar execução", - "chartType": "Tipo de gráfico", - "savedTests": "Testes salvos", - "foldGraphs": "Gráficos de Folds", - "nestedCvResults": "Resultados Aninhados", - "outerFoldFetch": "Folds Externos", - "finalFoldFetch": "Folds Finais", - "twoSided": "Ambos os lados (Modelo 1 ≠ Modelo 2)", - "greater": "Maior (Modelo 1 > Modelo 2)", - "less": "Menor (Modelo 1 < Modelo 2)", "editRunParameters": "Editar parâmetros e executar novamente o modelo", "epoch": "Época", "exitModelDetailToAddModels": "Volte para a visão geral da sessão para adicionar mais modelos.", "experimentResults": "Resultados do experimento {{name}}", "explainability": "Explicabilidade", - "explainersCount_one": "• <1>{{count}} explicador", "explainersCount_many": "• <1>{{count}} explicadores", + "explainersCount_one": "• <1>{{count}} explicador", "explainersCount_other": "• <1>{{count}} explicadores", + "finalFoldFetch": "Folds Finais", + "foldGraphs": "Gráficos de Folds", + "foldNumber": "Número do Fold", + "frequency": "Frequência", "generalMetrics": "Métricas Gerais", "globalExplainer": "Explicador Global", "globalExplainers": "Explicadores Globais", "goalMetric": "Métrica Objetivo", + "graphs": "Gráficos", + "greater": "Maior (Modelo 1 > Modelo 2)", "heatmap": "Mapa de calor", + "helperTests": "Testes de Pressupostos", "hideResults": "Ocultar Resultados", "higherIsBetter": "Maior é melhor", + "histogramPlot": "Histograma", "hyperparameterOptimizationPlots": "Gráficos de Otimização de Hiperparâmetros", "hyperparameterOptimizerConfiguration": "Configuração do Otimizador de Hiperparâmetros", "hyperparameters": "Hiperparâmetros", + "innerFolds": "Número de folds", + "innerLoopConfiguration": "Configure a partição de dados para o loop interno de validação cruzada aninhada", + "innerSplitter": "Divisor para o loop interno", "inputColumns": "Colunas de entrada", + "less": "Menor (Modelo 1 < Modelo 2)", + "lines": "Linhas", "liveMetrics": "Métricas ao Vivo", "localExplainer": "Explicador Local", "localExplainers": "Explicadores Locais", "lowerIsBetter": "Menor é melhor", - "manualPredictions": "Previsões Manuais", "manual": "Manuais", + "manualPredictions": "Previsões Manuais", + "metric": "Métrica", + "metricSplit": "Divisão", + "metricToOptimize": "Métrica a Otimizar", + "metricValue": "Valor da Métrica", "metrics": "Métricas", "metricsEmptyForDisplaySet": "As métricas de resultado para {{set}} estão vazias.", - "metricToOptimize": "Métrica a Otimizar", + "minTwoRunsRequired": "Selecione pelo menos 2 execuções concluídas para realizar testes estatísticos", + "model1": "Modelo 1", + "model2": "Modelo 2", "modelComparison": "Comparação de Modelos", "modelConfiguration": "Configuração do Modelo", - "modelCount_one": "{{count}} modelo", "modelCount_many": "{{count}} modelos", + "modelCount_one": "{{count}} modelo", "modelCount_other": "{{count}} modelos", - "bar": "Barra", "modelsModule": "Módulo de Modelos", + "modelsToCompare": "Modelos a comparar", + "name": "Nome", "nameYourSession": "Nomeie sua Sessão", + "nemenyiPairwiseComparisons": "Comparações post-hoc de Nemenyi após o teste de Friedman", + "nestedCrossValidation": "Validação Cruzada Aninhada", + "nestedCv": "Validação Cruzada Aninhada", + "nestedCvResults": "Resultados Aninhados", "noCompatibleExplainersFound": "Nenhum explicador compatível encontrado", "noCompatibleModelsFound": "Nenhum modelo compatível encontrado", "noCompletedRuns": "Ainda não há execuções concluídas", "noConfigurationAvailable": "Nenhuma configuração disponível para esta execução", "noDatasetPredictionsYet": "Ainda não há previsões de conjunto de dados", "noExplainersMatchSearch": "Nenhum explicador corresponde à sua busca", + "noFinishedRuns": "Não há execuções concluídas disponíveis", "noGlobalExplainersYet": "Ainda não há explicadores globais", "noHyperparameterPlotsAvailable": "Não há gráficos de hiperparâmetros disponíveis. Apenas execuções com pelo menos um parâmetro otimizável geram gráficos.", "noLocalExplainersYet": "Ainda não há explicadores locais", @@ -222,19 +176,51 @@ "noModelsMatchSearch": "Nenhum modelo corresponde à sua busca", "noPredictionsYet": "Ainda não há previsões", "noRunsYet": "Ainda não há execuções. Adicione modelos pelo painel direito.", + "noSavedTests": "Ainda não há testes salvos.", "noSessionSelected": "Nenhuma Sessão Selecionada", + "noTestsMatch": "Nenhum teste coincide com sua pesquisa", + "nonParametricTests": "Testes Não Paramétricos", + "normal": "Normal", + "normalityByRunSummary": "{{normal}} de {{total}} execuções parecem seguir uma distribuição normal (α = {{alpha}}).", + "notNormal": "Não normal", + "notSignificant": "Não Significativo", "operations": "Operações", "operationsWillBeDeletedWarning": "Estas operações serão excluídas permanentemente e não podem ser recuperadas. Tem certeza de que deseja continuar?", + "optimizer": "Otimizador", "optimizerConfiguration": "Configure as definições do otimizador de hiperparâmetros", "optimizerParameters": "Parâmetros do Otimizador", + "outerFoldFetch": "Folds Externos", + "outerFolds": "Número de folds", + "outerSplitter": "Divisor para o loop externo", + "outerSplitterInherited": "Configuração do divisor inicial da sessão herdada para o loop externo de validação cruzada aninhada", "outputColumns": "Colunas de saída", + "parametricTests": "Testes Paramétricos", "pleaseSelectMetricToOptimize": "Por favor, selecione uma métrica para otimizar.", "predictions": "Previsões", - "predictionsCount_one": "• <1>{{count}} previsão", "predictionsCount_many": "• <1>{{count}} previsões", + "predictionsCount_one": "• <1>{{count}} previsão", "predictionsCount_other": "• <1>{{count}} previsões", - "divideColumnsAndSplits": "Divida colunas e configure as partições do conjunto de dados", "prepareDataset": "Preparar Conjunto de Dados", + "preprocessingFailed": "O pré-processamento falhou", + "preprocessingInProgress": "Processando...", + "preprocessingOptional": "Pré-processamento (opcional)", + "preprocessingOptionalDescription": "Os conversores selecionados são ajustados apenas com os dados de treinamento e aplicados ao restante, para evitar vazamento de dados.", + "profile_avoidFalseAlarms": "Evitar Falsos Alarmes", + "profile_balanced": "Equilíbrio", + "profile_detectPositives": "Detectar Positivos", + "profile_probabilityQuality": "Qualidade de Probabilidades", + "profile_regression_error": "Erro Balanceado", + "profile_regression_fit": "Ajuste do Modelo", + "profile_text_avoidFalseAlarms": "Evitar Falsos Alarmes", + "profile_text_balanced": "Equilíbrio", + "profile_text_detectPositives": "Detectar Positivos", + "profile_text_probabilityQuality": "Qualidade de Probabilidades", + "profile_translation_balanced": "Tradução Balanceada", + "profile_translation_quality": "Qualidade de Tradução", + "repetition": "Repetição", + "reports": "Relatórios", + "result": "Resultado", + "resultSaved": "Salvo", "retrainConfirmDetails": "Tem certeza de que deseja retreinar a execução \"<1>{{runName}}\"?", "retrainModel": "Retreinar Modelo?", "retrainWillDeleteOperations": "Esta execução possui operações existentes que serão excluídas", @@ -244,48 +230,72 @@ "runFailedNoHyperparameterPlots": "Execução Falhou. Não há gráficos de hiperparâmetros disponíveis.", "runInProgressCannotEdit": "A execução está em andamento e não pode ser editada.", "runName": "Nome da Execução", - "confirmParameterUpdate": "Confirmar Atualização de Parâmetros", "runNotFound": "Execução não encontrada", "runNotStartedNoHyperparameterPlots": "Execução Não Iniciada. Não há gráficos de hiperparâmetros disponíveis.", + "runsSelected": "execuções selecionadas", + "sampleQuantiles": "Quantis da Amostra", "saveConfirmDetails": "Salvar \"<1>{{runName}}\" redefinirá seu status para 'Não Iniciado' e apagará suas métricas e resultados atuais. Tem certeza de que deseja continuar?", + "saveDetails": "Detalhes para salvar", "saveParameterChanges": "Salvar Alterações de Parâmetros?", + "saveResult": "Salvar resultado", "saveWillDeleteOperationsDetails": "Salvar \"<1>{{runName}}\" redefinirá a execução. O seguinte será excluído ao retreinar:", + "savedTests": "Testes salvos", + "score": "Pontuação", + "scoreHeaderTooltip": "Pontuação ponderada 0–100 conforme o perfil selecionado. As métricas de erro sem limite superior (MAE, RMSE, TER, …) são normalizadas em relação ao pior modelo nesta comparação. Passe o cursor sobre uma célula para ver o detalhamento.", + "scoreProfile": "Perfil de Pontuação", "searchDatasetsSessions": "Buscar Conjuntos de Dados e Sessões", "searchMetric": "Buscar métrica...", "searchModels": "Buscar Modelos...", "searchTests": "Buscar Testes...", - "statisticalTests": "Testes Estatísticos", - "parametricTests": "Testes Paramétricos", - "nonParametricTests": "Testes Não Paramétricos", - "minTwoRunsRequired": "Selecione pelo menos 2 execuções concluídas para realizar testes estatísticos", - "noTestsMatch": "Nenhum teste coincide com sua pesquisa", - "runsSelected": "execuções selecionadas", - "configureTest": "Configurar Teste", + "selectAtLeastRuns": "Selecione pelo menos {{min}} modelos", + "selectBetweenRuns": "Selecione entre {{min}} e {{max}} modelos", + "selectColumnsDescription": "Escolha quais colunas são de entrada e qual é a de saída para esta sessão.", + "selectColumnsTitle": "Selecionar Colunas", "selectDataset": "Selecionar Conjunto de Dados", - "selectDatasetForSession": "Selecione um conjunto de dados para sua sessão", "selectDatasetAndPrepare": "Nomeie sua sessão, selecione um conjunto de dados e configure suas colunas e partições.", "selectDatasetFirst": "Selecione um conjunto de dados para configurar suas colunas e partições de linhas.", + "selectDatasetForSession": "Selecione um conjunto de dados para sua sessão", + "selectExactlyRuns_many": "Selecione exatamente {{count}} modelos", + "selectExactlyRuns_one": "Selecione exatamente {{count}} modelo", + "selectExactlyRuns_other": "Selecione exatamente {{count}} modelos", "selectSessionToViewModels": "Selecione uma sessão para ver os modelos disponíveis.", + "selectSessionsToDelete": "Selecionar sessões para excluir", "selectTask": "Selecionar uma Tarefa", "selectTaskForSession": "Selecione uma Tarefa para sua Sessão", "sessionConfiguration": "Configuração da Sessão", "sessionName": "Nome da Sessão", "showResults": "Mostrar Resultados", + "significanceLevel": "Nível de significância", + "significant": "Significativo", + "significantsCount_many": "{{count}} / {{total}} Significativos", + "significantsCount_one": "{{count}} / {{total}} Significativo", + "significantsCount_other": "{{count}} / {{total}} Significativos", + "statistic": "Estatística", + "statisticalTests": "Testes Estatísticos", "step": "Etapa", + "technicalDetails": "Detalhes Técnicos", "test": "Teste", + "testDescription": "Descrição (opcional)", "testMetrics": "Métricas de Teste", + "testName": "Nome do Teste (opcional)", + "testNameHelp": "Se deixar em branco, será usado o nome do teste.", "testSet": "conjunto de teste", + "theoricalQuantiles": "Quantis Teóricos", "thereAreNoMetricsForThisRun": "não há métricas associadas ao {{set}} nesta execução", "tourDisabledMessage": "Volte ao início para começar o tour", "train": "Treinamento", - "trainingMetrics": "Métricas de Treinamento", "trainSet": "conjunto de treinamento", + "trainingMetrics": "Métricas de Treinamento", "trial": "Tentativa", + "tukeyPairwiseComparisons": "Comparações post-hoc de Tukey após o teste ANOVA", + "twoSided": "Ambos os lados (Modelo 1 ≠ Modelo 2)", "validation": "Validação", "validationMetrics": "Métricas de Validação", "validationSet": "conjunto de validação", "viewResultsAs": "Ver resultados como colunas ou gráficos", - "reports": "Relatórios" + "wilcoxonPairwiseComparisons": "Testes de Wilcoxon aos pares com correção de Holmes", + "withHpo": "Com HPO", + "withoutHpo": "Sem HPO" }, "message": { "allRunsCompleted": "{{experiment}} concluiu todas as suas execuções.", diff --git a/DashAI/front/src/utils/i18n/locales/zh/datasets.json b/DashAI/front/src/utils/i18n/locales/zh/datasets.json index b2c4a291d..b84dba860 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/datasets.json +++ b/DashAI/front/src/utils/i18n/locales/zh/datasets.json @@ -51,6 +51,7 @@ "datasetHub": "数据文件中心" }, "error": { + "cannotSaveEmptyDataset": "无法保存数据集：所有列已被删除。", "converterFailed": "处理转换器时发生错误。", "converterFailedWithInfo": "处理转换器时出错：{{error}}", "createConverterError": "创建转换器失败", @@ -67,68 +68,52 @@ "failedToCreateDataset": "创建数据集时出错：{{error}}", "failedToCreateDatasetFromNotebook": "从笔记本创建数据集失败", "failedToCreateExplorer": "创建探索器失败", + "failedToCreateFolder": "创建文件夹失败", "failedToDeleteDataset": "删除数据集失败", "failedToDeleteDatasets": "删除所选数据集失败", - "failedToCreateFolder": "创建文件夹失败", - "failedToUpdateFolder": "更新文件夹失败", "failedToDeleteFolder": "删除文件夹失败", - "folderNameExists": "已存在同名文件夹", - "failedToMoveDataset": "移动数据集失败", "failedToDeleteNotebook": "删除笔记本失败", "failedToDeleteNotebooks": "删除所选笔记本失败", "failedToFetchNotebooks": "获取笔记本列表失败", "failedToLoadDatasetInfo": "获取数据集信息失败", + "failedToMoveDataset": "移动数据集失败", + "failedToResetExplorerResults": "恢复原始图表失败", "failedToUpdateDataset": "更新数据集失败", "failedToUpdateExplorerResults": "更新探索器结果失败", - "failedToResetExplorerResults": "恢复原始图表失败", + "failedToUpdateFolder": "更新文件夹失败", "failedToUpdateNotebook": "更新笔记本失败", + "fetchingConverters": "获取可用转换器时出错。", "fetchingDataloaders": "获取兼容数据加载器时出错。", "fetchingDatasetColumns": "获取数据集列时出错。", "fetchingExplorersConverters": "获取探索器/转换器失败", "fileTypeNotAllowed": "所选数据加载器不允许此文件类型。", + "folderNameExists": "已存在同名文件夹", "loadingDatasetPreview": "加载数据集预览时出错", "noDatasetDataAvailable": "无可用数据集数据", "noDatasetFileAvailable": "无可用数据集文件", - "notebookNameEmpty": "笔记本名称不能为空", "noValidColumnsForExplorer": "此探索器没有有效的可用列。", "noValidColumnsWithDtypesMentioned": "此数据集没有所需类型（{{dtypes}}）的列。", + "notebookNameEmpty": "笔记本名称不能为空", "processConverterError": "处理转换器失败", "requiredFieldsMissing": "必填字段缺失", "requiresExactColumns_one": "需要恰好 {{required}} 个有效列，但只有 {{available}} 个可用。", "requiresExactColumns_other": "需要恰好 {{required}} 个有效列，但只有 {{available}} 个可用。", "requiresMinColumns_one": "至少需要 {{required}} 个有效列，但只有 {{available}} 个可用。", "requiresMinColumns_other": "至少需要 {{required}} 个有效列，但只有 {{available}} 个可用。", - "zipContentsNotCompatible": "ZIP 文件不包含与所选数据加载器兼容的文件", - "cannotSaveEmptyDataset": "无法保存数据集：所有列已被删除。" + "zipContentsNotCompatible": "ZIP 文件不包含与所选数据加载器兼容的文件" }, "label": { - "task": "任务", + "aboutTool": "关于 {{toolName}}", "all": "全部", + "allTypeChangesValid": "所有类型更改均有效，可以安全应用。", "allowedDataTypes": "允许的数据类型：<1><0>", "allowedValueTypes": "允许的值类型：<1><0>", - "excludedDataTypes": "排除的数据类型：{{dtypes}}", - "allTypeChangesValid": "所有类型更改均有效，可以安全应用。", "analysisTools": "分析工具", "appearance": "外观", "appliedTransformations": "已应用的变换：", "associatedDataset": "关联数据集", "atLeastTwoColorStopsRequired": "未定义色标点。请至少添加两个色标点。", "availableDatasets": "可用数据集", - "confirmDeleteFolder": "删除文件夹", - "confirmDeleteFolderContent": "确定要删除文件夹 \"{{name}}\" 吗？其中的数据集将移至无文件夹。", - "noFolder": "无文件夹", - "newFolder": "新建文件夹", - "folderName": "文件夹名称", - "confirmDeleteDataset": "确定要删除数据集 \"{{name}}\" 吗？此操作无法撤销。", - "confirmDeleteDatasetLinkedWarning": "所有与此数据集关联的笔记本和会话也将被删除。", - "confirmBulkDeleteDatasets_one": "确定要删除所选的数据集吗？此操作无法撤销。", - "confirmBulkDeleteDatasets_other": "确定要删除所选的 {{count}} 个数据集吗？此操作无法撤销。", - "selectDatasetsToDelete": "选择要删除的数据集", - "confirmDeleteNotebook": "确定要删除笔记本 \"{{name}}\" 吗？此操作无法撤销。", - "confirmBulkDeleteNotebooks": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", - "confirmBulkDeleteNotebooks_one": "确定要删除所选的笔记本吗？此操作无法撤销。", - "confirmBulkDeleteNotebooks_other": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", - "selectNotebooksToDelete": "选择要删除的笔记本", "avg": "平均值", "avgLength": "平均长度", "avgWordCount": "平均词数", @@ -153,6 +138,7 @@ "classTargetColumn": "类别/目标列", "clickToUpload": "点击上传", "color": "颜色", + "colorStops": "色标点", "colorbarBorderColor": "色标边框颜色", "colorbarBorderWidth": "色标边框宽度", "colorbarTickFontColor": "色标刻度字体颜色", @@ -160,7 +146,7 @@ "colors": "颜色", "colorscale": "色阶", "colorscaleMode": "色阶模式", - "colorStops": "色标点", + "columnInsights": "列分析", "columnName": "列名", "columnTypesDistribution": "列类型分布", "configureAndUpload": "配置并上传", @@ -169,10 +155,20 @@ "configureParametersStep": "步骤 {{step}}：配置参数", "configureScope": "配置范围", "configureToolTitle": "配置 {{toolType}}：{{toolName}}", - "columnInsights": "列分析", + "confirmBulkDeleteDatasets_one": "确定要删除所选的数据集吗？此操作无法撤销。", + "confirmBulkDeleteDatasets_other": "确定要删除所选的 {{count}} 个数据集吗？此操作无法撤销。", + "confirmBulkDeleteNotebooks": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", + "confirmBulkDeleteNotebooks_one": "确定要删除所选的笔记本吗？此操作无法撤销。", + "confirmBulkDeleteNotebooks_other": "确定要删除所选的 {{count}} 个笔记本吗？此操作无法撤销。", + "confirmDeleteDataset": "确定要删除数据集 \"{{name}}\" 吗？此操作无法撤销。", + "confirmDeleteDatasetLinkedWarning": "所有与此数据集关联的笔记本和会话也将被删除。", + "confirmDeleteFolder": "删除文件夹", + "confirmDeleteFolderContent": "确定要删除文件夹 \"{{name}}\" 吗？其中的数据集将移至无文件夹。", + "confirmDeleteNotebook": "确定要删除笔记本 \"{{name}}\" 吗？此操作无法撤销。", "constantColumns": "常量列", "convert": "转换", "converter": "转换器", + "converterOutput": "输出", "correlation": "相关性", "correlationAnalysis": "相关性分析", "correlations": "相关性", @@ -180,43 +176,46 @@ "createNewNotebook": "笔记本", "createNewNotebookDescription": "使用现有数据集开始新的分析会话。", "customArray": "自定义数组", - "dataloaderConfiguration": "数据加载器配置", "dataQuality": "数据质量", "dataQualityScoreTooltip": "数据质量评分基于多种因素计算，包括缺失值、重复行和数据一致性。评分越高表示数据质量越好。", "dataQualitySummary": "数据质量摘要", + "dataType": "数据类型", + "dataloaderConfiguration": "数据加载器配置", "datasetDescription": "{{rows}} 行，{{columns}} 列", "datasetLoading": "加载数据集中...", "datasetModule": "数据集模块", "datasetModuleSubtitle": "上传数据集：使用先进的探索性分析工具探索、分析和变换您的数据。创建交互式笔记本，生成可视化图表，并直观地应用数据变换。", "datasetName": "数据集名称", "datasetPreview": "数据集预览", - "aboutTool": "关于 {{toolName}}", "datasetPreviewFor": "笔记本：{{name}} 预览", - "dataType": "数据类型", + "decreasingColor": "下降颜色", "deleteConverterConfirmation": "确定要删除转换器 \"{{converter}}\" 吗？删除此转换器也会移除之后应用的所有转换器和探索器。此操作无法撤销。", "deleteExplorerConfirmation": "确定要删除探索器 \"{{explorer}}\" 吗？此操作无法撤销。", "detailsForExplorer": "探索器详情：{{name}}", "dimensionIdx": "维度 {{idx}}：{{label}}", - "dimensionsLabels": "维度标签", "dimensionTitle": "维度标题", + "dimensionsLabels": "维度标签", "distributionMetrics": "分布指标", "dragAndDropFileHere": "拖放文件到此处", "dragToResize": "拖动以调整大小并查看更多内容", + "dropToolHere": "拖放此处以添加", "duplicatedRows": "重复行", "editPlotLayout": "编辑图表布局", "endIndex": "结束索引", + "excludedDataTypes": "排除的数据类型：{{dtypes}}", "explorationPath": "探索路径", "explorationType": "探索类型", "explore": "探索", "explorer": "探索器", + "exportCSV": "下载为 CSV", "exportCardImage": "卡片导出为图像", "exportChartImage": "图表导出为图像", - "exportCSV": "下载为 CSV", "exportImage": "下载", "exportJSON": "下载为 JSON", "exportMetrics": "将指标导出为 JSON", "fileSizeMB": "文件大小（MB）", "firstLastStopsAtExtremes": "色标点必须包含起点位置 0 和终点位置 1。", + "folderName": "文件夹名称", "fontFamily": "字体", "foundDuplicateRows_one": "在数据集中发现 {{count}} 行重复数据", "foundDuplicateRows_other": "在数据集中发现 {{count}} 行重复数据", @@ -227,19 +226,20 @@ "highCardinality": "高基数", "highCardinalityDetected": "以下列中检测到高基数：{{columns}}", "ifYourDatasetHaveSplits": "如果您的数据集有划分，请以 zip 文件格式上传", + "importFromHub": "数据文件中心", + "importFromHubDescription": "从 HuggingFace 和 OpenML 等外部来源浏览和下载数据集。", + "increasingColor": "上升颜色", "indices": "索引（逗号分隔，或输入 'all'）", "inferenceRows": "推断行数", "inferenceRowsDescription": "用于预览/类型推断的行数（最少 2 行）。", - "previewRows": "预览行数", - "previewRowsDescription": "预览中显示的行数（最少 2 行）。", "inferredConfiguration": "类型推断配置", "insightConstantColumn": "此列只有一个唯一值，不提供任何分析信息。", - "insightHighCardinality": "此分类列有超过 100 个唯一值。考虑分组或编码。", "insightEmptyMessage": "未检测到问题。当发现潜在观察时，列分析将显示在此处。", + "insightHighCardinality": "此分类列有超过 100 个唯一值。考虑分组或编码。", "insightHighNanRatio": "此列 {{value}}% 的值缺失。考虑插补或删除。", "insightLowUniqueness": "唯一值比例较低（{{value}}%）。可能是被错误分类为文本的分类变量。", - "insightOutliers_one": "检测到 {{count}} 个潜在异常值（IQR 方法）。", "insightOutliers_many": "检测到 {{count}} 个潜在异常值（IQR 方法）。", + "insightOutliers_one": "检测到 {{count}} 个潜在异常值（IQR 方法）。", "insightOutliers_other": "检测到 {{count}} 个潜在异常值（IQR 方法）。", "insightPossibleId": "所有值均唯一。此列可能是标识符，对建模无用。", "insightSkewed": "右偏分布（偏度：{{value}}）。考虑应用对数变换。", @@ -261,19 +261,15 @@ "legendYPosition": "图例 Y 位置", "lengthDistribution": "长度分布", "lengthMetrics": "长度指标", - "lowerBound": "下界", + "lineColor": "线条颜色", "lowUniquenessWarning": "警告：此文本列的唯一值比例非常低。这可能是被错误分类为文本的分类变量，可能导致分析问题。", + "lowerBound": "下界", "marginBottom": "底部边距", "marginLeft": "左侧边距", "marginRight": "右侧边距", - "margins": "边距", "marginTop": "顶部边距", + "margins": "边距", "markerColor": "标记颜色", - "lineColor": "线条颜色", - "increasingColor": "上升颜色", - "decreasingColor": "下降颜色", - "totalsColor": "总计颜色", - "transparent": "透明", "max": "最大值", "maxLength": "最大长度", "mean": "均值", @@ -287,29 +283,28 @@ "missingValuesDetected": "以下列中检测到缺失值：{{columns}}", "missingValuesOverview": "缺失值概览", "mostFrequent": "最频繁", - "nameYourNotebook": "为笔记本命名", "nComponentsColumnInfo": "您选择了 {{n}} 列。n_components 必须小于或等于 {{n}}。请减少 n_components 或选择更多列以避免错误。", + "nameYourNotebook": "为笔记本命名", "newDatasetCreatedWithTransformations": "将使用这些变换创建新数据集。可在其他模块中使用，不影响原始数据。", + "newFolder": "新建文件夹", "noDataQualityIssuesDetected": "未检测到数据质量问题", - "noDuplicateRows": "数据集中未检测到重复行。", - "qualityIssuesFound_one": "检测到 {{count}} 个问题", - "qualityIssuesFound_other": "检测到 {{count}} 个问题", "noDataset": "无数据集", "noDatasetsAvailable": "无可用数据集", + "noDuplicateRows": "数据集中未检测到重复行。", "noExplorersOrConverters": "添加您的第一个探索器或转换器开始探索！", + "noFolder": "无文件夹", "noMissingValues": "数据集中未检测到缺失值。", - "none": "无", "noStrongCorrelationsFound": "未检测到强相关性", + "noToolsMatched": "未找到匹配搜索的工具。", + "noTransformationsApplied": "未应用任何变换。", + "noTransformationsAppliedYet": "尚未应用任何变换。", + "none": "无", "notebookCreationNote": "将创建所选数据集的副本，以便在笔记本中工作而不改变原始数据。", "notebookDescription": "笔记本描述", "notebookHistory": "笔记本历史：{{notebook}}", "notebookInformation": "笔记本信息", "notebookName": "笔记本名称", "notebooks": "笔记本", - "noToolsMatched": "未找到匹配搜索的工具。", - "dropToolHere": "拖放此处以添加", - "noTransformationsApplied": "未应用任何变换。", - "noTransformationsAppliedYet": "尚未应用任何变换。", "numericalAnalysis": "数值分析", "outliers": "异常值", "overview": "概览", @@ -317,11 +312,15 @@ "position": "位置", "possibleIDColumns": "可能的 ID 列", "presetScale": "预设色阶", - "processingTitle": "正在处理您的数据集...", + "previewRows": "预览行数", + "previewRowsDescription": "预览中显示的行数（最少 2 行）。", "processingMessage": "根据数据大小，这可能需要几分钟。", + "processingTitle": "正在处理您的数据集...", "proportion": "比例", "q1": "Q1", "q3": "Q3", + "qualityIssuesFound_one": "检测到 {{count}} 个问题", + "qualityIssuesFound_other": "检测到 {{count}} 个问题", "qualityScore": "质量评分：{{value}}{{unit}}", "range": "范围", "requiredColumns": "必需列", @@ -337,6 +336,7 @@ "saveProcessedDataset": "保存处理后的数据集", "scopeColumns": "范围 - 列", "scopeRows": "范围 - 行", + "searchConverters": "搜索转换器", "searchDatasetsNotebooks": "搜索数据集和笔记本", "searchExplorersConverters": "搜索探索器/转换器", "selectColorscale": "选择色阶", @@ -346,6 +346,14 @@ "selectDataset": "选择数据集", "selectDatasetFirst": "请先选择数据集", "selectDatasetForNotebook": "为笔记本选择数据集", + "selectDatasetsToDelete": "选择要删除的数据集", + "selectNotebookToAccessAnalysisTools": "选择笔记本以访问分析工具。", + "selectNotebooksToDelete": "选择要删除的笔记本", + "selectScopeDescriptionColumns": "在此配置转换器要应用到哪些列。", + "selectScopeDescriptionRows": "在此配置转换器要应用到哪些行。", + "selectScopeStep": "步骤 {{step}}：选择范围", + "selectTargetColumnDescription": "选择一列作为监督学习的目标变量。", + "selectUploadMethod": "选择上传数据的方式", "selectedColumns_one": "已选 {{count}} 列", "selectedColumns_other": "已选 {{count}} 列", "selectedDataloaderConfiguration": "{{dataloader}} 配置", @@ -353,26 +361,21 @@ "selectedOrder": "选择顺序", "selectedRows": "已选行：{{value}}", "selectionMode": "选择模式", - "selectNotebookToAccessAnalysisTools": "选择笔记本以访问分析工具。", - "selectScopeDescriptionColumns": "在此配置转换器要应用到哪些列。", - "selectScopeDescriptionRows": "在此配置转换器要应用到哪些行。", - "selectScopeStep": "步骤 {{step}}：选择范围", - "selectTargetColumnDescription": "选择一列作为监督学习的目标变量。", - "selectUploadMethod": "选择上传数据的方式", "shapeIndicators": "形状指示器", "showGrid": "{{axis}} 轴显示网格", - "showingRowsInference": "显示用于类型推断的 {{previewRowCount}} 行中的 {{sampleLength}} 行。", - "showingRowsPreview": "显示 {{previewRowCount}} 行中的 {{sampleLength}} 行。", "showLegend": "显示图例", "showMore": "显示更多（剩余 {{count}} 个）", "showMore_other": "显示更多（剩余 {{count}} 个）", "showZeroLine": "{{axis}} 轴显示零线", + "showingRowsInference": "显示用于类型推断的 {{previewRowCount}} 行中的 {{sampleLength}} 行。", + "showingRowsPreview": "显示 {{previewRowCount}} 行中的 {{sampleLength}} 行。", "skewness": "偏度", "someTypeChangesCannotBeApplied": "某些类型更改无法应用：", "startIndex": "起始索引", "stdDev": "标准差", "strongCorrelations": "强相关性", "targetColumn": "目标列", + "task": "任务", "text": "文本", "tickLabels": "刻度标签", "tickLabelsHelper": "每行一个标签", @@ -387,19 +390,19 @@ "totalPercentage": "占总量的 {{percentage}}%", "totalRows": "总行数", "totalRowsCount": "总行数：{{total}}", + "totalsColor": "总计颜色", "tourDisabledMessage": "返回首页以开始引导", "tourDisabledMessageNotebook": "返回数据集可视化以开始引导", "traceIdx": "轨迹 {{index}}：{{trace}}", + "transparent": "透明", "typeChangeWarnings": "类型更改警告：", "typeToSearchDatasets": "输入以搜索数据集...", "unique": "唯一", - "uniquenessFormula": "唯一性 = （唯一值 ÷ 总数）× 100", "uniquePercentage": "{{percentage}}% 唯一", "uniqueValues": "唯一值", + "uniquenessFormula": "唯一性 = （唯一值 ÷ 总数）× 100", "unknownDataset": "未知数据集", "uploadAndConfigure": "上传数据集并配置参数", - "importFromHub": "数据文件中心", - "importFromHubDescription": "从 HuggingFace 和 OpenML 等外部来源浏览和下载数据集。", "uploadDataset": "数据集", "uploadDatasetBeforeCreatingSession": "创建会话前需要上传数据集。请前往数据集模块上传您的数据。", "uploadDatasetDescription": "从各种来源和格式导入您的数据", diff --git a/DashAI/front/src/utils/i18n/locales/zh/experiments.json b/DashAI/front/src/utils/i18n/locales/zh/experiments.json index e1c6aa541..26091f645 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/zh/experiments.json @@ -34,53 +34,57 @@ "label": { "addModelsToExperiment": "向实验中添加模型", "addOptimizersToExperiment": "向实验中添加优化器", + "cardinalityAtLeast": "至少 {{min}}", + "cardinalityBetween": "{{min}} 到 {{max}}", "columnsInvalidRequirements": "当前输入列和输出列不满足 {{taskName}} 的要求", "columnsValidRequirements": "当前输入列和输出列满足 {{taskName}} 的要求", "configureExperimentsSubtitle": "配置实验以训练模型。", "configureModels": "配置模型", "configureOptimizer": "配置超参数优化", + "crossValidation": "交叉验证", "currentExperiments": "当前实验", "currentOptimizerSettings": "当前优化器设置 {{optimizer}}", + "cvType": "交叉验证类型", "datasetInputColumnRequirements": "<0>输入列的类型必须为<1><2>，且基数应为<3>{{cardinality}}。", "datasetOutputColumnRequirements": "<0>输出列的类型必须为<1><2>，且基数应为<3>{{cardinality}}。", - "cardinalityAtLeast": "至少 {{min}}", - "cardinalityBetween": "{{min}} 到 {{max}}", "duration": "时长", "endTime": "结束时间", "experimentName": "实验名称", "experimentsModuleTitle": "实验模块", + "groupColumn": "分组列", + "groupColumnDescription": "用于分组的列。属于同一组的样本将保持在同一折中，避免同一组同时出现在训练集和测试集中。", + "holdout": "留出法", "manual": "手动", "metricToOptimize": "待优化指标", "missingValues": "缺失值", "missingValuesDetected": "数据集在以下列中包含缺失值（NaN）：", "modelName": "模型名称", "modelsInExperiment": "实验中的当前模型", + "noConverterAdded": "尚未添加任何转换器。", "noDatasetsAvailable": "暂无可用数据集", "noDatasetsAvailableGoToDataTab": "请前往<1>数据标签页先上传数据集。", "noModelsAvailable": "暂无可用模型", "noOptimizersNoMetric": "不进行超参数优化", + "numFolds": "折数（k）", + "numFoldsDescription": "交叉验证的折数。折数越多可以提高评估的稳定性，但会增加处理时间。必须为大于 1 的整数。", + "numRepeats": "重复次数", + "numRepeatsDescription": "交叉验证过程重复的次数。数值越高结果越精确，但需要更多训练时间。必须为大于 1 的整数。", "optimizer": "优化器", "optimizerMetric": "优化指标", "parameterModification": "参数修改", - "prepareDataset": "准备数据集", "predefined": "预定义", + "prepareDataset": "准备数据集", "random": "随机", "recommendPreprocessMissingValues": "建议在训练模型前对数据集进行预处理以处理这些缺失值。", "rowIndexes": "行索引", "rowIndexesDescription": "使用逗号分隔的值或范围（如 0-100, 200）为每个划分指定行范围。", "seed": "随机种子", + "selectAColumn": "选择一列", + "selectDataset": "选择数据集", "selectDatasetColumns": "指定数据集中用作输入和输出的列。", "selectDatasetTitle": "为所选任务选择数据集", - "selectHowToDivideDataset": "选择如何将数据集划分为训练集、验证集和测试集。", "selectEvaluationStrategy": "评估策略", - "holdout": "留出法", - "crossValidation": "交叉验证", - "cvType": "交叉验证类型", - "numFolds": "折数（k）", - "numFoldsDescription": "交叉验证的折数。折数越多可以提高评估的稳定性，但会增加处理时间。必须为大于 1 的整数。", - "numRepeats": "重复次数", - "numRepeatsDescription": "交叉验证过程重复的次数。数值越高结果越精确，但需要更多训练时间。必须为大于 1 的整数。", - "groupColumn": "分组列", + "selectHowToDivideDataset": "选择如何将数据集划分为训练集、验证集和测试集。", "selectInputOutputColumnsDescription": "从列表中选择列名。", "selectMetrics": "选择指标", "selectModelFirst": "请先选择模型", @@ -88,20 +92,17 @@ "setNameAndTask": "设置名称和任务", "shuffle": "随机打乱", "shuffleDescription": "确定在定义子集时是否对数据进行随机打乱。设为 true 则打乱，否则为 false。", + "splitType": "划分类型", "splits": "划分", "splitsDescription": "分配给每个子集的数据比例。值必须在 0 到 1 之间且总和为 1。", - "splitType": "划分类型", "startTime": "开始时间", "stratify": "按类分层", "stratifyDescription": "定义是否根据每个子集中的类别分布按比例分割数据。按类分层时需启用随机打乱。", + "stratifyRequiresShuffle": "需要启用随机打乱", "useManualSplittingBySpecifyingRowIndexes": "通过指定每个子集的行索引进行手动划分", "usePredefinedSplitsFromDataset": "使用数据集中的预定义划分", "usePredefinedSplitsFromDatasetNotAvailable": "使用数据集中的预定义划分（不可用）", - "useRandomRowsBySpecifyingPortion": "通过指定每个子集使用数据集的比例进行随机行划分", - "selectAColumn": "选择一列", - "groupColumnDescription": "用于分组的列。属于同一组的样本将保持在同一折中，避免同一组同时出现在训练集和测试集中。", - "stratifyRequiresShuffle": "需要启用随机打乱", - "selectDataset": "选择数据集" + "useRandomRowsBySpecifyingPortion": "通过指定每个子集使用数据集的比例进行随机行划分" }, "message": { "confirmDeleteRun": "确定要删除此运行吗？此操作无法撤销。", diff --git a/DashAI/front/src/utils/i18n/locales/zh/models.json b/DashAI/front/src/utils/i18n/locales/zh/models.json index 0abfc3a6e..6ab14cc6d 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/models.json +++ b/DashAI/front/src/utils/i18n/locales/zh/models.json @@ -1,22 +1,22 @@ { "button": { + "addConverter": "添加转换器", + "addNewPrediction": "添加新预测", "createExplainer": "创建解释器", "createGlobalExplainer": "新建全局解释器", "createLocalExplainer": "新建局部解释器", "createPrediction": "创建预测", - "addNewPrediction": "添加新预测", - "uploadNewDataset": "上传新数据集", "createSession": "创建会话", "deleteAndRetrain": "删除并重新训练", "deleteRun": "删除运行", "hideOperations": "隐藏操作", "hideParameters": "隐藏参数", "modelsHub": "模型中心", - "newSession": "新建会话", "modifyParameters": "修改参数", - "newPrediction": "新建预测", "newDatasetPrediction": "新建数据集预测", "newManualPrediction": "新建手动预测", + "newPrediction": "新建预测", + "newSession": "新建会话", "retrain": "重新训练", "runAll": "全部运行", "runAllModels": "运行所有模型", @@ -24,11 +24,13 @@ "saveAndRunModel": "保存并运行模型", "showOperations": "显示操作", "showParameters": "显示参数", - "updateAndRetrain": "更新并重新训练" + "updateAndRetrain": "更新并重新训练", + "uploadNewDataset": "上传新数据集" }, "error": { "completeRequiredFields": "请填写所有必填字段", "createRun": "创建新运行时出错：{{name}}", + "createRunReason": "创建新运行 \"{{name}}\" 时出错：{{reason}}", "createSession": "创建新会话时出错", "datasetRequired": "数据集为必填项", "enterModelName": "请输入模型名称", @@ -70,103 +72,33 @@ }, "label": { "addModelToSession": "将模型添加到会话", + "allRepetitions": "所有重复", + "alternativeHypothesis": "备择假设", + "applyPreprocessing": "应用预处理", + "applyPreprocessingDescription": "所选的转换器仅在训练数据上拟合，然后应用于其余数据，以避免数据泄漏。如果不应用，模型将按原样使用数据进行训练。", "availableExplainers": "可用解释器", "availableModels": "可用模型", - "noSavedTests": "暂无已保存的检验。", - "name": "名称", - "metric": "指标", - "metricSplit": "划分", - "significantsCount_other": "{{count}} / {{total}} 显著", - "bestModel": "最优", - "score": "评分", - "scoreProfile": "评分配置文件", - "scoreHeaderTooltip": "根据所选配置文件计算的 0–100 加权评分。无上限的误差指标（MAE、RMSE、TER 等）会相对于本次比较中最差的模型进行归一化。将鼠标悬停在单元格上可查看明细。", - "profile_balanced": "均衡", - "profile_detectPositives": "检测阳性", - "profile_avoidFalseAlarms": "避免误报", - "profile_probabilityQuality": "概率质量", - "profile_regression_fit": "模型拟合度", - "profile_regression_error": "均衡误差", - "profile_translation_quality": "翻译质量", - "profile_translation_balanced": "均衡翻译", - "profile_text_balanced": "均衡", - "profile_text_detectPositives": "检测阳性", - "profile_text_avoidFalseAlarms": "避免误报", - "profile_text_probabilityQuality": "概率质量", - "nestedCrossValidation": "嵌套交叉验证", - "outerSplitterInherited": "会话初始划分器配置已继承，用于嵌套交叉验证的外层循环", - "innerLoopConfiguration": "配置嵌套交叉验证内层循环的数据划分", - "outerSplitter": "外层循环划分器", - "outerFolds": "折数", - "innerSplitter": "内层循环划分器", - "innerFolds": "折数", - "statisticalTests": "统计检验", - "modelsToCompare": "待比较模型", - "significanceLevel": "显著性水平", - "noFinishedRuns": "没有可用的已完成运行", - "result": "结果", - "statistic": "统计量", - "significant": "显著", - "notSignificant": "不显著", - "technicalDetails": "技术细节", - "model1": "模型 1", - "model2": "模型 2", - "nemenyiPairwiseComparisons": "Friedman 检验后的 Nemenyi 事后成对比较", - "tukeyPairwiseComparisons": "ANOVA 检验后的 Tukey 事后成对比较", - "wilcoxonPairwiseComparisons": "带 Holm 校正的成对 Wilcoxon 检验", - "alternativeHypothesis": "备择假设", - "correctionMethod": "校正方法", - "repetition": "重复", - "allRepetitions": "所有重复", "averaged": "平均值", - "lines": "折线", - "histogramPlot": "直方图", - "foldNumber": "折编号", - "theoricalQuantiles": "理论分位数", - "sampleQuantiles": "样本分位数", - "metricValue": "指标值", - "frequency": "频率", - "graphs": "图表", - "helperTests": "假设检验", - "selectAtLeastRuns": "请至少选择 {{min}} 个模型", - "selectExactlyRuns": "请恰好选择 {{count}} 个模型", - "selectExactlyRuns_other": "请恰好选择 {{count}} 个模型", - "normalityByRunSummary": "{{total}} 个运行中有 {{normal}} 个似乎服从正态分布（α = {{alpha}}）。", - "notNormal": "非正态", - "normal": "正态", - "selectBetweenRuns": "请选择 {{min}} 到 {{max}} 个模型", - "saveDetails": "保存详情", - "testName": "检验名称（可选）", - "testNameHelp": "如果留空，将使用检验的名称。", - "testDescription": "描述（可选）", - "resultSaved": "已保存", - "saveResult": "保存结果", - "withHpo": "含 HPO", - "withoutHpo": "不含 HPO", - "nestedCv": "嵌套交叉验证", + "bar": "条形图", + "bestModel": "最优", + "chartType": "图表类型", "chooseTaskForSessionWithDataset": "为使用数据集 \"{{datasetName}}\" 的会话选择机器学习任务。", "configuration": "配置", "configureModel": "配置模型", "configureOptimizer": "配置优化器", "configureSession": "配置会话", "configureTasksTrainCompareModels": "在有组织的会话中配置任务、训练并比较模型。选择一个任务开始建模工作流。", - "confirmDeleteSession": "确定要删除会话 \"{{name}}\" 吗？此操作无法撤销。", + "configureTest": "配置检验", "confirmBulkDeleteSessions_one": "确定要删除所选的会话吗？此操作无法撤销。", "confirmBulkDeleteSessions_other": "确定要删除所选的 {{count}} 个会话吗？此操作无法撤销。", - "selectSessionsToDelete": "选择要删除的会话", + "confirmDeleteSession": "确定要删除会话 \"{{name}}\" 吗？此操作无法撤销。", + "confirmParameterUpdate": "确认参数更新", + "correctionMethod": "校正方法", "customMetrics": "自定义指标", "datasetPredictions": "数据集预测", + "divideColumnsAndSplits": "划分列并配置数据集划分", "dropModelHere": "拖放此处以添加", "editRun": "编辑运行", - "chartType": "图表类型", - "savedTests": "已保存的检验", - "foldGraphs": "折图表", - "nestedCvResults": "嵌套结果", - "outerFoldFetch": "外层折", - "finalFoldFetch": "最终折", - "twoSided": "双侧（模型 1 ≠ 模型 2）", - "greater": "更大（模型 1 > 模型 2）", - "less": "更小（模型 1 < 模型 2）", "editRunParameters": "编辑参数并重新运行模型", "epoch": "轮次", "exitModelDetailToAddModels": "返回会话概览以添加更多模型。", @@ -174,39 +106,64 @@ "explainability": "可解释性", "explainersCount_one": "• <1>{{count}} 个解释器", "explainersCount_other": "• <1>{{count}} 个解释器", + "finalFoldFetch": "最终折", + "foldGraphs": "折图表", + "foldNumber": "折编号", + "frequency": "频率", "generalMetrics": "通用指标", "globalExplainer": "全局解释器", "globalExplainers": "全局解释器", "goalMetric": "目标指标", + "graphs": "图表", + "greater": "更大（模型 1 > 模型 2）", "heatmap": "热图", + "helperTests": "假设检验", "hideResults": "隐藏结果", "higherIsBetter": "越高越好", + "histogramPlot": "直方图", "hyperparameterOptimizationPlots": "超参数优化图表", "hyperparameterOptimizerConfiguration": "超参数优化器配置", "hyperparameters": "超参数", + "innerFolds": "折数", + "innerLoopConfiguration": "配置嵌套交叉验证内层循环的数据划分", + "innerSplitter": "内层循环划分器", "inputColumns": "输入列", + "less": "更小（模型 1 < 模型 2）", + "lines": "折线", "liveMetrics": "实时指标", "localExplainer": "局部解释器", "localExplainers": "局部解释器", "lowerIsBetter": "越低越好", - "manualPredictions": "手动预测", "manual": "手动", + "manualPredictions": "手动预测", + "metric": "指标", + "metricSplit": "划分", + "metricToOptimize": "待优化指标", + "metricValue": "指标值", "metrics": "指标", "metricsEmptyForDisplaySet": "{{set}} 的结果指标为空。", - "metricToOptimize": "待优化指标", + "minTwoRunsRequired": "请至少选择 2 个已完成的运行以进行统计检验", + "model1": "模型 1", + "model2": "模型 2", "modelComparison": "模型比较", "modelConfiguration": "模型配置", "modelCount_one": "{{count}} 个模型", "modelCount_other": "{{count}} 个模型", - "bar": "条形图", "modelsModule": "模型模块", + "modelsToCompare": "待比较模型", + "name": "名称", "nameYourSession": "为会话命名", + "nemenyiPairwiseComparisons": "Friedman 检验后的 Nemenyi 事后成对比较", + "nestedCrossValidation": "嵌套交叉验证", + "nestedCv": "嵌套交叉验证", + "nestedCvResults": "嵌套结果", "noCompatibleExplainersFound": "未找到兼容的解释器", "noCompatibleModelsFound": "未找到兼容模型", "noCompletedRuns": "暂无已完成的运行", "noConfigurationAvailable": "此运行没有可用的配置", "noDatasetPredictionsYet": "暂无数据集预测", "noExplainersMatchSearch": "没有解释器匹配您的搜索", + "noFinishedRuns": "没有可用的已完成运行", "noGlobalExplainersYet": "暂无全局解释器", "noHyperparameterPlotsAvailable": "无超参数图表可用。只有至少包含一个可优化参数的运行才会生成图表。", "noLocalExplainersYet": "暂无局部解释器", @@ -216,19 +173,50 @@ "noModelsMatchSearch": "没有模型匹配您的搜索", "noPredictionsYet": "暂无预测记录", "noRunsYet": "暂无运行记录。请从右侧面板添加模型。", + "noSavedTests": "暂无已保存的检验。", "noSessionSelected": "未选择会话", + "noTestsMatch": "没有检验匹配您的搜索", + "nonParametricTests": "非参数检验", + "normal": "正态", + "normalityByRunSummary": "{{total}} 个运行中有 {{normal}} 个似乎服从正态分布（α = {{alpha}}）。", + "notNormal": "非正态", + "notSignificant": "不显著", "operations": "操作", "operationsWillBeDeletedWarning": "这些操作将被永久删除且无法恢复。确定要继续吗？", + "optimizer": "优化器", "optimizerConfiguration": "配置超参数优化器设置", "optimizerParameters": "优化器参数", - "optimizer": "优化器", + "outerFoldFetch": "外层折", + "outerFolds": "折数", + "outerSplitter": "外层循环划分器", + "outerSplitterInherited": "会话初始划分器配置已继承，用于嵌套交叉验证的外层循环", "outputColumns": "输出列", + "parametricTests": "参数检验", "pleaseSelectMetricToOptimize": "请选择要优化的指标。", "predictions": "预测", "predictionsCount_one": "• <1>{{count}} 个预测", "predictionsCount_other": "• <1>{{count}} 个预测", - "divideColumnsAndSplits": "划分列并配置数据集划分", "prepareDataset": "准备数据集", + "preprocessingFailed": "预处理失败", + "preprocessingInProgress": "正在处理…", + "preprocessingOptional": "预处理（可选）", + "preprocessingOptionalDescription": "所选的转换器仅在训练数据上拟合，然后应用于其余数据，以避免数据泄漏。", + "profile_avoidFalseAlarms": "避免误报", + "profile_balanced": "均衡", + "profile_detectPositives": "检测阳性", + "profile_probabilityQuality": "概率质量", + "profile_regression_error": "均衡误差", + "profile_regression_fit": "模型拟合度", + "profile_text_avoidFalseAlarms": "避免误报", + "profile_text_balanced": "均衡", + "profile_text_detectPositives": "检测阳性", + "profile_text_probabilityQuality": "概率质量", + "profile_translation_balanced": "均衡翻译", + "profile_translation_quality": "翻译质量", + "repetition": "重复", + "reports": "报告", + "result": "结果", + "resultSaved": "已保存", "retrainConfirmDetails": "确定要重新训练运行 \"<1>{{runName}}\" 吗？", "retrainModel": "重新训练模型？", "retrainWillDeleteOperations": "此运行有将被删除的现有操作", @@ -238,47 +226,69 @@ "runFailedNoHyperparameterPlots": "运行失败。无超参数图表可用。", "runInProgressCannotEdit": "运行正在进行中，无法编辑。", "runName": "运行名称", - "confirmParameterUpdate": "确认参数更新", "runNotFound": "未找到该运行", "runNotStartedNoHyperparameterPlots": "运行未开始。无超参数图表可用。", + "runsSelected": "个已选运行", + "sampleQuantiles": "样本分位数", "saveConfirmDetails": "保存“<1>{{runName}}”将把其状态重置为“未开始”，并清除其当前的指标和结果。确定要继续吗？", + "saveDetails": "保存详情", "saveParameterChanges": "保存参数更改？", + "saveResult": "保存结果", "saveWillDeleteOperationsDetails": "保存 \"<1>{{runName}}\" 将重置该运行。再次训练时以下内容将被删除：", + "savedTests": "已保存的检验", + "score": "评分", + "scoreHeaderTooltip": "根据所选配置文件计算的 0–100 加权评分。无上限的误差指标（MAE、RMSE、TER 等）会相对于本次比较中最差的模型进行归一化。将鼠标悬停在单元格上可查看明细。", + "scoreProfile": "评分配置文件", "searchDatasetsSessions": "搜索数据集和会话", "searchMetric": "搜索指标...", "searchModels": "搜索模型...", "searchTests": "搜索检验...", - "parametricTests": "参数检验", - "nonParametricTests": "非参数检验", - "minTwoRunsRequired": "请至少选择 2 个已完成的运行以进行统计检验", - "noTestsMatch": "没有检验匹配您的搜索", - "runsSelected": "个已选运行", - "configureTest": "配置检验", + "selectAtLeastRuns": "请至少选择 {{min}} 个模型", + "selectBetweenRuns": "请选择 {{min}} 到 {{max}} 个模型", + "selectColumnsDescription": "选择哪些列作为此会话的输入，哪一列作为输出。", + "selectColumnsTitle": "选择列", "selectDataset": "选择数据集", - "selectDatasetForSession": "为会话选择数据集", "selectDatasetAndPrepare": "为会话命名，选择数据集并配置列和划分。", "selectDatasetFirst": "选择数据集以配置其列和行划分。", + "selectDatasetForSession": "为会话选择数据集", + "selectExactlyRuns": "请恰好选择 {{count}} 个模型", + "selectExactlyRuns_other": "请恰好选择 {{count}} 个模型", "selectSessionToViewModels": "选择会话以查看可用模型。", + "selectSessionsToDelete": "选择要删除的会话", "selectTask": "选择任务", "selectTaskForSession": "为会话选择任务", "sessionConfiguration": "会话配置", "sessionName": "会话名称", "showResults": "显示结果", + "significanceLevel": "显著性水平", + "significant": "显著", + "significantsCount_other": "{{count}} / {{total}} 显著", + "statistic": "统计量", + "statisticalTests": "统计检验", "step": "步骤", + "technicalDetails": "技术细节", "test": "测试", + "testDescription": "描述（可选）", "testMetrics": "测试集指标", + "testName": "检验名称（可选）", + "testNameHelp": "如果留空，将使用检验的名称。", "testSet": "测试集", + "theoricalQuantiles": "理论分位数", "thereAreNoMetricsForThisRun": "此运行中没有与 {{set}} 关联的指标", "tourDisabledMessage": "返回首页以开始引导", "train": "训练", - "trainingMetrics": "训练集指标", "trainSet": "训练集", + "trainingMetrics": "训练集指标", "trial": "试验", + "tukeyPairwiseComparisons": "ANOVA 检验后的 Tukey 事后成对比较", + "twoSided": "双侧（模型 1 ≠ 模型 2）", "validation": "验证", "validationMetrics": "验证集指标", "validationSet": "验证集", "viewResultsAs": "以列或图表方式查看结果", - "reports": "报告" + "wilcoxonPairwiseComparisons": "带 Holm 校正的成对 Wilcoxon 检验", + "withHpo": "含 HPO", + "withoutHpo": "不含 HPO" }, "message": { "allRunsCompleted": "{{experiment}} 已完成所有运行。", diff --git a/tests/back/api/test_explainer_preprocessing.py b/tests/back/api/test_explainer_preprocessing.py new file mode 100644 index 000000000..5b36f031e --- /dev/null +++ b/tests/back/api/test_explainer_preprocessing.py @@ -0,0 +1,186 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import ( + Dataset, + GlobalExplainer, + LocalExplainer, +) + + +def _create_and_train_session_with_binarizer(client: TestClient, dataset_id: int): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_id, + "task_name": "TabularClassificationTask", + "name": "explainer-preprocessing-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert session_response.status_code == 201, session_response.text + model_session_id = session_response.json()["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "ExplainerPreprocessingRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Training for explainer preprocessing test", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + return model_session_id, run_id + + +def test_local_explainer_manual_input_applies_the_persisted_preprocessor( + client: TestClient, dataset_1: Dataset +): + """A manual-mode local explanation, like manual prediction, only ever + carries the raw feature the converter scoped on. The explainer's own + fit dataset (the model_session's training data) and the explained + instance must both be transformed by the persisted Binarizer before + reaching the model's feature space. + """ + model_session_id, run_id = _create_and_train_session_with_binarizer( + client, dataset_1.id + ) + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + local_explainer = LocalExplainer( + name="preprocessing_local_explainer", + run_id=run_id, + explainer_name="KernelShap", + dataset_id=dataset_1.id, + scope={"mode": "manual"}, + parameters={}, + fit_parameters={ + "sample_background_data": False, + "background_fraction": 0.5, + "sampling_method": "shuffle", + }, + ) + db.add(local_explainer) + db.commit() + db.refresh(local_explainer) + explainer_id = local_explainer.id + + job_response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": explainer_id, + "explainer_scope": "local", + "manual_input_data": [{"SepalLengthCm": 3.0}], + } + ), + }, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + client.delete(f"/api/v1/model-session/{model_session_id}") + + +def test_global_explainer_applies_the_persisted_preprocessor( + client: TestClient, dataset_1: Dataset +): + """The global explainer reuses the same background-data preparation as + the local one (data_x/data_y computed once in ExplainerJob.run before + branching on scope), so it inherits the same preprocessor fix without + needing its own — this test exists to confirm that, not just assume it. + """ + model_session_id, run_id = _create_and_train_session_with_binarizer( + client, dataset_1.id + ) + + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + global_explainer = GlobalExplainer( + name="preprocessing_global_explainer", + run_id=run_id, + explainer_name="PermutationFeatureImportance", + parameters={"scoring": "accuracy", "n_repeats": 5}, + ) + db.add(global_explainer) + db.commit() + db.refresh(global_explainer) + explainer_id = global_explainer.id + + job_response = client.post( + "/api/v1/job/", + data={ + "job_type": "ExplainerJob", + "kwargs": json.dumps( + { + "explainer_id": explainer_id, + "explainer_scope": "global", + } + ), + }, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/api/test_model_job_preprocessing.py b/tests/back/api/test_model_job_preprocessing.py new file mode 100644 index 000000000..c9c22efe1 --- /dev/null +++ b/tests/back/api/test_model_job_preprocessing.py @@ -0,0 +1,178 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset + + +def _create_session_with_binarizer(client: TestClient, dataset_id: int, name: str): + return client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_id, + "task_name": "TabularClassificationTask", + "name": name, + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + + +def test_model_job_trains_using_a_converter_produced_group_column( + client: TestClient, dataset_1: Dataset +): + session_response = _create_session_with_binarizer( + client, dataset_1.id, "preprocessing-training-session" + ) + assert session_response.status_code == 201, session_response.text + session_body = session_response.json() + assert session_body["preprocessing_status"] == "ready" + model_session_id = session_body["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "PreprocessingRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Training with a group column from a converter", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + run_after = client.get(f"/api/v1/run/{run_id}").json() + assert run_after["status"] == 3 # RunStatus.FINISHED + assert run_after["run_path"] is not None + + client.delete(f"/api/v1/run/{run_id}") + client.delete(f"/api/v1/model-session/{model_session_id}") + + +def test_model_job_trains_with_cross_validation_and_per_fold_preprocessing( + client: TestClient, dataset_1: Dataset +): + response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_1.id, + "task_name": "TabularClassificationTask", + "name": "preprocessing-cv-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "CrossValidationEvaluationStrategy", + "splits": json.dumps( + { + "n_splits": 3, + "test_size": 0.0, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + "splitter_name": "KFoldSplitter", + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert response.status_code == 201, response.text + session_body = response.json() + assert session_body["preprocessing_status"] == "ready" + model_session_id = session_body["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "PreprocessingCVRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "CV training with per-fold preprocessing", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + run_after = client.get(f"/api/v1/run/{run_id}").json() + assert run_after["status"] == 3 # RunStatus.FINISHED + + client.delete(f"/api/v1/run/{run_id}") + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/api/test_model_session_api.py b/tests/back/api/test_model_session_api.py index 23624d537..d7f018847 100644 --- a/tests/back/api/test_model_session_api.py +++ b/tests/back/api/test_model_session_api.py @@ -50,6 +50,158 @@ def test_delete_model_session(client: TestClient, dataset_1: Dataset) -> None: assert response.status_code == 404, response.text +def test_create_model_session_without_preprocessing_is_ready_immediately( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "no-preprocessing-session", + }, + ) + assert response.status_code == 201, response.text + assert response.json()["preprocessing_status"] == "ready" + + session_id = response.json()["id"] + client.delete(f"/api/v1/model-session/{session_id}") + + +def test_create_model_session_with_preprocessing_fits_and_resolves_columns( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "with-preprocessing-session", + "input_columns": [], + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert response.status_code == 201, response.text + body = response.json() + # The test client runs the job queue in immediate mode, so by the time + # the request returns, PreprocessingJob already fit and resolved this. + # Binarizer is an EncodingConverter: it appends "bin_" and keeps the + # original column unchanged (a passthrough, like BagOfWords keeping its + # source text column) — the group is only the genuinely new column it + # produced, not the untouched original alongside it. + assert body["preprocessing_status"] == "ready" + assert body["input_columns"] == ["bin_SepalLengthCm"] + # The frontend tracks this exact job via the shared job-polling + # mechanism (useJobTracker), so it must be a real, non-empty id. + assert body["preprocessing_job_id"] + + session_id = body["id"] + client.delete(f"/api/v1/model-session/{session_id}") + + +def test_create_model_session_with_preprocessing_requires_input_column_refs( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/", + json={ + **SESSION_PARAMS, + "dataset_id": dataset_1.id, + "name": "missing-refs-session", + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + }, + ) + assert response.status_code == 422, response.text + + +def test_validate_columns_accepts_a_group_ref_matching_the_task_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": ["SepalWidthCm"], + "outputs_columns": ["Species"], + "input_refs": [ + {"kind": "raw", "name": "SepalWidthCm"}, + {"kind": "group", "step": 0}, + ], + "converter_output_types": {"0": "Integer"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "valid" + + +def test_validate_columns_rejects_a_group_ref_with_an_incompatible_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": [], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "group", "step": 0}], + "converter_output_types": {"0": "Text"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "invalid" + + +def test_validate_columns_accepts_a_slotted_group_ref_matching_the_task_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": [], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "group", "step": 0, "slot": "Integer"}], + "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "valid" + + +def test_validate_columns_rejects_a_slotted_group_ref_with_an_incompatible_type( + client: TestClient, dataset_1: Dataset +) -> None: + response = client.post( + "/api/v1/model-session/validation", + json={ + "task_name": "TabularClassificationTask", + "dataset_id": dataset_1.id, + "inputs_columns": [], + "outputs_columns": ["Species"], + "input_refs": [{"kind": "group", "step": 0, "slot": "Categorical"}], + "converter_output_types": {"0:Integer": "Integer", "0:Categorical": "Text"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["dataset_status"] == "invalid" + + def test_bulk_delete_model_sessions(client: TestClient, dataset_1: Dataset) -> None: created_ids = [] for name in ["bulk_delete_session_1", "bulk_delete_session_2"]: diff --git a/tests/back/api/test_predict_preprocessing.py b/tests/back/api/test_predict_preprocessing.py new file mode 100644 index 000000000..ebd9a9d04 --- /dev/null +++ b/tests/back/api/test_predict_preprocessing.py @@ -0,0 +1,107 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset + + +def _create_and_train_session_with_binarizer(client: TestClient, dataset_id: int): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_id, + "task_name": "TabularClassificationTask", + "name": "predict-preprocessing-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "SepalLengthCm"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert session_response.status_code == 201, session_response.text + model_session_id = session_response.json()["id"] + + run_response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "PredictPreprocessingRun", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Training for prediction preprocessing test", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert run_response.status_code == 201, run_response.text + run_id = run_response.json()["id"] + + job_response = client.post( + "/api/v1/job/", + data={"job_type": "ModelJob", "kwargs": json.dumps({"run_id": run_id})}, + ) + assert job_response.status_code == 201, job_response.text + job_id = job_response.json()["id"] + job_status = client.get(f"/api/v1/job/status/{job_id}").json() + assert job_status["status"] == "finished", job_status + + return model_session_id, run_id + + +def test_manual_prediction_applies_the_persisted_preprocessor( + client: TestClient, dataset_1: Dataset +): + """A manual prediction row carries only the raw feature the converter + scoped on ("SepalLengthCm"), never the converter-produced column + ("bin_SepalLengthCm") the model was actually trained on. Predicting + must apply the persisted fitted Binarizer to the raw row before handing + it to the model, without ever re-fitting on it. + """ + model_session_id, run_id = _create_and_train_session_with_binarizer( + client, dataset_1.id + ) + + response = client.post( + "/api/v1/predict/preview", + data={ + "run_id": str(run_id), + "manual_input_data": json.dumps([{"SepalLengthCm": 3.0}]), + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert "columns" in body + assert len(body["rows"]) == 1 + + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/api/test_run_preprocessing_guard.py b/tests/back/api/test_run_preprocessing_guard.py new file mode 100644 index 000000000..4ea339e22 --- /dev/null +++ b/tests/back/api/test_run_preprocessing_guard.py @@ -0,0 +1,145 @@ +import json + +from fastapi.testclient import TestClient + +from DashAI.back.dependencies.database.models import Dataset, ModelSession + + +def test_run_creation_is_rejected_while_preprocessing_is_pending( + client: TestClient, dataset_1: Dataset +): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_1.id, + "task_name": "TabularClassificationTask", + "name": "pending-guard-session", + "input_columns": ["SepalLengthCm"], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + }, + ) + assert session_response.status_code == 201, session_response.text + model_session_id = session_response.json()["id"] + + # The test client runs the job queue in immediate mode, so a session + # never actually stays "pending" long enough to observe over the API. + # Force it here to exercise the guard the way production would hit it + # while PreprocessingJob is still running asynchronously. + container = client.app.container + session_factory = container["session_factory"] + with session_factory() as db: + model_session = db.get(ModelSession, model_session_id) + model_session.preprocessing_status = "pending" + db.commit() + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "ShouldBeBlocked", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Should be rejected", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 409, response.text + + client.delete(f"/api/v1/model-session/{model_session_id}") + + +def test_run_creation_surfaces_the_preprocessing_error_when_failed( + client: TestClient, dataset_1: Dataset +): + session_response = client.post( + "/api/v1/model-session/", + json={ + "dataset_id": dataset_1.id, + "task_name": "TabularClassificationTask", + "name": "failed-guard-session", + "input_columns": [], + "output_columns": ["Species"], + "train_metrics": [], + "validation_metrics": [], + "test_metrics": [], + "evaluation_strategy": "HoldoutEvaluationStrategy", + "splits": json.dumps( + { + "train": 0.5, + "test": 0.2, + "validation": 0.3, + "is_random": True, + "has_changed": True, + "seed": 42, + "shuffle": True, + "stratify": False, + } + ), + "preprocessing": [ + { + # Binarizer only accepts numeric columns; scoping it on the + # categorical target makes PreprocessingJob's fit fail. + "converter": "Binarizer", + "params": {"threshold": 3.0}, + "scope": [{"kind": "raw", "name": "Species"}], + }, + ], + "input_column_refs": [{"kind": "group", "step": 0}], + }, + ) + assert session_response.status_code == 201, session_response.text + body = session_response.json() + assert body["preprocessing_status"] == "failed" + model_session_id = body["id"] + + response = client.post( + "/api/v1/run/", + json={ + "model_session_id": model_session_id, + "model_name": "KNeighborsClassifier", + "name": "ShouldBeBlocked", + "parameters": {"n_neighbors": 3, "weights": "uniform", "algorithm": "auto"}, + "optimizer_name": "", + "optimizer_parameters": { + "n_trials": 10, + "sampler": "TPESampler", + "pruner": "None", + }, + "goal_metric": "", + "description": "Should be rejected", + "plot_history_path": "path/to/history.png", + "plot_slice_path": "path/to/slice.png", + "plot_contour_path": "path/to/contour.png", + "plot_importance_path": "path/to/importance.png", + }, + ) + assert response.status_code == 409, response.text + assert "preprocessing" in response.json()["detail"].lower() + + client.delete(f"/api/v1/model-session/{model_session_id}") diff --git a/tests/back/converters/test_base_converter_metadata.py b/tests/back/converters/test_base_converter_metadata.py index 3ac5f1307..1191b67fd 100644 --- a/tests/back/converters/test_base_converter_metadata.py +++ b/tests/back/converters/test_base_converter_metadata.py @@ -137,3 +137,88 @@ def transform(self, x, y=None): meta = _CatTextConverter.get_metadata() assert meta["allowed_types"] == ["Categorical", "Text"] assert meta["allowed_dtypes"] == ["string"] + + +def test_get_metadata_reports_a_representative_output_type_when_declared(): + class _WithOutput(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Integer]} + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _WithOutput.get_metadata() + assert meta["output_type"] == "Integer" + assert meta["output_dtype"] == "int64" + + +def test_get_metadata_output_type_is_none_when_get_output_type_returns_none(): + meta = _FloatIntConverter.get_metadata() # returns None today, by design + assert meta["output_type"] is None + assert meta["output_dtype"] is None + + +def test_get_metadata_output_type_is_none_when_the_converter_cannot_be_built(): + class _RequiresArgConverter(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Integer]} + + def __init__(self, required_param): + self.required_param = required_param + + def get_output_type(self, column_name=None): + return None + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _RequiresArgConverter.get_metadata() + assert meta["output_type"] is None + assert meta["output_dtype"] is None + + +def test_get_metadata_preserves_input_type_defaults_to_false(): + meta = _FloatIntConverter.get_metadata() + assert meta["preserves_input_type"] is False + + +def test_get_metadata_reports_preserves_input_type_when_declared(): + class _SelectionLikeConverter(BaseConverter): + SCHEMA = None + metadata = {"allowed_types": [Float, Integer]} + PRESERVES_INPUT_TYPE = True + + def get_output_type(self, column_name=None): + return None + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + return x + + meta = _SelectionLikeConverter.get_metadata() + assert meta["preserves_input_type"] is True + + +def test_feature_selection_and_variance_threshold_declare_preserves_input_type(): + from DashAI.back.converters.category.feature_selection import ( + FeatureSelectionConverter, + ) + from DashAI.back.converters.scikit_learn.variance_threshold import ( + VarianceThreshold, + ) + + assert FeatureSelectionConverter.get_metadata()["preserves_input_type"] is True + assert VarianceThreshold.get_metadata()["preserves_input_type"] is True diff --git a/tests/back/converters/test_dataset_columns.py b/tests/back/converters/test_dataset_columns.py new file mode 100644 index 000000000..754758d47 --- /dev/null +++ b/tests/back/converters/test_dataset_columns.py @@ -0,0 +1,53 @@ +import pandas as pd + +from DashAI.back.converters.dataset_columns import ( + rebuild_dataset_with_transformed_columns, +) +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) + + +def _dataset(columns, schema): + return transform_dataset_with_schema( + to_dashai_dataset(pd.DataFrame(columns)), schema + ) + + +def test_replaces_scoped_column_in_place_and_keeps_others(): + base = _dataset( + {"a": [1, 2, 3], "b": [10, 20, 30]}, + { + "a": {"type": "Integer", "dtype": "int64"}, + "b": {"type": "Integer", "dtype": "int64"}, + }, + ) + transformed = _dataset( + {"a": [0, 1, 0]}, + {"a": {"type": "Integer", "dtype": "int64"}}, + ) + + result = rebuild_dataset_with_transformed_columns(base, transformed, ["a"], [0]) + + assert result.column_names == ["a", "b"] + assert result.to_pandas()["a"].tolist() == [0, 1, 0] + assert result.to_pandas()["b"].tolist() == [10, 20, 30] + + +def test_appends_new_columns_not_in_the_scope(): + base = _dataset( + {"a": [1, 2, 3]}, + {"a": {"type": "Integer", "dtype": "int64"}}, + ) + transformed = _dataset( + {"a_bin_0": [1, 0, 1], "a_bin_1": [0, 1, 0]}, + { + "a_bin_0": {"type": "Integer", "dtype": "int64"}, + "a_bin_1": {"type": "Integer", "dtype": "int64"}, + }, + ) + + result = rebuild_dataset_with_transformed_columns(base, transformed, ["a"], [0]) + + assert result.column_names == ["a_bin_0", "a_bin_1"] diff --git a/tests/back/migrations/__init__.py b/tests/back/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/back/migrations/test_add_preprocessing_to_model_session.py b/tests/back/migrations/test_add_preprocessing_to_model_session.py new file mode 100644 index 000000000..6dd1ddf2b --- /dev/null +++ b/tests/back/migrations/test_add_preprocessing_to_model_session.py @@ -0,0 +1,30 @@ +import sqlalchemy as sa +from alembic import command +from alembic.config import Config + + +def test_migration_adds_preprocessing_columns(tmp_path): + db_path = tmp_path / "test.db" + alembic_cfg = Config("alembic.ini") + alembic_cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}") + + command.upgrade(alembic_cfg, "head") + + engine = sa.create_engine(f"sqlite:///{db_path}") + inspector = sa.inspect(engine) + columns = {c["name"] for c in inspector.get_columns("model_session")} + + assert { + "preprocessing", + "input_column_refs", + "preprocessing_status", + "preprocessing_error", + "preprocessing_artifacts_path", + "preprocessing_job_id", + } <= columns + + command.downgrade(alembic_cfg, "-1") + columns_after_downgrade = { + c["name"] for c in sa.inspect(engine).get_columns("model_session") + } + assert "preprocessing" not in columns_after_downgrade diff --git a/tests/back/preprocessing/__init__.py b/tests/back/preprocessing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/back/preprocessing/test_column_ref.py b/tests/back/preprocessing/test_column_ref.py new file mode 100644 index 000000000..18510c20a --- /dev/null +++ b/tests/back/preprocessing/test_column_ref.py @@ -0,0 +1,127 @@ +import pytest + +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + ConverterStep, + GroupColumnRef, + RawColumnRef, + parse_column_refs, + resolve_refs, +) + + +def test_raw_ref_resolves_to_its_own_name(): + assert resolve_refs([RawColumnRef(name="age")], {}) == ["age"] + + +def test_group_ref_resolves_to_that_steps_produced_columns(): + refs = [RawColumnRef(name="age"), GroupColumnRef(step=0)] + resolved = resolve_refs(refs, {0: ["bow_apple", "bow_banana"]}) + assert resolved == ["age", "bow_apple", "bow_banana"] + + +def test_group_ref_to_an_unfit_step_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs([GroupColumnRef(step=0)], {}) + + +def test_slotted_group_ref_resolves_to_only_that_types_columns(): + refs = [GroupColumnRef(step=0, slot="Categorical")] + resolved_columns = {0: ["most_frequent_city", "imputed_age"]} + resolved_slots = { + 0: { + "Categorical": ["most_frequent_city"], + "Integer": ["imputed_age"], + } + } + assert resolve_refs(refs, resolved_columns, resolved_slots) == [ + "most_frequent_city" + ] + + +def test_slotted_group_ref_with_no_resolved_slots_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs([GroupColumnRef(step=0, slot="Categorical")], {0: ["x"]}) + + +def test_slotted_group_ref_to_a_type_the_step_never_produced_raises_key_error(): + with pytest.raises(KeyError): + resolve_refs( + [GroupColumnRef(step=0, slot="Text")], + {0: ["x"]}, + {0: {"Integer": ["x"]}}, + ) + + +def test_unslotted_group_ref_still_resolves_to_every_column_when_slots_exist(): + refs = [GroupColumnRef(step=0)] + resolved_columns = {0: ["most_frequent_city", "imputed_age"]} + resolved_slots = { + 0: { + "Categorical": ["most_frequent_city"], + "Integer": ["imputed_age"], + } + } + assert resolve_refs(refs, resolved_columns, resolved_slots) == [ + "most_frequent_city", + "imputed_age", + ] + + +def test_sequence_rejects_a_step_referencing_itself_or_later(): + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Binarizer", + params={}, + scope=[GroupColumnRef(step=0)], + ) + ] + ) + with pytest.raises(ValueError, match="not strictly before"): + sequence.validate_scopes() + + +def test_sequence_accepts_a_step_referencing_an_earlier_one(): + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="BagOfWordsConverter", + params={}, + scope=[RawColumnRef(name="text")], + ), + ConverterStep( + converter="StandardScaler", + params={}, + scope=[GroupColumnRef(step=0)], + ), + ] + ) + sequence.validate_scopes() # does not raise + + +def test_parse_column_refs_round_trips_json_dicts(): + raw = [{"kind": "raw", "name": "age"}, {"kind": "group", "step": 2}] + parsed = parse_column_refs(raw) + assert parsed == [RawColumnRef(name="age"), GroupColumnRef(step=2)] + + +def test_parse_column_refs_round_trips_a_slotted_group_ref(): + raw = [{"kind": "group", "step": 0, "slot": "Categorical"}] + parsed = parse_column_refs(raw) + assert parsed == [GroupColumnRef(step=0, slot="Categorical")] + + +def test_converter_sequence_model_dump_round_trips(): + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Binarizer", + params={"threshold": 0.5}, + scope=[RawColumnRef(name="age")], + ) + ] + ) + dumped = sequence.model_dump(mode="json") + restored = ConverterSequence.model_validate(dumped) + assert restored == sequence diff --git a/tests/back/preprocessing/test_session_preprocessor.py b/tests/back/preprocessing/test_session_preprocessor.py new file mode 100644 index 000000000..e2bd22c6c --- /dev/null +++ b/tests/back/preprocessing/test_session_preprocessor.py @@ -0,0 +1,502 @@ +import pandas as pd + +from DashAI.back.converters.base_converter import BaseConverter +from DashAI.back.dataloaders.classes.dashai_dataset import ( + to_dashai_dataset, + transform_dataset_with_schema, +) +from DashAI.back.preprocessing.column_ref import ( + ConverterSequence, + ConverterStep, + GroupColumnRef, + RawColumnRef, +) +from DashAI.back.preprocessing.session_preprocessor import SessionPreprocessor +from DashAI.back.types.value_types import Integer + + +def _dataset(columns, schema): + return transform_dataset_with_schema( + to_dashai_dataset(pd.DataFrame(columns)), schema + ) + + +class _DoublingConverter(BaseConverter): + """Deterministic: doubles a single numeric column in place.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + self._column = x.column_names[0] + return self + + def transform(self, x, y=None): + frame = x.to_pandas() + frame[self._column] = frame[self._column] * 2 + return to_dashai_dataset(frame) + + +class _FakeVocabConverter(BaseConverter): + """Variable output: emits one column per 'word' seen in fit, mirroring + Bag-of-Words' vocabulary-at-fit-time behavior without needing sklearn.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + column = x.column_names[0] + self._vocab = sorted(set(x.to_pandas()[column].tolist())) + return self + + def transform(self, x, y=None): + import pyarrow as pa + + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + + column = x.column_names[0] + values = x.to_pandas()[column].tolist() + output_type = self.get_output_type() + data = { + f"vocab_{word}": [1 if v == word else 0 for v in values] + for word in self._vocab + } + table = pa.table(data) + types = dict.fromkeys(data, output_type) + return DashAIDataset(table, types=types) + + +class _FakeAdditiveConverter(BaseConverter): + """Mirrors BagOfWordsConverter's real behavior: keeps its scope column + verbatim and appends brand-new derived columns alongside it, instead of + replacing the scope column or dropping it.""" + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def get_output_type(self, column_name=None): + import pyarrow as pa + + return Integer(arrow_type=pa.int64()) + + def fit(self, x, y=None): + return self + + def transform(self, x, y=None): + frame = x.to_pandas() + column = x.column_names[0] + frame["derived"] = frame[column] * 10 + output_type = self.get_output_type() + types = {**x.types, "derived": output_type} + return to_dashai_dataset(frame, types=types) + + +class _FakeTypeObj: + """Minimal stand-in for a DashAIDataType — only display_name() matters + to SessionPreprocessor._classify_by_type.""" + + def __init__(self, name): + self._name = name + + def display_name(self): + return self._name + + +class _MixedTypeConverter(BaseConverter): + """Mirrors SimpleImputer's most_frequent/FeatureSelectionConverter: + keeps every scope column's own value and declared type unchanged. Used + to test that fit_transform classifies real output columns into one slot + per distinct type when a scope mixes them, without needing a real + Categorical dataset column — the per-column type is just a hardcoded + mapping passed in as a param, exactly like a real converter would derive + it from its own fitted state. + """ + + SCHEMA = None + metadata = {"allowed_types": [Integer], "allowed_dtypes": []} + CHANGES_ROW_COUNT = False + + def __init__(self, type_by_column=None, **kwargs): + self._type_by_column = type_by_column or {} + + def fit(self, x, y=None): + return self + + def get_output_type(self, column_name=None): + return _FakeTypeObj(self._type_by_column.get(column_name, "Integer")) + + def transform(self, x, y=None): + return x + + +class _FakeRegistry: + def __init__(self, classes): + self._classes = classes + + def __getitem__(self, name): + return {"class": self._classes[name]} + + +def _split(train_ages, val_ages=None): + train = _dataset( + {"age": train_ages}, {"age": {"type": "Integer", "dtype": "int64"}} + ) + split = {"train": train} + if val_ages is not None: + split["validation"] = _dataset( + {"age": val_ages}, {"age": {"type": "Integer", "dtype": "int64"}} + ) + return split + + +def test_fits_only_on_train_and_transforms_every_split_present(): + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2, 3], val_ages=[10, 20]) + transformed, resolved = preprocessor.fit_transform(split) + + assert transformed["train"].to_pandas()["age"].tolist() == [2, 4, 6] + assert transformed["validation"].to_pandas()["age"].tolist() == [20, 40] + assert resolved == {0: ["age"]} + + +def test_variable_output_step_records_whatever_columns_it_produced(): + registry = _FakeRegistry({"FakeVocab": _FakeVocabConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="FakeVocab", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2, 1]) + transformed, resolved = preprocessor.fit_transform(split) + + assert sorted(resolved[0]) == ["vocab_1", "vocab_2"] + assert set(transformed["train"].column_names) == {"vocab_1", "vocab_2"} + + +def test_additive_step_that_keeps_its_scope_column_excludes_it_from_the_group(): + registry = _FakeRegistry({"Additive": _FakeAdditiveConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Additive", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2, 3]) + transformed, resolved = preprocessor.fit_transform(split) + + # "age" is carried through unchanged (a passthrough, like BagOfWords + # keeping its source text column) — it must not be part of the step's + # own group, only the genuinely new "derived" column is. + assert resolved == {0: ["derived"]} + assert set(transformed["train"].column_names) == {"age", "derived"} + + +def test_chained_step_can_reference_the_previous_steps_group(): + registry = _FakeRegistry( + {"FakeVocab": _FakeVocabConverter, "Doubler": _DoublingConverter} + ) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="FakeVocab", params={}, scope=[RawColumnRef(name="age")] + ), + ConverterStep( + converter="Doubler", params={}, scope=[GroupColumnRef(step=0)] + ), + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + split = _split(train_ages=[1, 2]) + transformed, resolved = preprocessor.fit_transform(split) + + # Doubler's scope is step 0's whole output group (both vocab columns); + # it doubles column_names[0] but its transform returns every column in + # its scope, so step 1 produces the same column set step 0 did. + assert set(resolved[1]) == set(resolved[0]) + assert len(resolved[1]) == 2 + + +def test_fit_transform_classifies_a_steps_real_output_columns_by_type(): + registry = _FakeRegistry({"MixedType": _MixedTypeConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="MixedType", + params={"type_by_column": {"age": "Integer", "city": "Categorical"}}, + scope=[RawColumnRef(name="age"), RawColumnRef(name="city")], + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + train = _dataset( + {"age": [1, 2, 3], "city": [10, 20, 10]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + preprocessor.fit_transform({"train": train}) + + assert preprocessor.resolved_slots[0] == { + "Integer": ["age"], + "Categorical": ["city"], + } + + +def test_chained_step_can_reference_a_specific_slot_of_an_earlier_steps_group(): + registry = _FakeRegistry( + {"MixedType": _MixedTypeConverter, "Doubler": _DoublingConverter} + ) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="MixedType", + params={"type_by_column": {"age": "Integer", "city": "Categorical"}}, + scope=[RawColumnRef(name="age"), RawColumnRef(name="city")], + ), + ConverterStep( + converter="Doubler", + params={}, + scope=[GroupColumnRef(step=0, slot="Integer")], + ), + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + train = _dataset( + {"age": [1, 2, 3], "city": [10, 20, 10]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + transformed, _ = preprocessor.fit_transform({"train": train}) + + # Doubler was scoped only to the "Integer" slot (age); city, the + # "Categorical" slot, is untouched. + assert transformed["train"].to_pandas()["age"].tolist() == [2, 4, 6] + assert transformed["train"].to_pandas()["city"].tolist() == [10, 20, 10] + + +def test_transform_only_resolves_a_slotted_scope_using_persisted_resolved_slots(): + registry = _FakeRegistry( + {"MixedType": _MixedTypeConverter, "Doubler": _DoublingConverter} + ) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="MixedType", + params={"type_by_column": {"age": "Integer", "city": "Categorical"}}, + scope=[RawColumnRef(name="age"), RawColumnRef(name="city")], + ), + ConverterStep( + converter="Doubler", + params={}, + scope=[GroupColumnRef(step=0, slot="Integer")], + ), + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + train = _dataset( + {"age": [1, 2, 3], "city": [10, 20, 10]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + preprocessor.fit_transform({"train": train}) + + new_data = _dataset( + {"age": [5], "city": [99]}, + { + "age": {"type": "Integer", "dtype": "int64"}, + "city": {"type": "Integer", "dtype": "int64"}, + }, + ) + result = preprocessor.transform_only({"train": new_data}) + + assert result["train"].to_pandas()["age"].tolist() == [10] + assert result["train"].to_pandas()["city"].tolist() == [99] + + +def test_transform_only_applies_an_already_fitted_sequence_without_refitting(): + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + new_split = _split(train_ages=[100]) + transformed = preprocessor.transform_only(new_split) + + assert transformed["train"].to_pandas()["age"].tolist() == [200] + + +def test_transform_dataset_is_a_single_dataset_convenience_wrapper(): + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + single = _dataset({"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}}) + result = preprocessor.transform_dataset(single) + + assert result.to_pandas()["age"].tolist() == [10] + + +def test_a_pickled_and_restored_preprocessor_still_transforms_correctly(): + import pickle + + registry = _FakeRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + restored = pickle.loads(pickle.dumps(preprocessor)) + single = _dataset({"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}}) + + assert restored.transform_dataset(single).to_pandas()["age"].tolist() == [10] + + +class _CrashesOnEmptyConverter(_DoublingConverter): + """Mirrors sklearn transformers that reject a 0-row array, e.g. Binarizer.""" + + def transform(self, x, y=None): + if x.num_rows == 0: + raise ValueError("Found array with 0 sample(s)") + return super().transform(x, y) + + +def test_fit_transform_does_not_crash_on_an_empty_split(): + registry = _FakeRegistry({"CrashesOnEmpty": _CrashesOnEmptyConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="CrashesOnEmpty", + params={}, + scope=[RawColumnRef(name="age")], + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + + train = _dataset({"age": [1, 2, 3]}, {"age": {"type": "Integer", "dtype": "int64"}}) + empty_test = _dataset({"age": []}, {"age": {"type": "Integer", "dtype": "int64"}}) + split = {"train": train, "test": empty_test} + + transformed, resolved = preprocessor.fit_transform(split) + + assert transformed["train"].to_pandas()["age"].tolist() == [2, 4, 6] + assert transformed["test"].num_rows == 0 + assert transformed["test"].column_names == transformed["train"].column_names + assert resolved == {0: ["age"]} + + +def test_transform_only_does_not_crash_on_an_empty_split(): + registry = _FakeRegistry({"CrashesOnEmpty": _CrashesOnEmptyConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="CrashesOnEmpty", + params={}, + scope=[RawColumnRef(name="age")], + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform( + { + "train": _dataset( + {"age": [1, 2, 3]}, {"age": {"type": "Integer", "dtype": "int64"}} + ) + } + ) + + empty_test = _dataset({"age": []}, {"age": {"type": "Integer", "dtype": "int64"}}) + transformed = preprocessor.transform_only( + { + "train": _dataset( + {"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}} + ), + "test": empty_test, + } + ) + + assert transformed["train"].to_pandas()["age"].tolist() == [10] + assert transformed["test"].num_rows == 0 + + +def test_a_preprocessor_pickles_even_when_its_registry_cannot_be_pickled(): + import pickle + + class _UnpicklableRegistry(_FakeRegistry): + def __init__(self, classes): + super().__init__(classes) + # Mirrors ComponentRegistry, which is not picklable because it + # holds RelationshipManager lambdas. + self._unpicklable = lambda: None + + registry = _UnpicklableRegistry({"Doubler": _DoublingConverter}) + sequence = ConverterSequence( + steps=[ + ConverterStep( + converter="Doubler", params={}, scope=[RawColumnRef(name="age")] + ) + ] + ) + preprocessor = SessionPreprocessor(sequence, registry) + preprocessor.fit_transform(_split(train_ages=[1, 2, 3])) + + restored = pickle.loads(pickle.dumps(preprocessor)) + single = _dataset({"age": [5]}, {"age": {"type": "Integer", "dtype": "int64"}}) + + assert restored.transform_dataset(single).to_pandas()["age"].tolist() == [10] + assert restored.component_registry is None diff --git a/tests/back/test_config_builder.py b/tests/back/test_config_builder.py new file mode 100644 index 000000000..c67411003 --- /dev/null +++ b/tests/back/test_config_builder.py @@ -0,0 +1,6 @@ +from DashAI.back.dependencies.config_builder import build_config_dict + + +def test_preprocessing_path_is_under_local_path(tmp_path): + config = build_config_dict(local_path=tmp_path, logging_level="ERROR") + assert config["PREPROCESSING_PATH"] == tmp_path / "preprocessing"