diff --git a/DashAI/back/evaluation/cv.py b/DashAI/back/evaluation/cv.py index 00ea17788..bc492c1b1 100644 --- a/DashAI/back/evaluation/cv.py +++ b/DashAI/back/evaluation/cv.py @@ -4,6 +4,7 @@ from kink import di from DashAI.back.core.enums.metrics import LevelEnum, SplitEnum +from DashAI.back.core.utils import MultilingualString from DashAI.back.dependencies.database.models import Metric, Run from DashAI.back.evaluation.base_evaluation_strategy import BaseEvaluationStrategy from DashAI.back.splitters.base_splitter import BaseSplitter @@ -425,6 +426,41 @@ class CrossValidationEvaluationStrategy(FoldEvaluationStrategy): ``ForecastingCrossValidationEvaluationStrategy`` handles that. """ + DESCRIPTION = MultilingualString( + en=( + "Cross validation cuts the dataset into folds. Each fold takes a " + "turn as the validation set while the model trains on the rest, and " + "the scores are averaged, so the result leans less on any single " + "cut." + ), + es=( + "La validacion cruzada corta el conjunto en pliegues. Cada pliegue " + "actua por turno como conjunto de validacion mientras el modelo " + "entrena con el resto, y los puntajes se promedian, asi el resultado " + "depende menos de un solo corte." + ), + pt=( + "A validacao cruzada corta o conjunto em dobras. Cada dobra serve " + "por vez como conjunto de validacao enquanto o modelo treina no " + "resto, e as pontuacoes sao promediadas, entao o resultado depende " + "menos de um unico corte." + ), + de=( + "Die Kreuzvalidierung teilt den Datensatz in Folds. Jeder Fold dient " + "reihum als Validierungsmenge, waehrend das Modell auf dem Rest " + "trainiert, und die Ergebnisse werden gemittelt, sodass das Resultat " + "weniger von einer einzelnen Teilung abhaengt." + ), + zh=( + "交叉验证把数据集切成" + "若干折。每一折轮流作为" + "验证集,模型在其余部分" + "上训练,最后取平均分," + "因此结果不那么依赖某" + "一次切分。" + ), + ) + COMPATIBLE_COMPONENTS = [ "TabularClassificationTask", "TextClassificationTask", diff --git a/DashAI/back/evaluation/forecasting_cv.py b/DashAI/back/evaluation/forecasting_cv.py index f9ba5e870..b07b7cf24 100644 --- a/DashAI/back/evaluation/forecasting_cv.py +++ b/DashAI/back/evaluation/forecasting_cv.py @@ -1,6 +1,7 @@ """Cross-validation for models that forecast a series from its own history.""" from DashAI.back.core.enums.metrics import SplitEnum +from DashAI.back.core.utils import MultilingualString from DashAI.back.evaluation.cv import FoldEvaluationStrategy @@ -21,5 +22,42 @@ class ForecastingCrossValidationEvaluationStrategy(FoldEvaluationStrategy): origin forward through time. """ + DESCRIPTION = MultilingualString( + en=( + "Walks the origin forward through the series. Each fold trains on " + "everything up to a point and is scored on the rows just after it, " + "then the origin moves on and the model is refitted with more " + "history. Training partitions are not scored." + ), + es=( + "Avanza el origen a lo largo de la serie. Cada pliegue entrena con " + "todo lo anterior a un punto y se evalua con las filas justo " + "posteriores; luego el origen avanza y el modelo se reajusta con mas " + "historia. Las particiones de entrenamiento no se evaluan." + ), + pt=( + "Avanca a origem ao longo da serie. Cada dobra treina com tudo o que " + "vem antes de um ponto e e avaliada nas linhas logo depois; entao a " + "origem avanca e o modelo e reajustado com mais historico. As " + "particoes de treino nao sao avaliadas." + ), + de=( + "Schiebt den Ursprung durch die Zeitreihe. Jeder Fold trainiert auf " + "allem bis zu einem Punkt und wird auf den unmittelbar folgenden " + "Zeilen bewertet, dann rueckt der Ursprung weiter und das Modell " + "wird mit mehr Historie neu angepasst. Trainingspartitionen werden " + "nicht bewertet." + ), + zh=( + "让起点沿序列向前推进。" + "每一折用某个时点之前的" + "全部数据训练,并在紧随" + "其后的行上评分;随后起" + "点前移,模型用更多历史" + "重新拟合。训练部分不参" + "与评分。" + ), + ) + COMPATIBLE_COMPONENTS = ["ForecastingTask"] SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST) diff --git a/DashAI/back/evaluation/forecasting_holdout.py b/DashAI/back/evaluation/forecasting_holdout.py index dba6abf06..0620456c8 100644 --- a/DashAI/back/evaluation/forecasting_holdout.py +++ b/DashAI/back/evaluation/forecasting_holdout.py @@ -1,6 +1,7 @@ """Holdout evaluation for models that forecast a series from its own history.""" from DashAI.back.core.enums.metrics import SplitEnum +from DashAI.back.core.utils import MultilingualString from DashAI.back.evaluation.holdout import SinglePartitionEvaluationStrategy @@ -44,5 +45,44 @@ class ForecastingHoldoutEvaluationStrategy(SinglePartitionEvaluationStrategy): they must not be fitted on it. """ + DESCRIPTION = MultilingualString( + en=( + "Cuts the series once, in time order: the model trains on the " + "earliest rows and is scored on the ones that come after. The " + "training partition is not scored, since a forecaster asked about " + "dates it was fitted on reports a fit, not a forecast." + ), + es=( + "Corta la serie una sola vez, en orden temporal: el modelo entrena " + "con las filas mas antiguas y se evalua con las que vienen despues. " + "La particion de entrenamiento no se evalua, porque preguntarle a un " + "pronosticador por fechas con las que fue ajustado da un ajuste, no " + "un pronostico." + ), + pt=( + "Corta a serie uma unica vez, em ordem temporal: o modelo treina nas " + "linhas mais antigas e e avaliado nas que vem depois. A particao de " + "treino nao e avaliada, porque perguntar a um previsor sobre datas " + "em que ele foi ajustado da um ajuste, nao uma previsao." + ), + de=( + "Teilt die Zeitreihe ein einziges Mal in zeitlicher Reihenfolge: Das " + "Modell trainiert auf den fruehesten Zeilen und wird auf den " + "folgenden bewertet. Die Trainingspartition wird nicht bewertet, " + "denn ein Prognosemodell, das nach Daten seiner eigenen Anpassung " + "gefragt wird, liefert eine Anpassung und keine Prognose." + ), + zh=( + "按时间顺序只切分序列" + "一次:模型在最早的行" + "上训练,并在其后的行" + "上评分。训练部分不参" + "与评分,因为让预测模" + "型回答它自己拟合过的" + "日期,得到的是拟合而" + "不是预测。" + ), + ) + COMPATIBLE_COMPONENTS = ["ForecastingTask"] SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST) diff --git a/DashAI/back/evaluation/holdout.py b/DashAI/back/evaluation/holdout.py index 6fb602ccc..35edd01bc 100644 --- a/DashAI/back/evaluation/holdout.py +++ b/DashAI/back/evaluation/holdout.py @@ -1,4 +1,5 @@ from DashAI.back.core.enums.metrics import LevelEnum, SplitEnum +from DashAI.back.core.utils import MultilingualString from DashAI.back.dependencies.database.models import Metric, Run from DashAI.back.evaluation.base_evaluation_strategy import BaseEvaluationStrategy @@ -180,6 +181,37 @@ class HoldoutEvaluationStrategy(SinglePartitionEvaluationStrategy): partition alone, so it is the model the recorded metrics describe. """ + DESCRIPTION = MultilingualString( + en=( + "Holdout cuts the dataset once. The model trains on one part and is " + "scored on rows it never saw. Fast, but the score depends on which " + "rows happened to land where." + ), + es=( + "Holdout corta el conjunto una sola vez. El modelo entrena con una " + "parte y se evalua con filas que nunca vio. Es rapido, pero el " + "resultado depende de que filas cayeron en cada parte." + ), + pt=( + "O holdout corta o conjunto uma unica vez. O modelo treina em uma " + "parte e e avaliado em linhas que nunca viu. E rapido, mas o " + "resultado depende de quais linhas cairam em cada parte." + ), + de=( + "Holdout teilt den Datensatz ein einziges Mal. Das Modell trainiert " + "auf einem Teil und wird auf Zeilen bewertet, die es nie gesehen " + "hat. Schnell, aber das Ergebnis haengt davon ab, welche Zeilen wo " + "gelandet sind." + ), + zh=( + "留出法只切分数据集一次。" + "模型在一部分上训练," + "并在从未见过的行上评分。" + "速度快,但结果取决于" + "哪些行落在了哪一部分。" + ), + ) + COMPATIBLE_COMPONENTS = [ "TabularClassificationTask", "TextClassificationTask", diff --git a/DashAI/back/splitters/base_splitter.py b/DashAI/back/splitters/base_splitter.py index 908a474b3..97ac740f5 100644 --- a/DashAI/back/splitters/base_splitter.py +++ b/DashAI/back/splitters/base_splitter.py @@ -22,12 +22,6 @@ class BaseSplitter(ConfigObject, metaclass=ABCMeta): TYPE: Final[str] = "Splitter" - # How this splitter carves the dataset, which decides the evaluation - # strategy it belongs to and therefore where the frontend may offer it. - # "holdout" splitters produce one set of partitions; "folds" splitters - # produce several train and validation pairs. Without this the frontend - # cannot tell the two apart and has to hardcode a splitter name, which is - # how a shuffling splitter ended up being offered for time series. PARTITIONING: str = "holdout" # Name of the partition the model was fitted on. Every other partition a @@ -35,18 +29,29 @@ class BaseSplitter(ConfigObject, metaclass=ABCMeta): # carry an explanation when the test partition came out empty. TRAINING_PARTITION: str = "train" + GEOMETRY: str = "unknown" + @classmethod def get_metadata(cls) -> Dict[str, Any]: """Return metadata describing how this splitter carves the dataset. + ``geometry`` names the shape of the carve rather than the splitter, so + the frontend can draw a preview of the splits without knowing which + splitter produced them. It is declared rather than inferred from the + schema parameters: two splitters can take the same ``n_splits`` and + still lay their folds out differently, and a preview that guesses would + draw a confident picture of the wrong thing. A splitter whose shape has + no renderer yet leaves the default, and the frontend draws nothing. + Returns ------- Dict[str, Any] Mapping with ``partitioning``, which the frontend uses to decide whether the splitter belongs to the holdout or the - cross-validation strategy. + cross-validation strategy, and ``geometry``, which it uses to + preview the splits. """ - return {"partitioning": cls.PARTITIONING} + return {"partitioning": cls.PARTITIONING, "geometry": cls.GEOMETRY} @classmethod def explainable_partitions( diff --git a/DashAI/back/splitters/fold_splitter.py b/DashAI/back/splitters/fold_splitter.py index 7b867e51b..8ce9fc90a 100644 --- a/DashAI/back/splitters/fold_splitter.py +++ b/DashAI/back/splitters/fold_splitter.py @@ -52,6 +52,8 @@ class FoldSplitter(BaseSplitter): # cross-validation strategy instead of the holdout one. PARTITIONING: str = "folds" + GEOMETRY: str = "blocked_folds" + # How the rows of the test set are chosen. ``"random"`` samples them # uniformly, ``"stratified"`` preserves the target distribution, and # ``"group"`` moves whole groups so a group never spans the carve. diff --git a/DashAI/back/splitters/group_k_fold.py b/DashAI/back/splitters/group_k_fold.py index 1473ad5a9..bd484fae1 100644 --- a/DashAI/back/splitters/group_k_fold.py +++ b/DashAI/back/splitters/group_k_fold.py @@ -210,6 +210,47 @@ class GroupKFoldSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["GroupKFoldSplitter", "StratifiedGroupKFoldSplitter"] SCHEMA = GroupKFoldSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "K folds that never split a group across the train and " + "validation sides. Use it when several rows describe the same " + "subject, so a model is never scored on a subject it was " + "trained on. Grouping changes which rows land in each fold, " + "not how many, and the rows held out of the folds are whole " + "groups." + ), + es=( + "K pliegues que nunca reparten un grupo entre entrenamiento y " + "validacion. Usalo cuando varias filas describen al mismo " + "sujeto, para que un modelo nunca se evalue con un sujeto con " + "el que fue entrenado. Agrupar cambia que filas caen en cada " + "pliegue, no cuantas, y las filas reservadas fuera de los " + "pliegues son grupos enteros." + ), + pt=( + "K dobras que nunca separam um grupo entre treino e " + "validacao. Use quando varias linhas descrevem o mesmo " + "sujeito, para que o modelo nunca seja avaliado num sujeito " + "com que foi treinado. Agrupar muda quais linhas caem em cada " + "dobra, nao quantas, e as linhas reservadas fora das dobras " + "sao grupos inteiros." + ), + de=( + "K Folds, die eine Gruppe nie zwischen Training und " + "Validierung aufteilen. Sinnvoll, wenn mehrere Zeilen " + "dasselbe Subjekt beschreiben, damit ein Modell nie auf einem " + "Subjekt bewertet wird, mit dem es trainiert wurde. " + "Gruppieren aendert, welche Zeilen in welchen Fold fallen, " + "nicht wie viele, und die zurueckgelegten Zeilen sind ganze " + "Gruppen." + ), + zh=( + "k 个折不会把同一组拆到训练和验证两边。当多行描述同一个" + "对象时使用,以免模型在已经训练过的对象上被评分。分组改变" + "的是哪些行落入每一折,而不是有多少行,预留出来的行都是完" + "整的组。" + ), + ) def __init__(self, splits_data): """Initialize the group-based K-fold splitter. diff --git a/DashAI/back/splitters/holdout.py b/DashAI/back/splitters/holdout.py index c147548d9..633227bf2 100644 --- a/DashAI/back/splitters/holdout.py +++ b/DashAI/back/splitters/holdout.py @@ -192,6 +192,8 @@ class PartitionSplitter(BaseSplitter): - https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html """ + GEOMETRY: str = "partitions" + @classmethod def explainable_partitions(cls, split_indexes): """Return the train, test and validation partitions of a holdout run. @@ -374,6 +376,42 @@ class HoldoutSplitter(PartitionSplitter): """ SCHEMA = HoldoutSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "Cuts the dataset into train, validation and test in the " + "proportions you set. Rows are sampled at random unless you " + "turn shuffling off, and can be drawn keeping the class " + "balance of the full dataset. The usual choice when a single " + "split is enough." + ), + es=( + "Corta el conjunto en entrenamiento, validacion y prueba con " + "las proporciones que elijas. Las filas se toman al azar " + "salvo que desactives la mezcla, y pueden tomarse conservando " + "el balance de clases del conjunto completo. Es la opcion " + "habitual cuando basta con un solo corte." + ), + pt=( + "Corta o conjunto em treino, validacao e teste nas proporcoes " + "que voce escolher. As linhas sao sorteadas ao acaso a menos " + "que voce desative o embaralhamento, e podem ser sorteadas " + "mantendo o balanco de classes do conjunto completo. E a " + "escolha usual quando um unico corte basta." + ), + de=( + "Teilt den Datensatz in den gewaehlten Anteilen in Training, " + "Validierung und Test. Die Zeilen werden zufaellig gezogen, " + "sofern das Mischen nicht abgeschaltet ist, und koennen unter " + "Beibehaltung der Klassenverteilung des gesamten Datensatzes " + "gezogen werden. Die uebliche Wahl, wenn eine einzelne " + "Teilung genuegt." + ), + zh=( + "按你设定的比例把数据集切成训练、验证和测试三部分。除非关" + "闭打乱,行都是随机抽取的,并且可以在抽取时保持完整数据集" + "的类别比例。只需一次切分时的常用选择。" + ), + ) # Listed per task rather than left universal, so the frontend can resolve # a holdout splitter from the task instead of hardcoding this class. COMPATIBLE_COMPONENTS = [ diff --git a/DashAI/back/splitters/k_fold.py b/DashAI/back/splitters/k_fold.py index f6baf3558..70ddd155e 100644 --- a/DashAI/back/splitters/k_fold.py +++ b/DashAI/back/splitters/k_fold.py @@ -169,6 +169,40 @@ class KFoldSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["KFoldSplitter", "StratifiedKFoldSplitter"] SCHEMA = KFoldSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "Divides the rows into k equal folds. Each fold is scored " + "once while the other k-1 train the model. The plain cross " + "validation split, with optional shuffling. Rows held out of " + "the folds are sampled from the whole dataset." + ), + es=( + "Divide las filas en k pliegues iguales. Cada pliegue se " + "evalua una vez mientras los otros k-1 entrenan el modelo. Es " + "el corte de validacion cruzada simple, con mezcla opcional. " + "Las filas reservadas fuera de los pliegues se toman al azar " + "de todo el conjunto." + ), + pt=( + "Divide as linhas em k dobras iguais. Cada dobra e avaliada " + "uma vez enquanto as outras k-1 treinam o modelo. E o corte " + "de validacao cruzada simples, com embaralhamento opcional. " + "As linhas reservadas fora das dobras sao sorteadas de todo o " + "conjunto." + ), + de=( + "Teilt die Zeilen in k gleich grosse Folds. Jeder Fold wird " + "einmal bewertet, waehrend die anderen k-1 das Modell " + "trainieren. Die einfache Kreuzvalidierung, auf Wunsch mit " + "Mischen. Die aus den Folds zurueckgelegten Zeilen werden aus " + "dem gesamten Datensatz gezogen." + ), + zh=( + "把行分成 k 个大小相等的折。每一折被评分一次,其余 k" + "-1 折用于训练模型。最基本的交叉验证切分,可选是否打乱" + "顺序。从各折中预留出来的行是从整个数据集中随机抽取的。" + ), + ) def split_indexes( self, x: DashAIDataset, y: DashAIDataset diff --git a/DashAI/back/splitters/leave_one_out.py b/DashAI/back/splitters/leave_one_out.py index 487238f66..6ff1dcaed 100644 --- a/DashAI/back/splitters/leave_one_out.py +++ b/DashAI/back/splitters/leave_one_out.py @@ -113,6 +113,41 @@ class LeaveOneOutSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["KFoldSplitter", "StratifiedKFoldSplitter"] SCHEMA = LeaveOneOutSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "One fold per row: the model is refitted for every row and " + "scored on that row alone. Almost unbiased, but it trains as " + "many models as there are rows, so keep it for small " + "datasets. Rows held out of the folds are sampled from the " + "whole dataset." + ), + es=( + "Un pliegue por fila: el modelo se reajusta para cada fila y " + "se evalua solo con esa fila. Casi no tiene sesgo, pero " + "entrena tantos modelos como filas haya, asi que conviene " + "reservarlo para conjuntos pequenos. Las filas reservadas " + "fuera de los pliegues se toman al azar de todo el conjunto." + ), + pt=( + "Uma dobra por linha: o modelo e reajustado para cada linha e " + "avaliado apenas nela. Quase nao tem vies, mas treina tantos " + "modelos quantas forem as linhas, entao guarde-o para " + "conjuntos pequenos. As linhas reservadas fora das dobras sao " + "sorteadas de todo o conjunto." + ), + de=( + "Ein Fold je Zeile: Das Modell wird fuer jede Zeile neu " + "angepasst und allein auf dieser Zeile bewertet. Nahezu " + "unverzerrt, trainiert aber so viele Modelle wie es Zeilen " + "gibt, also nur fuer kleine Datensaetze. Die zurueckgelegten " + "Zeilen werden aus dem gesamten Datensatz gezogen." + ), + zh=( + "每行一折:模型为每一行重新拟合,并仅在该行上评分。几乎无" + "偏,但要训练与行数相同的模型,因此只适合小数据集。从各折" + "中预留出来的行是从整个数据集中随机抽取的。" + ), + ) def split_indexes( self, x: DashAIDataset, y: DashAIDataset diff --git a/DashAI/back/splitters/repeated_k_fold.py b/DashAI/back/splitters/repeated_k_fold.py index 7ad19220c..c38cecfbf 100644 --- a/DashAI/back/splitters/repeated_k_fold.py +++ b/DashAI/back/splitters/repeated_k_fold.py @@ -175,6 +175,51 @@ class RepeatedKFoldSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["KFoldSplitter", "StratifiedKFoldSplitter"] SCHEMA = RepeatedKFoldSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "Runs the whole k-fold split several times over different " + "random cuts and scores every fold. Rows are shuffled before " + "each repeat, so no two repeats cut alike. It costs one " + "training run per fold per repeat, and in return the average " + "depends far less on any single cut. Rows held out of the " + "folds are sampled from the whole dataset." + ), + es=( + "Repite el corte k-fold completo varias veces sobre cortes " + "aleatorios distintos y evalua cada pliegue. Las filas se " + "mezclan antes de cada repeticion, asi que no hay dos " + "repeticiones que corten igual. Cuesta un entrenamiento por " + "pliegue y repeticion, y a cambio el promedio depende mucho " + "menos de un solo corte. Las filas reservadas fuera de los " + "pliegues se toman al azar de todo el conjunto." + ), + pt=( + "Repete o corte k-fold inteiro varias vezes sobre cortes " + "aleatorios diferentes e avalia cada dobra. As linhas sao " + "embaralhadas antes de cada repeticao, entao nao ha duas " + "repeticoes que cortem igual. Custa um treino por dobra e " + "repeticao, e em troca a media depende bem menos de um unico " + "corte. As linhas reservadas fora das dobras sao sorteadas de " + "todo o conjunto." + ), + de=( + "Fuehrt die gesamte k-Fold-Teilung mehrfach ueber " + "unterschiedliche Zufallsteilungen aus und bewertet jeden " + "Fold. Die Zeilen werden vor jeder Wiederholung gemischt, " + "keine zwei Wiederholungen teilen also gleich. Das kostet " + "einen Trainingslauf je Fold und Wiederholung, dafuer haengt " + "der Mittelwert weit weniger von einer einzelnen Teilung ab. " + "Die zurueckgelegten Zeilen werden aus dem gesamten Datensatz " + "gezogen." + ), + zh=( + "在不同的随机切分上重复整个 k 折切分多次,并对每一折评" + "分。每次重复前都会打乱行顺序,因此没有两次重复的切分相同" + "。代价是每折每次重复都要训练一次,回报是平均分对单次切分" + "的依赖大幅降低。从各折中预留出来的行是从整个数据集中随机" + "抽取的。" + ), + ) def __init__(self, splits_data): """Initialize the repeated K-fold splitter. diff --git a/DashAI/back/splitters/repeated_stratified_k_fold.py b/DashAI/back/splitters/repeated_stratified_k_fold.py index f04f23c33..328c149bd 100644 --- a/DashAI/back/splitters/repeated_stratified_k_fold.py +++ b/DashAI/back/splitters/repeated_stratified_k_fold.py @@ -174,6 +174,41 @@ class RepeatedStratifiedKFoldSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["KFoldSplitter", "StratifiedKFoldSplitter"] SCHEMA = RepeatedStratifiedKFoldSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "Repeats a stratified k-fold split over several random cuts, " + "keeping the class balance in every fold of every repeat. " + "Stratifying changes which rows land in each fold, not how " + "many, and the rows held out of the folds keep that same " + "balance." + ), + es=( + "Repite un corte k-fold estratificado sobre varios cortes " + "aleatorios, conservando el balance de clases en cada pliegue " + "de cada repeticion. Estratificar cambia que filas caen en " + "cada pliegue, no cuantas, y las filas reservadas fuera de " + "los pliegues conservan ese mismo balance." + ), + pt=( + "Repete um corte k-fold estratificado sobre varios cortes " + "aleatorios, mantendo o balanco de classes em cada dobra de " + "cada repeticao. Estratificar muda quais linhas caem em cada " + "dobra, nao quantas, e as linhas reservadas fora das dobras " + "mantem esse mesmo balanco." + ), + de=( + "Wiederholt eine stratifizierte k-Fold-Teilung ueber mehrere " + "Zufallsteilungen und behaelt die Klassenverteilung in jedem " + "Fold jeder Wiederholung bei. Stratifizieren aendert, welche " + "Zeilen in welchen Fold fallen, nicht wie viele, und die " + "zurueckgelegten Zeilen behalten dieselbe Verteilung." + ), + zh=( + "在多个随机切分上重复分层 k 折切分,每次重复的每一" + "折都保持类别比例。分层改变的是哪些行落入每一折,而不" + "是有多少行,预留出来的行也保持同样的比例。" + ), + ) def __init__(self, splits_data): """Initialize the repeated stratified K-fold splitter. diff --git a/DashAI/back/splitters/rolling_origin.py b/DashAI/back/splitters/rolling_origin.py index 1b6059510..c425e8441 100644 --- a/DashAI/back/splitters/rolling_origin.py +++ b/DashAI/back/splitters/rolling_origin.py @@ -150,6 +150,48 @@ class RollingOriginSplitter(FoldSplitter): """ SCHEMA = RollingOriginSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "Walks the origin forward through a series. Each fold trains " + "on everything up to a point and is scored on the next " + "horizon rows, then the origin advances by step and the model " + "is refitted with more history. Rows keep their original " + "order, and the rows held out of the folds are the tail of " + "the series." + ), + es=( + "Avanza el origen a lo largo de una serie. Cada pliegue " + "entrena con todo lo anterior a un punto y se evalua con las " + "siguientes filas del horizonte; luego el origen avanza segun " + "el paso y el modelo se reajusta con mas historia. Las filas " + "conservan su orden original y las reservadas fuera de los " + "pliegues son la cola de la serie." + ), + pt=( + "Avanca a origem ao longo de uma serie. Cada dobra treina com " + "tudo o que vem antes de um ponto e e avaliada nas proximas " + "linhas do horizonte; entao a origem avanca conforme o passo " + "e o modelo e reajustado com mais historico. As linhas mantem " + "a ordem original e as reservadas fora das dobras sao a cauda " + "da serie." + ), + de=( + "Schiebt den Ursprung durch eine Zeitreihe. Jeder Fold " + "trainiert auf allem bis zu einem Punkt und wird auf den " + "naechsten Zeilen des Horizonts bewertet, dann rueckt der " + "Ursprung um die Schrittweite weiter und das Modell wird mit " + "mehr Historie neu angepasst. Die Zeilen behalten ihre " + "urspruengliche Reihenfolge, und die zurueckgelegten Zeilen " + "sind das Ende der Zeitreihe." + ), + zh=( + "让起点沿序列向前推进。每一折用某个时点之前的全部数据训练" + ",并在接下来的预测跨度行上评分;随后起点按步长前移,模型" + "用更多历史重新拟合。行保持原始顺序,从各折中预留出来的行" + "是序列的尾部。" + ), + ) + GEOMETRY: str = "expanding_window" TEST_SPLIT_STRATEGY: str = "temporal" COMPATIBLE_COMPONENTS = ["ForecastingTask"] COMPATIBLE_INNER_SPLITTERS = ["RollingOriginSplitter"] diff --git a/DashAI/back/splitters/stratified_group_k_fold.py b/DashAI/back/splitters/stratified_group_k_fold.py index 04b34e176..c29713cb9 100644 --- a/DashAI/back/splitters/stratified_group_k_fold.py +++ b/DashAI/back/splitters/stratified_group_k_fold.py @@ -205,6 +205,45 @@ class StratifiedGroupKFoldSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["GroupKFoldSplitter", "StratifiedGroupKFoldSplitter"] SCHEMA = StratifiedGroupKFoldSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "K folds that keep whole groups on one side and still " + "approximate the class balance of the full dataset. Use it " + "when rows share a subject and a class is also rare. This " + "changes which rows land in each fold, not how many, and the " + "rows held out of the folds are whole groups." + ), + es=( + "K pliegues que mantienen cada grupo entero de un solo lado y " + "aun asi aproximan el balance de clases del conjunto " + "completo. Usalo cuando las filas comparten sujeto y ademas " + "hay una clase poco frecuente. Esto cambia que filas caen en " + "cada pliegue, no cuantas, y las filas reservadas fuera de " + "los pliegues son grupos enteros." + ), + pt=( + "K dobras que mantem cada grupo inteiro de um so lado e ainda " + "aproximam o balanco de classes do conjunto completo. Use " + "quando as linhas compartilham um sujeito e ha tambem uma " + "classe rara. Isso muda quais linhas caem em cada dobra, nao " + "quantas, e as linhas reservadas fora das dobras sao grupos " + "inteiros." + ), + de=( + "K Folds, die ganze Gruppen auf einer Seite halten und dabei " + "die Klassenverteilung des gesamten Datensatzes annaehernd " + "erhalten. Sinnvoll, wenn Zeilen ein Subjekt teilen und " + "zugleich eine Klasse selten ist. Das aendert, welche Zeilen " + "in welchen Fold fallen, nicht wie viele, und die " + "zurueckgelegten Zeilen sind ganze Gruppen." + ), + zh=( + "k 个折既把整组保留在同一边,又尽量接近完整数据集的类别" + "比例。当多行共享同一对象且某类别又罕见时使用。这改变的是" + "哪些行落入每一折,而不是有多少行,预留出来的行都是完整的" + "组。" + ), + ) def __init__(self, splits_data): """Initialize the stratified group-based K-fold splitter. diff --git a/DashAI/back/splitters/stratified_k_fold.py b/DashAI/back/splitters/stratified_k_fold.py index 23fe2cada..4b24ebd5b 100644 --- a/DashAI/back/splitters/stratified_k_fold.py +++ b/DashAI/back/splitters/stratified_k_fold.py @@ -167,6 +167,44 @@ class StratifiedKFoldSplitter(FoldSplitter): ) COMPATIBLE_INNER_SPLITTERS = ["KFoldSplitter", "StratifiedKFoldSplitter"] SCHEMA = StratifiedKFoldSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "K folds that each keep the class balance of the full " + "dataset. Use it when a class is rare enough that an ordinary " + "fold could miss it entirely. Stratifying changes which rows " + "land in each fold, not how many, and the rows held out of " + "the folds keep that same balance." + ), + es=( + "K pliegues que conservan el balance de clases del conjunto " + "completo. Usalo cuando una clase es tan poco frecuente que " + "un pliegue corriente podria no contener ninguna de sus " + "filas. Estratificar cambia que filas caen en cada pliegue, " + "no cuantas, y las filas reservadas fuera de los pliegues " + "conservan ese mismo balance." + ), + pt=( + "K dobras que mantem o balanco de classes do conjunto " + "completo. Use quando uma classe e rara o bastante para que " + "uma dobra comum possa nao conter nenhuma de suas linhas. " + "Estratificar muda quais linhas caem em cada dobra, nao " + "quantas, e as linhas reservadas fora das dobras mantem esse " + "mesmo balanco." + ), + de=( + "K Folds, die jeweils die Klassenverteilung des gesamten " + "Datensatzes beibehalten. Sinnvoll, wenn eine Klasse so " + "selten ist, dass ein gewoehnlicher Fold sie ganz verfehlen " + "koennte. Stratifizieren aendert, welche Zeilen in welchen " + "Fold fallen, nicht wie viele, und die zurueckgelegten Zeilen " + "behalten dieselbe Verteilung." + ), + zh=( + "k 个折都保持完整数据集的类别比例。当某一类别罕见到普通" + "切分可能完全遗漏它时使用。分层改变的是哪些行落入每一折," + "而不是有多少行,预留出来的行也保持同样的比例。" + ), + ) def split_indexes( self, x: DashAIDataset, y: DashAIDataset diff --git a/DashAI/back/splitters/temporal_holdout.py b/DashAI/back/splitters/temporal_holdout.py index 341058e92..b23986bfe 100644 --- a/DashAI/back/splitters/temporal_holdout.py +++ b/DashAI/back/splitters/temporal_holdout.py @@ -131,6 +131,39 @@ class TemporalHoldoutSplitter(PartitionSplitter): """ SCHEMA = TemporalHoldoutSplitterSchema + DESCRIPTION = MultilingualString( + en=( + "Cuts a series in time order: the earliest rows train, the " + "next ones validate and the last ones test. Never shuffles, " + "so a model is never fitted on rows that come after the ones " + "it is scored on." + ), + es=( + "Corta una serie en orden temporal: las filas mas antiguas " + "entrenan, las siguientes validan y las ultimas prueban. " + "Nunca mezcla, asi que un modelo jamas se ajusta con filas " + "posteriores a aquellas con las que se evalua." + ), + pt=( + "Corta uma serie em ordem temporal: as linhas mais antigas " + "treinam, as seguintes validam e as ultimas testam. Nunca " + "embaralha, entao um modelo jamais e ajustado com linhas " + "posteriores as que servem para avalia-lo." + ), + de=( + "Teilt eine Zeitreihe in zeitlicher Reihenfolge: Die " + "fruehesten Zeilen trainieren, die naechsten validieren und " + "die letzten testen. Es wird nie gemischt, ein Modell wird " + "also nie auf Zeilen angepasst, die nach den bewerteten " + "liegen." + ), + zh=( + "按时间顺序切分序列:最早的行用于训练,接下来的用于验" + "证,最后的用于测试。从不打乱顺序,因此模型绝不会用评" + "分行之后的数据拟合。" + ), + ) + GEOMETRY: str = "sequential_partitions" COMPATIBLE_COMPONENTS = ["ForecastingTask"] DISPLAY_NAME: str = MultilingualString( en="Temporal Holdout", diff --git a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx index 6315d7314..3f73f7b6e 100644 --- a/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx +++ b/DashAI/front/src/components/models/modelSession/SplitDatasetRows.jsx @@ -15,9 +15,11 @@ import { MenuItem, FormControl, } from "@mui/material"; -import { DescriptionBlock } from "../../shared/FormSchemaFieldCard"; import FormSchema from "../../shared/FormSchema"; import FormSchemaLayout from "../../shared/FormSchemaLayout"; +import SplitsCard from "./SplitsCard"; +import SplitPreview from "./SplitPreview"; +import { geometryOf } from "../../../utils/splitPreview"; import { defaultHoldoutSplitter, filterByPartitioning, @@ -32,50 +34,6 @@ import { useTranslation } from "react-i18next"; import { useSnackbar } from "notistack"; import { getComponents } from "../../../api/component"; -/** - * Splits card shell — same Paper/header visual as FormSchemaFieldCard but WITHOUT - * the label-hiding CSS so Train / Validation / Test TextField labels stay visible. - */ -function SplitsCard({ label, description, errorMessage, children, warning }) { - return ( - - - - {label} - - - - {children} - - {(description || errorMessage || warning) && ( - - - - )} - - ); -} - function SplitDatasetRows({ datasetInfo, rowsPartitionsIndex, @@ -123,6 +81,8 @@ function SplitDatasetRows({ // The splitter's own parameters come from the schema generated form; only the // rules the schema cannot express are checked here. const splitterName = resolveSplitterName(strategyKind, cvType, holdoutType); + const selectedSplitter = + strategyKind === STRATEGY_KINDS.HOLDOUT ? holdoutType : cvType; const isIndexMode = splitType === SPLIT_TYPES.MANUAL || splitType === SPLIT_TYPES.PREDEFINED; const params = splitterParams ?? {}; @@ -253,23 +213,23 @@ function SplitDatasetRows({ setHoldoutType(defaultHoldoutSplitter(allowedHoldoutTypes)); }, [allowedHoldoutTypes]); + const selectedStrategy = findStrategy(allowedStrategies, evaluationStrategy); + // Publish the split shape so the rest of the session reads it instead of // comparing strategy names. useEffect(() => { - setStrategyKind( - strategyKindOf(findStrategy(allowedStrategies, evaluationStrategy)), - ); - }, [allowedStrategies, evaluationStrategy, setStrategyKind]); + setStrategyKind(strategyKindOf(selectedStrategy)); + }, [selectedStrategy, setStrategyKind]); // And for the strategy itself, which the session used to start on by name. useEffect(() => { if (!allowedStrategies.length) return; - if (findStrategy(allowedStrategies, evaluationStrategy)) return; + if (selectedStrategy) return; const holdout = allowedStrategies.find( (strategy) => strategyKindOf(strategy) === STRATEGY_KINDS.HOLDOUT, ); setEvaluationStrategy((holdout ?? allowedStrategies[0]).name); - }, [allowedStrategies, evaluationStrategy, setEvaluationStrategy]); + }, [allowedStrategies, selectedStrategy, setEvaluationStrategy]); const handleSplitTypeChange = (_e, newType) => { if (!newType) return; @@ -596,6 +556,16 @@ function SplitDatasetRows({ )} + + {/* Splitter parameters, generated from the component schema */} {splitterName && ( <> diff --git a/DashAI/front/src/components/models/modelSession/SplitPreview.jsx b/DashAI/front/src/components/models/modelSession/SplitPreview.jsx new file mode 100644 index 000000000..b02d9320f --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/SplitPreview.jsx @@ -0,0 +1,234 @@ +import React, { useMemo } from "react"; +import PropTypes from "prop-types"; +import { Box, Stack, Tooltip, Typography } from "@mui/material"; +import { useTheme } from "@mui/material/styles"; +import { useTranslation } from "react-i18next"; +import SplitsCard from "./SplitsCard"; +import { buildSplitPreview, SPLIT_ROLES } from "../../../utils/splitPreview"; + +const LEGEND_ORDER = [ + SPLIT_ROLES.TRAIN, + SPLIT_ROLES.VALIDATION, + SPLIT_ROLES.TEST, + SPLIT_ROLES.UNUSED, +]; + +const roleColors = (theme) => { + const [green, , orange] = theme.palette.chart?.palette ?? []; + return { + [SPLIT_ROLES.TRAIN]: theme.palette.primary.main, + [SPLIT_ROLES.VALIDATION]: orange ?? theme.palette.warning.main, + [SPLIT_ROLES.TEST]: green ?? theme.palette.success.main, + [SPLIT_ROLES.UNUSED]: theme.palette.action.disabledBackground, + }; +}; + +const asPercent = (fraction) => Math.round((fraction ?? 0) * 100); + +function SplitPreview({ + geometry, + description, + splitterDescription, + splitType, + params, + indexes, + datasetInfo, +}) { + const theme = useTheme(); + const { t } = useTranslation(["experiments", "common"]); + + const preview = useMemo( + () => + buildSplitPreview({ + geometry, + splitType, + params, + indexes, + datasetInfo, + }), + [geometry, splitType, params, indexes, datasetInfo], + ); + + const explainers = [description, splitterDescription].filter(Boolean); + + const colors = roleColors(theme); + + const roleLabel = (role) => + role === SPLIT_ROLES.UNUSED + ? t("experiments:splitPreview.unused") + : t(`common:${role}`); + + if (!preview) { + return ( + + + {explainers.map((explainer) => ( + + {explainer} + + ))} + + {t("experiments:splitPreview.unavailable")} + + + + ); + } + + const { rows, notes, error, shares } = preview; + const legendRoles = LEGEND_ORDER.filter( + (role) => (shares[role] ?? 0) > 0.0001, + ); + + return ( + + + {explainers.length > 0 && ( + + {explainers.map((explainer) => ( + + {explainer} + + ))} + + )} + + {error ? ( + + {t(`experiments:splitPreview.error.${error.key}`, error.values)} + + ) : ( + + {rows.map((row) => + row.ellipsis ? ( + + + + {t("experiments:splitPreview.moreFolds", { + hidden: row.hiddenCount, + })} + + + ) : ( + + + {t(row.labelKey, row.labelValues)} + + + {row.segments.map((piece, index) => ( + + + + ))} + + + ), + )} + + )} + + {!error && legendRoles.length > 0 && ( + + {legendRoles.map((role) => ( + + + + {`${roleLabel(role)} ${asPercent(shares[role])}%`} + + + ))} + + )} + + {notes.length > 0 && ( + + {notes.map((note) => ( + + {t(`experiments:splitPreview.notes.${note.key}`, note.values)} + + ))} + + )} + + + ); +} + +SplitPreview.propTypes = { + geometry: PropTypes.string, + description: PropTypes.string, + splitterDescription: PropTypes.string, + splitType: PropTypes.string, + params: PropTypes.object, + indexes: PropTypes.object, + datasetInfo: PropTypes.object, +}; + +export default SplitPreview; diff --git a/DashAI/front/src/components/models/modelSession/SplitsCard.jsx b/DashAI/front/src/components/models/modelSession/SplitsCard.jsx new file mode 100644 index 000000000..cb29e73ab --- /dev/null +++ b/DashAI/front/src/components/models/modelSession/SplitsCard.jsx @@ -0,0 +1,58 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { Box, Paper, Typography } from "@mui/material"; +import { DescriptionBlock } from "../../shared/FormSchemaFieldCard"; + +/** + * Splits card shell same Paper/header visual as FormSchemaFieldCard but without + * the label hiding CSS so Train / Validation / Test TextField labels stay visible. + */ +function SplitsCard({ label, description, errorMessage, children, warning }) { + return ( + + + + {label} + + + + {children} + + {(description || errorMessage || warning) && ( + + + + )} + + ); +} + +SplitsCard.propTypes = { + label: PropTypes.node, + description: PropTypes.node, + errorMessage: PropTypes.node, + children: PropTypes.node, + warning: PropTypes.bool, +}; + +export default SplitsCard; diff --git a/DashAI/front/src/utils/i18n/locales/de/experiments.json b/DashAI/front/src/utils/i18n/locales/de/experiments.json index 3f091f917..8e8dfa923 100644 --- a/DashAI/front/src/utils/i18n/locales/de/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/de/experiments.json @@ -129,5 +129,21 @@ "trainingSet": "Trainingsmenge", "validationMetrics": "Validierungsmetriken", "validationSet": "Validierungsmenge" + }, + "splitPreview": { + "title": "Vorschau der Aufteilung", + "unavailable": "Dieser Splitter beschreibt seine Form noch nicht, daher gibt es nichts zu zeichnen.", + "wholeDataset": "Datensatz", + "reservedTest": "Zurückgelegt", + "finalFit": "Finales Training", + "fold": "Fold {{index}}", + "moreFolds": "+{{hidden}} weitere Folds", + "unused": "Ungenutzt", + "notes": { + "foldsHidden": "Insgesamt {{total}} Folds; nur die ersten und der letzte werden gezeichnet." + }, + "error": { + "notEnoughRows": "{{folds}} Ursprünge mit einem Horizont von {{horizon}} und einer Schrittweite von {{step}} passen nicht in {{rows}} Zeilen. Verringere einen davon oder lege weniger zurück." + } } } diff --git a/DashAI/front/src/utils/i18n/locales/en/experiments.json b/DashAI/front/src/utils/i18n/locales/en/experiments.json index d166e2a37..a199863ff 100644 --- a/DashAI/front/src/utils/i18n/locales/en/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/en/experiments.json @@ -129,5 +129,21 @@ "trainingSet": "Train set", "validationMetrics": "Validation Metrics", "validationSet": "Validation set" + }, + "splitPreview": { + "title": "Split preview", + "unavailable": "This splitter does not describe its shape yet, so there is nothing to draw.", + "wholeDataset": "Dataset", + "reservedTest": "Held out", + "finalFit": "Final fit", + "fold": "Fold {{index}}", + "moreFolds": "+{{hidden}} more folds", + "unused": "Unused", + "notes": { + "foldsHidden": "{{total}} folds in total; only the first ones and the last are drawn." + }, + "error": { + "notEnoughRows": "{{folds}} origins with a horizon of {{horizon}} and a step of {{step}} do not fit in {{rows}} rows. Lower one of them, or hold out less." + } } } diff --git a/DashAI/front/src/utils/i18n/locales/es/experiments.json b/DashAI/front/src/utils/i18n/locales/es/experiments.json index 53cdeb71b..33abb4ca0 100644 --- a/DashAI/front/src/utils/i18n/locales/es/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/es/experiments.json @@ -129,5 +129,21 @@ "trainingSet": "Conjunto de entrenamiento", "validationMetrics": "Métricas de Validación", "validationSet": "Conjunto de validación" + }, + "splitPreview": { + "title": "Vista previa de la división", + "unavailable": "Este divisor aún no declara su forma, así que no hay nada que dibujar.", + "wholeDataset": "Conjunto", + "reservedTest": "Reservado", + "finalFit": "Ajuste final", + "fold": "Pliegue {{index}}", + "moreFolds": "+{{hidden}} pliegues más", + "unused": "Sin usar", + "notes": { + "foldsHidden": "{{total}} pliegues en total; solo se dibujan los primeros y el último." + }, + "error": { + "notEnoughRows": "{{folds}} orígenes con un horizonte de {{horizon}} y un paso de {{step}} no caben en {{rows}} filas. Reduce alguno o reserva menos." + } } } diff --git a/DashAI/front/src/utils/i18n/locales/pt/experiments.json b/DashAI/front/src/utils/i18n/locales/pt/experiments.json index 5e68b14b8..917e8f339 100644 --- a/DashAI/front/src/utils/i18n/locales/pt/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/pt/experiments.json @@ -129,5 +129,21 @@ "trainingSet": "Conjunto de treinamento", "validationMetrics": "Métricas de Validação", "validationSet": "Conjunto de validação" + }, + "splitPreview": { + "title": "Prévia da divisão", + "unavailable": "Este divisor ainda não declara sua forma, então não há nada para desenhar.", + "wholeDataset": "Conjunto", + "reservedTest": "Reservado", + "finalFit": "Ajuste final", + "fold": "Dobra {{index}}", + "moreFolds": "+{{hidden}} dobras a mais", + "unused": "Sem uso", + "notes": { + "foldsHidden": "{{total}} dobras no total; só as primeiras e a última são desenhadas." + }, + "error": { + "notEnoughRows": "{{folds}} origens com um horizonte de {{horizon}} e um passo de {{step}} não cabem em {{rows}} linhas. Reduza um deles ou reserve menos." + } } } diff --git a/DashAI/front/src/utils/i18n/locales/zh/experiments.json b/DashAI/front/src/utils/i18n/locales/zh/experiments.json index e1c6aa541..4c6ed3feb 100644 --- a/DashAI/front/src/utils/i18n/locales/zh/experiments.json +++ b/DashAI/front/src/utils/i18n/locales/zh/experiments.json @@ -129,5 +129,21 @@ "trainingSet": "训练集", "validationMetrics": "验证集指标", "validationSet": "验证集" + }, + "splitPreview": { + "title": "划分预览", + "unavailable": "该拆分器尚未声明其形状,因此无法绘制。", + "wholeDataset": "数据集", + "reservedTest": "预留", + "finalFit": "最终训练", + "fold": "第 {{index}} 折", + "moreFolds": "另有 {{hidden}} 折", + "unused": "未使用", + "notes": { + "foldsHidden": "共 {{total}} 折;只绘制最前面几折和最后一折。" + }, + "error": { + "notEnoughRows": "{{folds}} 个起点、{{horizon}} 的预测跨度和 {{step}} 的步长放不进 {{rows}} 行。请调小其中之一,或减少预留。" + } } } diff --git a/DashAI/front/src/utils/splitPreview.js b/DashAI/front/src/utils/splitPreview.js new file mode 100644 index 000000000..045db96e9 --- /dev/null +++ b/DashAI/front/src/utils/splitPreview.js @@ -0,0 +1,308 @@ +import { SPLIT_TYPES } from "./splitsPayload"; + +export const SPLIT_GEOMETRIES = { + PARTITIONS: "partitions", + SEQUENTIAL_PARTITIONS: "sequential_partitions", + BLOCKED_FOLDS: "blocked_folds", + EXPANDING_WINDOW: "expanding_window", +}; + +export const SPLIT_ROLES = { + TRAIN: "train", + VALIDATION: "validation", + TEST: "test", + UNUSED: "unused", +}; + +export const geometryOf = (splitter) => splitter?.metadata?.geometry ?? null; + +const MAX_DRAWN_FOLDS = 8; +const LEADING_DRAWN_FOLDS = 3; +const NOMINAL_ROWS = 100; +const NEGLIGIBLE = 1e-9; + +const numberOr = (value, fallback) => + typeof value === "number" && Number.isFinite(value) ? value : fallback; + +const clamp01 = (value) => Math.min(1, Math.max(0, numberOr(value, 0))); + +const segment = (role, fraction) => ({ role, fraction }); + +const drawable = (segments) => + segments.filter((piece) => piece.fraction > NEGLIGIBLE); + +const rowsOf = (datasetInfo) => + Math.max( + 1, + Math.round(numberOr(Number(datasetInfo?.total_rows), NOMINAL_ROWS)), + ); + +const proportionsFromDataset = (datasetInfo) => { + const totalRows = Number(datasetInfo?.total_rows); + if (!totalRows) return null; + return { + train: numberOr(datasetInfo?.train_size, 0) / totalRows, + validation: numberOr(datasetInfo?.val_size, 0) / totalRows, + test: numberOr(datasetInfo?.test_size, 0) / totalRows, + }; +}; + +const proportionsFromIndexes = (indexes, datasetInfo) => { + const totalRows = Number(datasetInfo?.total_rows); + if (!totalRows) return null; + return { + train: (indexes?.train?.length ?? 0) / totalRows, + validation: (indexes?.validation?.length ?? 0) / totalRows, + test: (indexes?.test?.length ?? 0) / totalRows, + }; +}; + +const proportionsFromParams = (params) => ({ + train: clamp01(params.train), + validation: clamp01(params.validation), + test: clamp01(params.test), +}); + +const partitionProportions = ({ splitType, params, indexes, datasetInfo }) => { + if (splitType === SPLIT_TYPES.PREDEFINED) { + return proportionsFromDataset(datasetInfo); + } + if (splitType === SPLIT_TYPES.MANUAL) { + return proportionsFromIndexes(indexes, datasetInfo); + } + return proportionsFromParams(params); +}; + +const partitionRow = ({ train, validation, test }) => { + const requested = train + validation + test; + const scale = requested > 1 ? 1 / requested : 1; + const unused = requested < 1 ? 1 - requested : 0; + + return { + key: "dataset", + labelKey: "experiments:splitPreview.wholeDataset", + segments: drawable([ + segment(SPLIT_ROLES.TRAIN, train * scale), + segment(SPLIT_ROLES.VALIDATION, validation * scale), + segment(SPLIT_ROLES.TEST, test * scale), + segment(SPLIT_ROLES.UNUSED, unused), + ]), + }; +}; + +const reservedRow = (pool, reserved) => ({ + key: "reserved", + labelKey: "experiments:splitPreview.reservedTest", + segments: drawable([ + segment(SPLIT_ROLES.UNUSED, pool), + segment(SPLIT_ROLES.TEST, reserved), + ]), +}); + +const finalFitRow = (pool, reserved) => ({ + key: "final", + labelKey: "experiments:splitPreview.finalFit", + segments: drawable([ + segment(SPLIT_ROLES.TRAIN, pool), + segment(SPLIT_ROLES.TEST, reserved), + ]), +}); + +const drawnFoldIndexes = (foldCount) => { + if (foldCount <= MAX_DRAWN_FOLDS) { + return Array.from({ length: foldCount }, (_, index) => index); + } + return [ + ...Array.from({ length: LEADING_DRAWN_FOLDS }, (_, index) => index), + null, + foldCount - 1, + ]; +}; + +const ellipsisRow = (hiddenCount) => ({ + key: "ellipsis", + ellipsis: true, + hiddenCount, +}); + +const foldRows = (foldCount, buildRow) => + drawnFoldIndexes(foldCount).map((index) => + index === null + ? ellipsisRow(foldCount - LEADING_DRAWN_FOLDS - 1) + : buildRow(index), + ); + +const blockedFoldRow = ({ index, foldCount, pool, reserved }) => { + const block = pool / foldCount; + return { + key: "fold-" + index, + labelKey: "experiments:splitPreview.fold", + labelValues: { index: index + 1 }, + segments: drawable([ + segment(SPLIT_ROLES.TRAIN, block * index), + segment(SPLIT_ROLES.VALIDATION, block), + segment(SPLIT_ROLES.TRAIN, block * (foldCount - index - 1)), + segment(SPLIT_ROLES.UNUSED, reserved), + ]), + }; +}; + +const expandingFoldRow = ({ + index, + pool, + reserved, + poolRows, + initialTrainRows, + horizon, + step, +}) => { + const origin = initialTrainRows + index * step; + const train = (origin / poolRows) * pool; + const validation = (horizon / poolRows) * pool; + return { + key: "fold-" + index, + labelKey: "experiments:splitPreview.fold", + labelValues: { index: index + 1 }, + segments: drawable([ + segment(SPLIT_ROLES.TRAIN, train), + segment(SPLIT_ROLES.VALIDATION, validation), + segment(SPLIT_ROLES.UNUSED, pool - train - validation + reserved), + ]), + }; +}; + +const hiddenFoldsNote = (foldCount) => + foldCount > MAX_DRAWN_FOLDS + ? [{ key: "foldsHidden", values: { total: foldCount } }] + : []; + +const partitionsPreview = ({ splitType, params, indexes, datasetInfo }) => { + const proportions = partitionProportions({ + splitType, + params, + indexes, + datasetInfo, + }); + if (!proportions) return null; + + return { rows: [partitionRow(proportions)], notes: [], error: null }; +}; + +const blockedFoldsPreview = ({ params, datasetInfo }) => { + const reserved = clamp01(params.test_size); + const pool = 1 - reserved; + const poolRows = Math.max(1, Math.round(rowsOf(datasetInfo) * pool)); + const declaredFolds = numberOr(params.n_splits, null); + const foldCount = Math.max( + 1, + Math.round(declaredFolds === null ? poolRows : declaredFolds), + ); + + const rows = [ + ...(reserved > NEGLIGIBLE ? [reservedRow(pool, reserved)] : []), + ...foldRows(foldCount, (index) => + blockedFoldRow({ index, foldCount, pool, reserved }), + ), + ...(reserved > NEGLIGIBLE ? [finalFitRow(pool, reserved)] : []), + ]; + + return { rows, notes: hiddenFoldsNote(foldCount), error: null }; +}; + +const expandingWindowPreview = ({ params, datasetInfo }) => { + const reserved = clamp01(params.test_size); + const pool = 1 - reserved; + const poolRows = Math.max(1, Math.round(rowsOf(datasetInfo) * pool)); + const foldCount = Math.max(1, Math.round(numberOr(params.n_splits, 5))); + const horizon = Math.max(1, Math.round(numberOr(params.horizon, 1))); + const step = Math.max(1, Math.round(numberOr(params.step, 1))); + const initialTrainRows = poolRows - horizon - (foldCount - 1) * step; + + if (initialTrainRows <= 0) { + return { + rows: [], + notes: [], + error: { + key: "notEnoughRows", + values: { folds: foldCount, horizon, step, rows: poolRows }, + }, + }; + } + + const rows = [ + ...(reserved > NEGLIGIBLE ? [reservedRow(pool, reserved)] : []), + ...foldRows(foldCount, (index) => + expandingFoldRow({ + index, + pool, + reserved, + poolRows, + initialTrainRows, + horizon, + step, + }), + ), + ...(reserved > NEGLIGIBLE ? [finalFitRow(pool, reserved)] : []), + ]; + + return { rows, notes: hiddenFoldsNote(foldCount), error: null }; +}; + +const sharesOf = (rows) => { + const representative = + rows.find((row) => !row.ellipsis && row.key.startsWith("fold-")) ?? + rows.find((row) => !row.ellipsis); + if (!representative) return {}; + + const shares = {}; + representative.segments.forEach(({ role, fraction }) => { + shares[role] = (shares[role] ?? 0) + fraction; + }); + + const reserved = rows.find((row) => row.key === "reserved"); + const heldOut = reserved?.segments.find( + (piece) => piece.role === SPLIT_ROLES.TEST, + ); + if (heldOut) { + shares[SPLIT_ROLES.TEST] = heldOut.fraction; + shares[SPLIT_ROLES.UNUSED] = Math.max( + 0, + (shares[SPLIT_ROLES.UNUSED] ?? 0) - heldOut.fraction, + ); + } + return shares; +}; + +export const buildSplitPreview = ({ + geometry, + splitType, + params, + indexes, + datasetInfo, +}) => { + const values = params ?? {}; + let preview = null; + + switch (geometry) { + case SPLIT_GEOMETRIES.PARTITIONS: + case SPLIT_GEOMETRIES.SEQUENTIAL_PARTITIONS: + preview = partitionsPreview({ + splitType, + params: values, + indexes, + datasetInfo, + }); + break; + case SPLIT_GEOMETRIES.BLOCKED_FOLDS: + preview = blockedFoldsPreview({ params: values, datasetInfo }); + break; + case SPLIT_GEOMETRIES.EXPANDING_WINDOW: + preview = expandingWindowPreview({ params: values, datasetInfo }); + break; + default: + return null; + } + + if (!preview) return null; + return { ...preview, shares: sharesOf(preview.rows) }; +}; diff --git a/tests/back/splitters/test_splitter_geometry.py b/tests/back/splitters/test_splitter_geometry.py new file mode 100644 index 000000000..db56ac145 --- /dev/null +++ b/tests/back/splitters/test_splitter_geometry.py @@ -0,0 +1,96 @@ +"""The split shape every splitter declares so the frontend can draw it. + +The session sidebar previews the splits a configuration will produce. Which +picture it draws is read from ``geometry`` rather than guessed from the schema +parameters, because guessing gets it wrong without saying so: two splitters can +both take ``n_splits`` and still lay their folds out differently, and a preview +that assumed contiguous blocks would draw a confident picture of a split that +never happens. + +A splitter whose shape has no renderer keeps the ``unknown`` default, and the +frontend draws nothing instead of drawing a lie. These tests pin the values the +renderers exist for. +""" + +import pytest + +from DashAI.back.initial_components import get_initial_components +from DashAI.back.splitters.base_splitter import BaseSplitter +from DashAI.back.splitters.fold_splitter import FoldSplitter +from DashAI.back.splitters.holdout import HoldoutSplitter +from DashAI.back.splitters.k_fold import KFoldSplitter +from DashAI.back.splitters.leave_one_out import LeaveOneOutSplitter +from DashAI.back.splitters.repeated_k_fold import RepeatedKFoldSplitter +from DashAI.back.splitters.rolling_origin import RollingOriginSplitter +from DashAI.back.splitters.temporal_holdout import TemporalHoldoutSplitter + +RENDERED_GEOMETRIES = { + "partitions", + "sequential_partitions", + "blocked_folds", + "expanding_window", +} + + +def _offered_splitters(): + """Collect the splitters a session can actually be pointed at. + + Read from the registration list rather than from the class tree: the tree + also holds the shared bases, which are never offered and have nothing to + declare. + + Returns + ------- + list + The registered splitter classes. + """ + return [ + component + for component in get_initial_components() + if isinstance(component, type) and issubclass(component, BaseSplitter) + ] + + +@pytest.mark.parametrize("splitter", _offered_splitters(), ids=lambda s: s.__name__) +def test_every_splitter_declares_a_geometry_the_frontend_can_draw(splitter): + assert splitter.get_metadata()["geometry"] in RENDERED_GEOMETRIES + + +def test_a_splitter_that_declares_nothing_is_not_drawn(): + assert BaseSplitter.GEOMETRY == "unknown" + assert BaseSplitter.GEOMETRY not in RENDERED_GEOMETRIES + + +@pytest.mark.parametrize( + ("splitter", "geometry"), + [ + (HoldoutSplitter, "partitions"), + (TemporalHoldoutSplitter, "sequential_partitions"), + (KFoldSplitter, "blocked_folds"), + (LeaveOneOutSplitter, "blocked_folds"), + (RepeatedKFoldSplitter, "blocked_folds"), + (RollingOriginSplitter, "expanding_window"), + ], +) +def test_splitters_report_the_shape_they_actually_produce(splitter, geometry): + assert splitter.get_metadata()["geometry"] == geometry + + +def test_the_series_splitters_are_not_drawn_as_shuffled_ones(): + shuffling = HoldoutSplitter.get_metadata()["geometry"] + + for splitter in (TemporalHoldoutSplitter, RollingOriginSplitter): + assert splitter.get_metadata()["geometry"] != shuffling + + +def test_folds_inherit_their_geometry_rather_than_repeating_it(): + assert FoldSplitter.GEOMETRY == "blocked_folds" + assert "geometry" not in vars(KFoldSplitter) + + +def test_geometry_is_added_next_to_what_the_metadata_already_carried(): + metadata = KFoldSplitter.get_metadata() + + assert metadata["partitioning"] == "folds" + assert "compatibleInnerSplitters" in metadata + assert metadata["geometry"] == "blocked_folds"