From 9dee3958ac6e562bea2256a01eb4fa569853dbd0 Mon Sep 17 00:00:00 2001 From: Syed Jafri Date: Mon, 13 Jul 2026 15:53:56 -0700 Subject: [PATCH 1/3] fix: filter full recipe template from serverless train( ) --- .../src/sagemaker/train/base_trainer.py | 90 +++++++++++++++++-- .../src/sagemaker/train/dpo_trainer.py | 9 +- .../sagemaker/train/multi_turn_rl_trainer.py | 8 +- .../src/sagemaker/train/rlvr_trainer.py | 11 ++- .../src/sagemaker/train/sft_trainer.py | 9 +- 5 files changed, 113 insertions(+), 14 deletions(-) diff --git a/sagemaker-train/src/sagemaker/train/base_trainer.py b/sagemaker-train/src/sagemaker/train/base_trainer.py index a453720f53..5edbc671bb 100644 --- a/sagemaker-train/src/sagemaker/train/base_trainer.py +++ b/sagemaker-train/src/sagemaker/train/base_trainer.py @@ -236,19 +236,79 @@ def _patch_resolved_recipe(self, resolved: Dict[str, Any]) -> None: if dotpath: _set_nested_value(resolved, dotpath, value) - def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any]) -> Dict[str, Any]: + def _get_user_provided_recipe_keys(self) -> set: + """Return the set of leaf keys the user explicitly provided. + + Collects keys from every source that represents an explicit user choice: + + * direct hyperparameter assignments (``trainer.hyperparameters.x = val``), + tracked in ``hyperparameters._user_set``; + * the programmatic ``overrides`` dict passed at construction; + * the user recipe YAML file passed at construction. + + These are the keys that should be forwarded to a serverless training job, + alongside the Hub override spec. Full recipe-template internal keys that + the user never touched are intentionally excluded. + + Returns: + Set of leaf key names the user explicitly provided. Empty when the + user provided nothing beyond Hub defaults. + """ + keys: set = set() + + # Direct hyperparameter assignments (always members of the Hub spec). + user_set = getattr(getattr(self, 'hyperparameters', None), '_user_set', None) + if isinstance(user_set, set): + keys.update(user_set) + + # Programmatic overrides dict (may contain non-spec recipe keys). + overrides = getattr(self, '_overrides', None) + if isinstance(overrides, dict) and overrides: + keys.update(flatten_resolved_recipe(overrides).keys()) + + # User recipe YAML file (may contain non-spec recipe keys). + recipe_path = getattr(self, '_recipe_path', None) + if recipe_path: + try: + from sagemaker.train.recipe_resolver import _load_user_recipe + + user_recipe = _load_user_recipe(recipe_path) + keys.update(flatten_resolved_recipe(user_recipe).keys()) + except Exception as e: # pragma: no cover - best-effort key discovery + logger.debug("Could not load user recipe to collect keys: %s", e) + + return keys + + + def _apply_recipe_to_hyperparameters( + self, + final_hyperparameters: Dict[str, Any], + only_user_overrides: bool = False, + ) -> Dict[str, Any]: """Apply resolved recipe values to final_hyperparameters dict. If recipe/overrides were provided, or if the user set hyperparameters directly via ``.hyperparameters.*``, merges resolved recipe values into - the hyperparameters dict. All leaf values from the resolved recipe are - applied — including keys not in the Hub spec subset — enabling - power users to override any parameter in the full recipe. + the hyperparameters dict. By default all leaf values from the resolved + recipe are applied — including keys not in the Hub spec subset. This is + used by the serverful/HyperPod paths, which render the values into a + recipe YAML file (no key-count limit). + Values are converted to strings (matching the SageMaker API expectation for hyperparameter values). Args: final_hyperparameters: The hyperparameters dict from to_dict(). + only_user_overrides: When True, only apply resolved values for keys the + user explicitly provided — direct hyperparameter assignments, the + ``overrides`` dict, or a recipe file — instead of flattening the + entire resolved recipe. This is required for serverless SageMaker + training jobs, whose ``CreateTrainingJob`` ``HyperParameters`` map + is limited to 100 members by the CreateTrainingJob: the resolved full recipe + template can contain several hundred internal leaf keys the user + never touched. User-provided keys are not restricted to the Hub + spec — power users may override any recipe key, and validity is + enforced by ``get_resolved_recipe()``. Returns: The updated hyperparameters dict with recipe values applied. @@ -259,12 +319,30 @@ def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any] try: resolved = self.get_resolved_recipe() except NoRecipeError: + return final_hyperparameters flat = flatten_resolved_recipe(resolved) + + allowed_keys = None + if only_user_overrides: + try: + allowed_keys = self._get_user_provided_recipe_keys() + except Exception as e: + logger.warning( + "Failed to determine user-provided recipe keys (%s); " + "falling back to submitting the full resolved recipe.", + e, + ) + allowed_keys = None + for k, v in flat.items(): - if v is not None: - final_hyperparameters[k] = str(v) if not isinstance(v, str) else v + if v is None: + continue + if allowed_keys is not None and k not in allowed_keys: + logger.info("Skipping recipe key '%s' (not user-provided).", k) + continue + final_hyperparameters[k] = str(v) if not isinstance(v, str) else v return final_hyperparameters diff --git a/sagemaker-train/src/sagemaker/train/dpo_trainer.py b/sagemaker-train/src/sagemaker/train/dpo_trainer.py index a83d1ed2fe..e56adde46e 100644 --- a/sagemaker-train/src/sagemaker/train/dpo_trainer.py +++ b/sagemaker-train/src/sagemaker/train/dpo_trainer.py @@ -308,8 +308,13 @@ def train(self, final_hyperparameters = self.hyperparameters.to_dict() - # Apply recipe/overrides if provided (overrides > recipe > Hub defaults) - final_hyperparameters = self._apply_recipe_to_hyperparameters(final_hyperparameters) + # Apply recipe/overrides if provided (overrides > recipe > Hub defaults). + # Serverless CreateTrainingJob limits HyperParameters to 100 members, so + # only forward the values the user explicitly overrode rather than the + # entire resolved recipe template. + final_hyperparameters = self._apply_recipe_to_hyperparameters( + final_hyperparameters, only_user_overrides=True + ) # Resolve is_multimodal: auto-detect from training dataset if not explicitly set if self.is_multimodal is None: effective_training_dataset = training_dataset or self.training_dataset diff --git a/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py b/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py index daf3c8f5d9..35e137ee50 100644 --- a/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py +++ b/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py @@ -289,8 +289,12 @@ def train( self._final_hyperparameters = self.hyperparameters.to_dict() - # Apply recipe/overrides if provided (overrides > recipe > Hub defaults) - self._final_hyperparameters = self._apply_recipe_to_hyperparameters(self._final_hyperparameters) + # Apply recipe/overrides if provided (overrides > recipe > Hub defaults). + # Only forward the values the user explicitly overrode rather than the + # entire resolved recipe template. + final_hyperparameters = self._apply_recipe_to_hyperparameters( + final_hyperparameters, only_user_overrides=True + ) _validate_hyperparameter_values(self._final_hyperparameters) diff --git a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py index f20b54e7b1..279cd20857 100644 --- a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py +++ b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py @@ -471,8 +471,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, final_hyperparameters = self.hyperparameters.to_dict() - # Apply recipe/overrides if provided (overrides > recipe > Hub defaults) - final_hyperparameters = self._apply_recipe_to_hyperparameters(final_hyperparameters) + # Apply recipe/overrides if provided (overrides > recipe > Hub defaults). + # Serverless CreateTrainingJob limits HyperParameters to 100 members, so + # only forward the values the user explicitly overrode rather than the + # entire resolved recipe template. + final_hyperparameters = self._apply_recipe_to_hyperparameters( + final_hyperparameters, only_user_overrides=False + ) # Resolve is_multimodal: auto-detect from training dataset if not explicitly set if self.is_multimodal is None: effective_training_dataset = training_dataset or self.training_dataset @@ -518,6 +523,8 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, if self.stopping_condition is not None: create_args["stopping_condition"] = self.stopping_condition + logger.info("Final hyperparameters before submitting training job: %s", final_hyperparameters) + try: training_job = TrainingJob.create(**create_args) except Exception as e: diff --git a/sagemaker-train/src/sagemaker/train/sft_trainer.py b/sagemaker-train/src/sagemaker/train/sft_trainer.py index 3f14c348f0..7892fb8b28 100644 --- a/sagemaker-train/src/sagemaker/train/sft_trainer.py +++ b/sagemaker-train/src/sagemaker/train/sft_trainer.py @@ -361,8 +361,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati final_hyperparameters = self.hyperparameters.to_dict() - # Apply recipe/overrides if provided (overrides > recipe > Hub defaults) - final_hyperparameters = self._apply_recipe_to_hyperparameters(final_hyperparameters) + # Apply recipe/overrides if provided (overrides > recipe > Hub defaults). + # Serverless CreateTrainingJob limits HyperParameters to 100 members, so + # only forward the values the user explicitly overrode rather than the + # entire resolved recipe template. + final_hyperparameters = self._apply_recipe_to_hyperparameters( + final_hyperparameters, only_user_overrides=True + ) # Resolve is_multimodal: auto-detect from training dataset if not explicitly set if self.is_multimodal is None: effective_training_dataset = training_dataset or self.training_dataset From 038a14d4c76b5efefdfbb8c47f4967cc618cdd4e Mon Sep 17 00:00:00 2001 From: Syed Jafri Date: Mon, 13 Jul 2026 15:58:15 -0700 Subject: [PATCH 2/3] fix: update hyperparam ref in mtrl trainer --- sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py b/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py index 35e137ee50..0c2a44e647 100644 --- a/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py +++ b/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py @@ -292,8 +292,8 @@ def train( # Apply recipe/overrides if provided (overrides > recipe > Hub defaults). # Only forward the values the user explicitly overrode rather than the # entire resolved recipe template. - final_hyperparameters = self._apply_recipe_to_hyperparameters( - final_hyperparameters, only_user_overrides=True + self._final_hyperparameters = self._apply_recipe_to_hyperparameters( + self._final_hyperparameters, only_user_overrides=True ) _validate_hyperparameter_values(self._final_hyperparameters) From c51d4e68ce6e4bd934cfb7eae076c33682a0be98 Mon Sep 17 00:00:00 2001 From: Syed Jafri Date: Mon, 13 Jul 2026 16:04:24 -0700 Subject: [PATCH 3/3] fix: enable only_user_overrides for RLVRTrainer serverless path --- sagemaker-train/src/sagemaker/train/rlvr_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py index 279cd20857..da151d7165 100644 --- a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py +++ b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py @@ -476,7 +476,7 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, # only forward the values the user explicitly overrode rather than the # entire resolved recipe template. final_hyperparameters = self._apply_recipe_to_hyperparameters( - final_hyperparameters, only_user_overrides=False + final_hyperparameters, only_user_overrides=True ) # Resolve is_multimodal: auto-detect from training dataset if not explicitly set if self.is_multimodal is None: