fix: filter full recipe template from serverless train( )

sagemaker-train/src/sagemaker/train/base_trainer.py

@@ -236,19 +236,79 @@ def _patch_resolved_recipe(self, resolved: Dict[str, Any]) -> None:
             if dotpath:
                 _set_nested_value(resolved, dotpath, value)
 
-    def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any]) -> Dict[str, Any]:
+    def _get_user_provided_recipe_keys(self) -> set:
+        """Return the set of leaf keys the user explicitly provided.
+
+        Collects keys from every source that represents an explicit user choice:
+
+        * direct hyperparameter assignments (``trainer.hyperparameters.x = val``),
+          tracked in ``hyperparameters._user_set``;
+        * the programmatic ``overrides`` dict passed at construction;
+        * the user recipe YAML file passed at construction.
+
+        These are the keys that should be forwarded to a serverless training job,
+        alongside the Hub override spec. Full recipe-template internal keys that
+        the user never touched are intentionally excluded.
+
+        Returns:
+            Set of leaf key names the user explicitly provided. Empty when the
+            user provided nothing beyond Hub defaults.
+        """
+        keys: set = set()
+
+        # Direct hyperparameter assignments (always members of the Hub spec).
+        user_set = getattr(getattr(self, 'hyperparameters', None), '_user_set', None)
+        if isinstance(user_set, set):
+            keys.update(user_set)
+
+        # Programmatic overrides dict (may contain non-spec recipe keys).
+        overrides = getattr(self, '_overrides', None)
+        if isinstance(overrides, dict) and overrides:
+            keys.update(flatten_resolved_recipe(overrides).keys())
+
+        # User recipe YAML file (may contain non-spec recipe keys).
+        recipe_path = getattr(self, '_recipe_path', None)
+        if recipe_path:
+            try:
+                from sagemaker.train.recipe_resolver import _load_user_recipe
+
+                user_recipe = _load_user_recipe(recipe_path)
+                keys.update(flatten_resolved_recipe(user_recipe).keys())
+            except Exception as e:  # pragma: no cover - best-effort key discovery
+                logger.debug("Could not load user recipe to collect keys: %s", e)
+
+        return keys
+
+
+    def _apply_recipe_to_hyperparameters(
+        self,
+        final_hyperparameters: Dict[str, Any],
+        only_user_overrides: bool = False,
+    ) -> Dict[str, Any]:
         """Apply resolved recipe values to final_hyperparameters dict.
 
         If recipe/overrides were provided, or if the user set hyperparameters
         directly via ``.hyperparameters.*``, merges resolved recipe values into
-        the hyperparameters dict. All leaf values from the resolved recipe are
-        applied — including keys not in the Hub spec subset — enabling
-        power users to override any parameter in the full recipe.
+        the hyperparameters dict. By default all leaf values from the resolved
+        recipe are applied — including keys not in the Hub spec subset. This is
+        used by the serverful/HyperPod paths, which render the values into a
+        recipe YAML file (no key-count limit).
+
         Values are converted to strings (matching the SageMaker API
         expectation for hyperparameter values).
 
         Args:
             final_hyperparameters: The hyperparameters dict from to_dict().
+            only_user_overrides: When True, only apply resolved values for keys the
+                user explicitly provided — direct hyperparameter assignments, the
+                ``overrides`` dict, or a recipe file — instead of flattening the
+                entire resolved recipe. This is required for serverless SageMaker
+                training jobs, whose ``CreateTrainingJob`` ``HyperParameters`` map
+                is limited to 100 members by the CreateTrainingJob: the resolved full recipe
+                template can contain several hundred internal leaf keys the user
+                never touched. User-provided keys are not restricted to the Hub
+                spec — power users may override any recipe key, and validity is
+                enforced by ``get_resolved_recipe()``.
 
         Returns:
             The updated hyperparameters dict with recipe values applied.
@@ -259,12 +319,30 @@ def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any]
         try:
             resolved = self.get_resolved_recipe()
         except NoRecipeError:
+
             return final_hyperparameters
 
         flat = flatten_resolved_recipe(resolved)
+
+        allowed_keys = None
+        if only_user_overrides:
+            try:
+                allowed_keys = self._get_user_provided_recipe_keys()
+            except Exception as e:
+                logger.warning(
+                    "Failed to determine user-provided recipe keys (%s); "
+                    "falling back to submitting the full resolved recipe.",
+                    e,
+                )
+                allowed_keys = None
+
         for k, v in flat.items():
-            if v is not None:
-                final_hyperparameters[k] = str(v) if not isinstance(v, str) else v
+            if v is None:
+                continue
+            if allowed_keys is not None and k not in allowed_keys:
+                logger.info("Skipping recipe key '%s' (not user-provided).", k)
+                continue
+            final_hyperparameters[k] = str(v) if not isinstance(v, str) else v
 
         return final_hyperparameters
 

sagemaker-train/src/sagemaker/train/dpo_trainer.py

@@ -308,8 +308,13 @@ def train(self,
 
         final_hyperparameters = self.hyperparameters.to_dict()
 
-        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults)
-        final_hyperparameters = self._apply_recipe_to_hyperparameters(final_hyperparameters)
+        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults).
+        # Serverless CreateTrainingJob limits HyperParameters to 100 members, so
+        # only forward the values the user explicitly overrode rather than the
+        # entire resolved recipe template.
+        final_hyperparameters = self._apply_recipe_to_hyperparameters(
+            final_hyperparameters, only_user_overrides=True
+        )
         # Resolve is_multimodal: auto-detect from training dataset if not explicitly set
         if self.is_multimodal is None:
             effective_training_dataset = training_dataset or self.training_dataset

sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py

@@ -289,8 +289,12 @@ def train(
 
         self._final_hyperparameters = self.hyperparameters.to_dict()
 
-        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults)
-        self._final_hyperparameters = self._apply_recipe_to_hyperparameters(self._final_hyperparameters)
+        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults).
+        # Only forward the values the user explicitly overrode rather than the
+        # entire resolved recipe template.
+        self._final_hyperparameters = self._apply_recipe_to_hyperparameters(
+            self._final_hyperparameters, only_user_overrides=True
+        )
 
         _validate_hyperparameter_values(self._final_hyperparameters)
 

sagemaker-train/src/sagemaker/train/rlvr_trainer.py

@@ -471,8 +471,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None,
 
         final_hyperparameters = self.hyperparameters.to_dict()
 
-        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults)
-        final_hyperparameters = self._apply_recipe_to_hyperparameters(final_hyperparameters)
+        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults).
+        # Serverless CreateTrainingJob limits HyperParameters to 100 members, so
+        # only forward the values the user explicitly overrode rather than the
+        # entire resolved recipe template.
+        final_hyperparameters = self._apply_recipe_to_hyperparameters(
+            final_hyperparameters, only_user_overrides=True
+        )
         # Resolve is_multimodal: auto-detect from training dataset if not explicitly set
         if self.is_multimodal is None:
             effective_training_dataset = training_dataset or self.training_dataset
@@ -518,6 +523,8 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None,
         if self.stopping_condition is not None:
             create_args["stopping_condition"] = self.stopping_condition
 
+        logger.info("Final hyperparameters before submitting training job: %s", final_hyperparameters)
+
         try:
             training_job = TrainingJob.create(**create_args)
         except Exception as e:

sagemaker-train/src/sagemaker/train/sft_trainer.py

@@ -361,8 +361,13 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati
 
         final_hyperparameters = self.hyperparameters.to_dict()
 
-        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults)
-        final_hyperparameters = self._apply_recipe_to_hyperparameters(final_hyperparameters)
+        # Apply recipe/overrides if provided (overrides > recipe > Hub defaults).
+        # Serverless CreateTrainingJob limits HyperParameters to 100 members, so
+        # only forward the values the user explicitly overrode rather than the
+        # entire resolved recipe template.
+        final_hyperparameters = self._apply_recipe_to_hyperparameters(
+            final_hyperparameters, only_user_overrides=True
+        )
         # Resolve is_multimodal: auto-detect from training dataset if not explicitly set
         if self.is_multimodal is None:
             effective_training_dataset = training_dataset or self.training_dataset