From 81b4883a62f6f888dfb403b2059584791988b21c Mon Sep 17 00:00:00 2001 From: chriscai-amd Date: Mon, 31 Aug 2026 09:19:03 +0000 Subject: [PATCH] Restrict the dlrmv4 learning rate to a band around the scaled reference Both divisions left opt_base_learning_rate unconstrained apart from positivity, so a submission could report any learning rate while still being scored against RCPs collected on a single recipe. The reference sweep follows a linear scaling law -- 1e-6 at global batch 8192, 2e-6 at 16384, 4e-6 at 32768, i.e. 1e-6 * GBS / 8192 -- and convergence is only comparable near it. Confine the learning rate to half to 1.5x that value, deriving the bounds in the global_batch_size POST block where the batch size is already parsed. Because the rules execute in log order, the checks guard on the bounds being resolved so a log that reports the learning rate before the batch size fails with the state dump rather than raising inside the comparison. The band applies to the open division too. That division is meant to showcase algorithmic changes, not to let submitters buy faster convergence with a learning rate the closed division would reject. Because the open rules do not pin the embedding rate to the dense one, opt_sparse_base_learning_rate is banded independently there; in the closed division the existing equality rule already covers it. Co-authored-by: Cursor --- .../training_6.1.0/closed_dlrmv4.yaml | 15 +++++++++++++-- .../training_6.1.0/open_dlrmv4.yaml | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml b/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml index 53a7f36..58083f8 100644 --- a/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml +++ b/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml @@ -5,12 +5,17 @@ 'eval_interval_samples': None, 'first_eval_samples': None, 'opt_base_learning_rate': None, + 'lr_min': None, + 'lr_max': None, 'train_samples': 2290835423, 'target_accuracy': 0.75, }) # Only the target learning rate and the global batch size may be tuned; every -# other hyperparameter below is fixed by the closed division. +# other hyperparameter below is fixed by the closed division. The learning rate +# is further confined to a band around the linear scaling law the reference +# sweep was collected on, 1e-6 * global_batch_size / 8192, since convergence is +# only comparable against the RCPs near that recipe. - KEY: NAME: global_batch_size @@ -38,6 +43,9 @@ math.floor((4480 * v['value'] + 135331840) / 3) / s['eval_interval_samples'] )) * s['eval_interval_samples'] + # Half to 1.5x the reference learning rate for this batch size. + s['lr_min'] = 0.5e-6 * v['value'] / 8192 + s['lr_max'] = 1.5e-6 * v['value'] / 8192 - KEY: NAME: train_samples @@ -64,7 +72,10 @@ - KEY: NAME: opt_base_learning_rate REQ: EXACTLY_ONE - CHECK: " v['value'] > 0 " + # The bounds are derived from global_batch_size, and the rules run in log + # order, so a log that reports the learning rate first fails the guard + # rather than raising on the comparison. + CHECK: " s['lr_min'] is not None and s['lr_min'] * (1 - 1e-9) <= v['value'] <= s['lr_max'] * (1 + 1e-9) " POST: " s['opt_base_learning_rate'] = v['value'] " - KEY: diff --git a/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml b/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml index cbf79a9..98d3b0c 100644 --- a/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml +++ b/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml @@ -1,7 +1,14 @@ +# The open division is for algorithmic changes, not for buying convergence with +# a learning rate the closed division would reject, so the same band around the +# reference scaling law applies here. + - KEY: NAME: global_batch_size REQ: EXACTLY_ONE CHECK: " is_integer(v['value']) and v['value'] > 0 " + POST: | + s['lr_min'] = 0.5e-6 * v['value'] / 8192 + s['lr_max'] = 1.5e-6 * v['value'] / 8192 - KEY: NAME: gradient_accumulation_steps @@ -30,6 +37,10 @@ - KEY: NAME: opt_base_learning_rate REQ: EXACTLY_ONE + # The bounds come from global_batch_size, and the rules run in log order, so + # a log that reports the learning rate first fails the guard rather than + # raising on the comparison. + CHECK: " s.get('lr_min') is not None and s['lr_min'] * (1 - 1e-9) <= v['value'] <= s['lr_max'] * (1 + 1e-9) " - KEY: NAME: opt_adam_beta_1 @@ -58,6 +69,9 @@ - KEY: NAME: opt_sparse_base_learning_rate REQ: EXACTLY_ONE + # Unlike the closed division this is not pinned to the dense rate, so it + # needs the band applied independently. + CHECK: " s.get('lr_min') is not None and s['lr_min'] * (1 - 1e-9) <= v['value'] <= s['lr_max'] * (1 + 1e-9) " - KEY: NAME: opt_learning_rate_warmup_steps