From 505d1d270127939a7aba8318c414091144f4d55b Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Wed, 24 Jun 2026 12:51:01 +0200
Subject: [PATCH 01/17] HITL enabled and working, needs polish and test

---
 compute_worker/compute_worker.py              | 50 ++++++++++++++++++-
 src/apps/api/serializers/competitions.py      |  6 ++-
 src/apps/competitions/models.py               |  1 +
 src/apps/competitions/tasks.py                | 13 +++++
 .../editor/_competition_details.tag           | 18 +++++++
 5 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 243dcd16d..75a3540c3 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -314,6 +314,9 @@ def run_wrapper(run_args):
         run.prepare()
         run.start()
         if run.is_scoring:
+            if run.human_in_the_loop:
+                run.wait_for_human_validation()
+                run._update_status(SubmissionStatus.FINISHED)
             run.push_scores()
         run.push_output()
     except DockerImagePullException as e:
@@ -470,6 +473,7 @@ def __init__(self, run_args):
         self.prediction_result = run_args["prediction_result"]
         self.scoring_result = run_args.get("scoring_result")
         self.execution_time_limit = run_args["execution_time_limit"]
+        self.human_in_the_loop = run_args.get("human_in_the_loop", False)
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -1445,11 +1449,55 @@ def start(self):
                 )
                 # Raise so upstream marks failed immediately
                 raise SubmissionException("Child task failed or non-zero return code")
-            self._update_status(SubmissionStatus.FINISHED)
+
+            if not self.human_in_the_loop:
+                self._update_status(SubmissionStatus.FINISHED)
 
         else:
             self._update_status(SubmissionStatus.SCORING)
 
+    def wait_for_human_validation(self):
+        container_output_dir = self.output_dir
+        host_output_dir = self._get_host_path(self.output_dir)
+
+        scores_path = os.path.join(host_output_dir, "scores.json")
+        if not os.path.exists(os.path.join(container_output_dir, "scores.json")):
+            scores_path = os.path.join(host_output_dir, "scores.txt")
+
+        approved_container = os.path.join(container_output_dir, "hitl_approved")
+        rejected_container = os.path.join(container_output_dir, "hitl_rejected")
+        approved_host = os.path.join(host_output_dir, "hitl_approved")
+        rejected_host = os.path.join(host_output_dir, "hitl_rejected")
+
+        logger.info("=" * 60)
+        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
+        logger.info("Inspect the scores file:")
+        logger.info(f"  cat {scores_path}")
+        logger.info(f"To approve : touch {approved_host}")
+        logger.info(f"To reject  : touch {rejected_host}")
+        logger.info("=" * 60)
+
+        poll_interval = 3
+        max_wait = 60 * 60 * 24
+
+        elapsed = 0
+        while elapsed < max_wait:
+            if os.path.exists(approved_container):  # ← poll le chemin conteneur
+                logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
+                return
+            if os.path.exists(rejected_container):  # ← poll le chemin conteneur
+                raise SubmissionException(
+                    f"HITL: scores rejected by the compute node operator "
+                    f"(submission {self.submission_id})"
+                )
+            time.sleep(poll_interval)
+            elapsed += poll_interval
+
+        raise SubmissionException(
+            f"HITL: 24h timeout reached without validation "
+            f"(submission {self.submission_id})"
+        )
+
     def push_scores(self):
         """This is only ran at the end of the scoring step"""
         # POST to some endpoint:
diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py
index a0fb14fda..3a566446e 100644
--- a/src/apps/api/serializers/competitions.py
+++ b/src/apps/api/serializers/competitions.py
@@ -272,7 +272,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def validate_phases(self, phases):
@@ -417,7 +418,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def get_leaderboards(self, instance):
diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py
index f919ae432..164e66536 100644
--- a/src/apps/competitions/models.py
+++ b/src/apps/competitions/models.py
@@ -89,6 +89,7 @@ class Competition(models.Model):
 
     # If true, forum is enabled (default=True)
     forum_enabled = models.BooleanField(default=True)
+    enable_human_in_the_loop = models.BooleanField(default=False)
 
     def __str__(self):
         return f"competition-{self.title}-{self.pk}-{self.competition_type}"
diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 3d7d8abcb..1f424b4ac 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -58,6 +58,7 @@
     "contact_email",
     "fact_sheet",
     "forum_enabled",
+    "enable_human_in_the_loop"
 ]
 
 TASK_FIELDS = [
@@ -166,6 +167,7 @@ def _send_to_compute_worker(submission, is_scoring):
         ),
         "id": submission.pk,
         "is_scoring": is_scoring,
+        "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop,
     }
 
     if (
@@ -260,6 +262,17 @@ def _send_to_compute_worker(submission, is_scoring):
             submission.queue = effective_queue
             submission.save(update_fields=["queue"])
 
+    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+        time_limit = 60 * 60 * 25
+
+    if (
+        submission.phase.competition.queue
+    ):  # if the competition is running on a custom queue, not the default queue
+        submission.queue = submission.phase.competition.queue
+        run_args["execution_time_limit"] = (
+            submission.phase.execution_time_limit
+        )  # use the competition time limit
+        submission.save(update_fields=["queue"])
     if submission.status == Submission.SUBMITTING:
         submission.status = Submission.SUBMITTED
         submission.save(update_fields=["status"])
diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag
index 199edd404..da135fef0 100644
--- a/src/static/riot/competitions/editor/_competition_details.tag
+++ b/src/static/riot/competitions/editor/_competition_details.tag
@@ -211,6 +211,22 @@
             </sup>
         </div>
 
+        <!--  Human in the Loop  -->
+        <div class="field">
+            <label>Human in the Loop</label>
+            <div class="ui checkbox">
+                <label>Enable Human in the Loop validation</label>
+                <input type="checkbox" ref="enable_human_in_the_loop" onchange="{form_updated}">
+            </div>
+            <sup>
+                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform"
+                        data-inverted=""
+                        data-position="bottom center">
+                    <i class="help icon circle"></i>
+                </span>
+            </sup>
+        </div>
+
         <!--  Public submissions  -->
         <div class="field">
             <label>Public Submissions</label>
@@ -314,6 +330,7 @@
             self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked
             self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked
             self.data["forum_enabled"] = self.refs.forum_enabled.checked
+            self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()
@@ -452,6 +469,7 @@
             self.refs.auto_run_submissions.checked = competition.auto_run_submissions
             self.refs.can_participants_make_submissions_public.checked = competition.can_participants_make_submissions_public
             self.refs.forum_enabled.checked = competition.forum_enabled
+            self.refs.enable_human_in_the_loop.checked = competition.enable_human_in_the_loop
             self.refs.make_programs_available.checked = competition.make_programs_available
             self.refs.make_input_data_available.checked = competition.make_input_data_available
             $(self.refs.docker_image).val(competition.docker_image)

From 13aa0ea7915c5151418d98f356d1f2cf83b2210b Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 29 Jun 2026 15:23:32 +0200
Subject: [PATCH 02/17] disable HITL for public CWs

---
 compute_worker/compute_worker.py                     |  4 ++--
 src/apps/competitions/tasks.py                       | 11 ++++++++---
 .../competitions/editor/_competition_details.tag     | 12 +++++++++++-
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 75a3540c3..1d1a7ea11 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -1482,10 +1482,10 @@ def wait_for_human_validation(self):
 
         elapsed = 0
         while elapsed < max_wait:
-            if os.path.exists(approved_container):  # ← poll le chemin conteneur
+            if os.path.exists(approved_container):
                 logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
                 return
-            if os.path.exists(rejected_container):  # ← poll le chemin conteneur
+            if os.path.exists(rejected_container):
                 raise SubmissionException(
                     f"HITL: scores rejected by the compute node operator "
                     f"(submission {self.submission_id})"
diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 1f424b4ac..4251e2592 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -157,6 +157,11 @@ def _get_user_group_queues(user, competition):
 
 
 def _send_to_compute_worker(submission, is_scoring):
+    hitl_active = (
+        submission.phase.competition.enable_human_in_the_loop
+        and submission.queue is not None
+    )
+
     run_args = {
         "user_pk": submission.owner.pk,
         "submissions_api_url": settings.SUBMISSIONS_API_URL,
@@ -167,7 +172,7 @@ def _send_to_compute_worker(submission, is_scoring):
         ),
         "id": submission.pk,
         "is_scoring": is_scoring,
-        "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop,
+        "human_in_the_loop": hitl_active,
     }
 
     if (
@@ -251,7 +256,7 @@ def _send_to_compute_worker(submission, is_scoring):
     logger.info(run_args)
 
     # Pad timelimit so worker has time to cleanup
-    time_padding = 60 * 20  # 20 minutes
+    time_padding = 60 * 20
     time_limit = submission.phase.execution_time_limit + time_padding
 
     effective_queue = submission.queue or submission.phase.competition.queue
@@ -262,7 +267,7 @@ def _send_to_compute_worker(submission, is_scoring):
             submission.queue = effective_queue
             submission.save(update_fields=["queue"])
 
-    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+    if is_scoring and hitl_active:
         time_limit = 60 * 60 * 25
 
     if (
diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag
index da135fef0..25afa4099 100644
--- a/src/static/riot/competitions/editor/_competition_details.tag
+++ b/src/static/riot/competitions/editor/_competition_details.tag
@@ -219,12 +219,17 @@
                 <input type="checkbox" ref="enable_human_in_the_loop" onchange="{form_updated}">
             </div>
             <sup>
-                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform"
+                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform. Only active on private queues."
                         data-inverted=""
                         data-position="bottom center">
                     <i class="help icon circle"></i>
                 </span>
             </sup>
+            <div ref="hitl_warning" class="ui yellow message" style="display:none; margin-top: 0.5em;">
+                <i class="exclamation triangle icon"></i>
+                Human in the Loop is only active on <strong>private queues</strong>.
+                It will have no effect until a private queue is selected for this competition or its participant groups.
+            </div>
         </div>
 
         <!--  Public submissions  -->
@@ -331,6 +336,11 @@
             self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked
             self.data["forum_enabled"] = self.refs.forum_enabled.checked
             self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked
+            const hitlChecked = self.refs.enable_human_in_the_loop.checked
+            const hasPrivateQueue = !!(self.data["queue"] && self.data["queue"] !== "")
+            if (self.refs.hitl_warning) {
+                self.refs.hitl_warning.style.display = (hitlChecked && !hasPrivateQueue) ? 'block' : 'none'
+            }
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()

From 6b857f70a3782581c3a253276c56657dbf6e0dde Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 30 Jun 2026 10:56:34 +0200
Subject: [PATCH 03/17] HITL secute on public CW + UI improvement

---
 src/apps/competitions/tasks.py                  | 17 +++++++++++++++++
 .../competitions/detail/submission_manager.tag  |  8 ++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 4251e2592..35bbebd32 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -375,6 +375,23 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False):
     )
     submission = qs.get(pk=submission_pk)
 
+    # ── HUMAN IN THE LOOP SECURE (private CW only)
+    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+        if submission.queue is None:
+            logger.error(
+                f"HITL: submission {submission_pk} rejected — "
+                f"Human in the Loop is enabled but no private queue is configured "
+                f"(competition or participant group)."
+            )
+            submission.status = Submission.FAILED
+            submission.status_details = (
+                "This competition requires Human in the Loop validation, but no "
+                "private compute queue is configured. Contact the organizer."
+            )
+            submission.save(update_fields=["status", "status_details"])
+            return
+    # ── HITL SECURE (private CW only)
+
     if submission.is_specific_task_re_run:
         # Should only be one task for a specified task submission
         tasks = Task.objects.filter(pk__in=task_pks)
diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag
index 3f41a69eb..37530345c 100644
--- a/src/static/riot/competitions/detail/submission_manager.tag
+++ b/src/static/riot/competitions/detail/submission_manager.tag
@@ -107,7 +107,8 @@
                 <td class="right aligned collapsing">
                     { submission.status }
                     <sup data-tooltip="{submission.status_details}">
-                        <i if="{submission.status === 'Failed'}" class="failed question circle icon"></i>
+                        <i if="{submission.status === 'Failed' && !is_hitl_failure(submission)}" class="failed question circle icon"></i>
+                        <i if="{submission.status === 'Failed' && is_hitl_failure(submission)}" class="orange lock icon"></i>
                     </sup>
                     <sup data-tooltip="An organizer will run your submission soon">
                         <i if="{submission.status === 'Submitting' && !submission.auto_run}"
@@ -311,7 +312,10 @@
             self.update()
         }
 
-        
+        self.is_hitl_failure = function (submission) {
+            return !!(submission.status_details && submission.status_details.indexOf('Human in the Loop') !== -1)
+        }
+
         self.update_submissions = function (filters) {
             self.loading = true
             self.update()

From 3d5ce23d6a3f2211f252a335e8374a3a66281842 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Wed, 8 Jul 2026 16:55:58 +0200
Subject: [PATCH 04/17] fix after rebase + detailed result + CW check of HITL
 enabled

---
 compute_worker/compute_worker.py | 84 +++++++++++++++++++++++++++++---
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 1d1a7ea11..54461b2d1 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -112,6 +112,9 @@ def to_bool(val):
     CODALAB_IGNORE_CLEANUP_STEP = to_bool(get("CODALAB_IGNORE_CLEANUP_STEP"))
 
     WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip()
+    HUMAN_IN_THE_LOOP = (
+        get("HUMAN_IN_THE_LOOP", "false").lower() == "true"
+    )
 
 
 # -----------------------------------------------
@@ -313,12 +316,20 @@ def run_wrapper(run_args):
     try:
         run.prepare()
         run.start()
+
         if run.is_scoring:
             if run.human_in_the_loop:
                 run.wait_for_human_validation()
-                run._update_status(SubmissionStatus.FINISHED)
-            run.push_scores()
-        run.push_output()
+                run.send_final_detailed_results()
+                run.push_scores()
+                run.push_output()
+            else:
+                run.push_scores()
+                run.push_output()
+        else:
+            run.push_output()
+
+        run._update_status(SubmissionStatus.FINISHED)
     except DockerImagePullException as e:
         msg = str(e).strip()
         if msg:
@@ -473,7 +484,17 @@ def __init__(self, run_args):
         self.prediction_result = run_args["prediction_result"]
         self.scoring_result = run_args.get("scoring_result")
         self.execution_time_limit = run_args["execution_time_limit"]
+        # ----- HITL ------
         self.human_in_the_loop = run_args.get("human_in_the_loop", False)
+        compute_hitl = Settings.HUMAN_IN_THE_LOOP
+        if self.human_in_the_loop != compute_hitl:
+            raise SubmissionException(
+                "Task rejected because the Site Worker and Compute Worker "
+                "do not have the same HUMAN_IN_THE_LOOP configuration "
+                f"(task={self.human_in_the_loop}, "
+                f"compute_worker={Settings.HUMAN_IN_THE_LOOP})."
+            )
+
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -517,6 +538,13 @@ def __init__(self, run_args):
     async def watch_detailed_results(self):
         """Watches files alongside scoring + program containers, currently only used
         for detailed_results.html"""
+
+        if self.human_in_the_loop:
+            logger.info(
+                "HITL enabled: skipping detailed_results streaming"
+            )
+            return
+
         if not self.detailed_results_url:
             return
         file_path = self.get_detailed_results_file_path()
@@ -600,6 +628,49 @@ async def send_detailed_results(self, file_path):
             logger.exception(e)
             return
 
+    def send_final_detailed_results(self):
+        if not self.detailed_results_url:
+            return
+
+        file_path = self.get_detailed_results_file_path()
+
+        if not file_path:
+            logger.info("No detailed_results.html found")
+            return
+
+        logger.info(
+            f"Uploading final detailed results {file_path} - {self.detailed_results_url}"
+        )
+
+        self._put_file(
+            self.detailed_results_url,
+            file=file_path,
+            content_type="text/html",
+        )
+
+        websocket_url = f"{self.websocket_url}?kind=detailed_results"
+
+        try:
+            websocket = asyncio.run(
+                asyncio.wait_for(
+                    websockets.connect(websocket_url),
+                    timeout=30.0,
+                )
+            )
+
+            asyncio.run(
+                websocket.send(
+                    json.dumps(
+                        {
+                            "kind": "detailed_result_update",
+                        }
+                    )
+                )
+            )
+
+        except Exception as e:
+            logger.exception(e)
+
     def _get_stdout_stderr_file_names(self, run_args):
         # run_args should be the run_args argument passed to __init__ from the run_wrapper.
         if not self.is_scoring:
@@ -1312,9 +1383,10 @@ def start(self):
                 )
 
             # During scoring we watch for detailed results
-            tasks.append(
-                self.watch_detailed_results()
-            )
+            if not self.human_in_the_loop:
+                tasks.append(
+                    self.watch_detailed_results()
+                )
         else:
             # During ingestion we run ingestion program directory and submission directory
             tasks.extend([

From 63548e96e27a07ae71056231fc0b125ef61f3924 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Thu, 9 Jul 2026 15:32:32 +0200
Subject: [PATCH 05/17] HITL improvement + Websocket fixed + more logs

---
 compute_worker/compute_worker.py | 48 ++++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 54461b2d1..05e4b1fd1 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -311,10 +311,19 @@ def rewrite_bundle_url_if_needed(url):
 def run_wrapper(run_args):
     # We need to convert the UUID given by celery into a byte like object otherwise things will break
     run_args.update(secret=str(run_args["secret"]))
+
     logger.info(f"Received run arguments: \n {colorize_run_args(json.dumps(run_args))}")
+    logger.info(
+        "HITL configuration : "
+        f"task={run_args.get('human_in_the_loop', False)} "
+        f"compute_worker={Settings.HUMAN_IN_THE_LOOP}"
+    )
+
     run = Run(run_args)
+
     try:
         run.prepare()
+        run.validate_hitl_configuration()
         run.start()
 
         if run.is_scoring:
@@ -486,15 +495,6 @@ def __init__(self, run_args):
         self.execution_time_limit = run_args["execution_time_limit"]
         # ----- HITL ------
         self.human_in_the_loop = run_args.get("human_in_the_loop", False)
-        compute_hitl = Settings.HUMAN_IN_THE_LOOP
-        if self.human_in_the_loop != compute_hitl:
-            raise SubmissionException(
-                "Task rejected because the Site Worker and Compute Worker "
-                "do not have the same HUMAN_IN_THE_LOOP configuration "
-                f"(task={self.human_in_the_loop}, "
-                f"compute_worker={Settings.HUMAN_IN_THE_LOOP})."
-            )
-
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -628,6 +628,13 @@ async def send_detailed_results(self, file_path):
             logger.exception(e)
             return
 
+        finally:
+            if websocket is not None:
+                try:
+                    await websocket.close()
+                except Exception as e:
+                    logger.exception(e)
+
     def send_final_detailed_results(self):
         if not self.detailed_results_url:
             return
@@ -1357,6 +1364,15 @@ def prepare(self):
         self._get_container_image(self.container_image)
         self._update_status(SubmissionStatus.RUNNING)
 
+    def validate_hitl_configuration(self):
+        if self.human_in_the_loop != Settings.HUMAN_IN_THE_LOOP:
+            raise SubmissionException(
+                "Task rejected because the Site Worker and Compute Worker "
+                "do not have the same HUMAN_IN_THE_LOOP configuration "
+                f"(task={self.human_in_the_loop}, "
+                f"compute_worker={Settings.HUMAN_IN_THE_LOOP})."
+            )
+
     def start(self):
 
         logger.info(f"Preparing to run: {ProgramKind.SCORING_PROGRAM if self.is_scoring else ProgramKind.INGESTION_PROGRAM}")
@@ -1505,9 +1521,15 @@ def start(self):
             # Check if scoring program failed
             # We have can have 2 or 3 gathered tasks: 3 gathered tasks in case when `ingestion_only_during_scoring` is True, 2 otherwise
             if self.ingestion_only_during_scoring:
-                program_results, _, _ = task_results
+                if self.human_in_the_loop:
+                    program_results, _ = task_results
+                else:
+                    program_results, _, _ = task_results
             else:
-                program_results, _ = task_results
+                if self.human_in_the_loop:
+                    (program_results,) = task_results
+                else:
+                    program_results, _ = task_results
             # Gather returns either normal values or exception instances when return_exceptions=True
             had_async_exc = isinstance(
                 program_results, BaseException
@@ -1553,6 +1575,10 @@ def wait_for_human_validation(self):
         max_wait = 60 * 60 * 24
 
         elapsed = 0
+
+        logger.info(
+            "Waiting for human validation..."
+        )
         while elapsed < max_wait:
             if os.path.exists(approved_container):
                 logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")

From 02264eb60df789b50e046a5ae8ef319d6e8fe8db Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Fri, 10 Jul 2026 16:04:38 +0200
Subject: [PATCH 06/17] submission status: awaiting validation

---
 compute_worker/compute_worker.py | 3 +++
 src/apps/competitions/models.py  | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 05e4b1fd1..84fcf1825 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -134,6 +134,7 @@ class SubmissionStatus:
     SUBMITTED = "Submitted"
     PREPARING = "Preparing"
     RUNNING = "Running"
+    AWAITING_VALIDATION = "Awaiting validation"
     SCORING = "Scoring"
     FINISHED = "Finished"
     FAILED = "Failed"
@@ -144,6 +145,7 @@ class SubmissionStatus:
         SUBMITTED,
         PREPARING,
         RUNNING,
+        AWAITING_VALIDATION,
         SCORING,
         FINISHED,
         FAILED,
@@ -328,6 +330,7 @@ def run_wrapper(run_args):
 
         if run.is_scoring:
             if run.human_in_the_loop:
+                run._update_status(SubmissionStatus.AWAITING_VALIDATION)
                 run.wait_for_human_validation()
                 run.send_final_detailed_results()
                 run.push_scores()
diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py
index 164e66536..97d7ff057 100644
--- a/src/apps/competitions/models.py
+++ b/src/apps/competitions/models.py
@@ -438,6 +438,7 @@ class Submission(models.Model):
     PREPARING = "Preparing"
     RUNNING = "Running"
     SCORING = "Scoring"
+    AWAITING_VALIDATION = "Awaiting validation"
     CANCELLED = "Cancelled"
     FINISHED = "Finished"
     FAILED = "Failed"
@@ -449,6 +450,7 @@ class Submission(models.Model):
         (PREPARING, "Preparing"),
         (RUNNING, "Running"),
         (SCORING, "Scoring"),
+        (AWAITING_VALIDATION, "Awaiting validation"),
         (CANCELLED, "Cancelled"),
         (FINISHED, "Finished"),
         (FAILED, "Failed"),

From 9bac0df5f1c3b3fafe5fc3c06a0904135f73cdef Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 20 Jul 2026 16:23:26 +0200
Subject: [PATCH 07/17] adding HITL documentation

---
 .../Running_a_benchmark/Competition-HITL.md   | 280 ++++++++++++++++++
 .../_attachments/doc_HITL_1.png               | Bin 0 -> 17180 bytes
 documentation/zensical.toml                   |   3 +-
 3 files changed, 282 insertions(+), 1 deletion(-)
 create mode 100644 documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
 create mode 100644 documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png

diff --git a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
new file mode 100644
index 000000000..b45049032
--- /dev/null
+++ b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
@@ -0,0 +1,280 @@
+# Human-in-the-Loop (HITL)
+
+## Introduction
+
+The **Human-in-the-Loop (HITL)** feature introduces a manual validation step into the Codabench submission workflow.
+
+When enabled, the Compute Worker executes the submission normally but **does not immediately publish the scoring results**. Instead, execution pauses after the scoring program has completed and waits for a human operator to explicitly approve or reject the submission.
+
+This mechanism is intended for competitions where the scoring results must be reviewed before being published, such as:
+
+* Medical AI competitions
+* Security-sensitive evaluations
+* Competitions involving confidential data
+* Any evaluation requiring manual verification from compute worker side
+
+---
+
+## Objectives
+
+The HITL feature guarantees:
+
+* the submission is fully executed before any result is published;
+* no score appears on the leaderboard before validation;
+* no detailed results are available before validation;
+* no output archive is sent back to the Site Worker before validation;
+* the operator can approve or reject the submission directly from the Compute Worker.
+
+---
+
+## Scope
+
+The HITL feature applies **only to private Compute Workers**.
+
+Public Compute Workers **must never execute HITL jobs**, as waiting for manual validation would permanently block shared workers.
+
+---
+
+## Activation
+
+### Competition configuration
+
+Each competition exposes the following option:
+
+```
+Enable Human-in-the-Loop
+```
+
+When enabled, every submission routed to a private Compute Worker is executed in HITL mode.
+![image1](_attachments/doc_HITL_1.png)
+
+---
+
+### Compute Worker configuration
+
+Each Compute Worker may enable HITL support using its `.env` file.
+
+``` title="Option"
+HUMAN_IN_THE_LOOP=true
+```
+
+If the variable is omitted, the Compute Worker behaves as a standard worker.
+The absence of the variable is interpreted as **False**, ensuring full backward compatibility with existing Compute Workers.
+
+---
+
+## Safety checks
+
+To prevent configuration errors, the Compute Worker verifies that both sides agree on the HITL configuration.
+The Site Worker includes the following field in every Celery task.
+
+Before starting the execution, the Compute Worker compares:
+
+* task `human_in_the_loop` Codabench side.
+* local `.env` variable `HUMAN_IN_THE_LOOP` Compute worker side.
+
+---
+
+### Mismatched configuration
+
+The submission is immediately rejected.
+The Compute Worker updates the submission status to **Failed** and reports the configuration mismatch.
+
+``` title="Example logs:"
+Task rejected because the Site Worker and Compute Worker
+do not share the same HUMAN_IN_THE_LOOP configuration
+(task=True, compute_worker=False)
+```
+
+This prevents accidental execution of HITL competitions on non-HITL workers.
+
+---
+
+## Submission lifecycle
+
+### Standard workflow
+
+When HITL is disabled, the workflow remains unchanged.
+
+During scoring:
+
+* scores are uploaded immediately;
+* detailed results are streamed continuously;
+* output files are returned immediately.
+
+---
+
+### HITL workflow
+
+When HITL is enabled:
+
+The scoring program executes normally.
+After scoring completes, the Compute Worker pauses before publishing any result.
+
+---
+
+## Awaiting validation status
+
+A new submission status is introduced:
+
+This status indicates:
+
+* scoring has completed successfully;
+* all output files are available locally;
+* publication is waiting for manual approval.
+
+This status allows users and organizers to distinguish between:
+
+* submissions that are still executing;
+* submissions waiting for human validation.
+
+---
+
+## Manual validation
+
+Once execution finishes, the Compute Worker logs instructions similar to:
+``` title="Compute worker terminal command (exemple with docker)"
+docker compose logs -f [container name]
+```
+
+``` title="HITL validation display on CW terminal"
+============================================================
+HUMAN IN THE LOOP — submission 42
+
+Inspect the scores file:
+
+cat /host/output/scores.json
+
+To approve:
+touch /host/output/hitl_approved
+
+To reject:
+touch /host/output/hitl_rejected
+============================================================
+```
+
+The operator inspects the generated files before deciding.
+This needs to be done inside of the compute worker host machine (not inside of the CW terminal).
+
+---
+
+### Approval
+
+The Compute Worker immediately resumes execution.
+The following artifacts are published:
+
+* detailed results;
+* scores;
+* output archive;
+* logs.
+
+The submission status becomes "Finished"
+
+---
+
+### Rejection
+
+After rejection from operator, the compute worker immediately aborts the submission.
+
+The submission status becomes "Failed"
+
+No scoring results are published.
+
+---
+
+### Timeout
+
+If no decision is received within 24 hours, the submission automatically fails.
+The Compute Worker reports a timeout error.
+
+---
+
+## Publication policy
+
+The fundamental principle of HITL is:
+
+> **Nothing leaves the Compute Worker before validation.**
+
+When HITL is enabled:
+
+| Artifact                                | Before validation | After approval |
+| --------------------------------------- | ----------------- | -------------- |
+| scores.json                             | ❌                 | ✅              |
+| leaderboard score                       | ❌                 | ✅              |
+| detailed_results.html                   | ❌                 | ✅              |
+| detailed results websocket notification | ❌                 | ✅              |
+| output archive                          | ❌                 | ✅              |
+| execution logs                          | Local only        | ✅              |
+
+The Compute Worker keeps every generated artifact locally until approval.
+
+---
+
+## Detailed Results behaviour
+
+### Without HITL
+
+```mermaid
+flowchart TD
+    A[Scoring] --> B[Watch detailed results]
+    B --> C[Upload detailed_results.html]
+    C --> D[Notify frontend]
+```
+
+### With HITL
+
+```mermaid
+flowchart TD
+    A[Scoring] --> B[Store detailed_results.html locally]
+    B --> C[Awaiting validation]
+    C --> D[Operator approval]
+    D --> E[Upload detailed_results.html]
+    E --> F[Notify frontend]
+```
+
+---
+
+## Output behaviour
+
+Prediction files, scoring outputs and logs are generated normally.
+However, publication is postponed until validation succeeds.
+This guarantees that no evaluation result becomes visible before manual approval.
+
+---
+
+## Error handling
+
+The following situations are handled explicitly.
+
+| Situation                                            | Behaviour                       |
+| ---------------------------------------------------- | ------------------------------- |
+| HITL enabled on competition and Compute Worker       | Execution pauses for validation |
+| HITL disabled everywhere                             | Standard workflow               |
+| HITL mismatch between Site Worker and Compute Worker | Submission fails immediately    |
+| Operator rejects submission                          | Submission fails                |
+| Validation timeout                                   | Submission fails                |
+| Approval received                                    | Results are published normally  |
+
+---
+
+## Design principles
+
+The HITL implementation follows four principles:
+
+1. **Backward compatibility**
+
+   * Existing deployments require no configuration changes.
+
+2. **Safety**
+
+   * Configuration mismatches are detected before execution.
+
+3. **Isolation**
+
+   * HITL is supported only on private Compute Workers.
+
+4. **Atomic publication**
+
+   * No artifact is published before manual approval.
+
+These principles ensure that sensitive competitions can safely introduce manual validation without impacting the standard Codabench execution workflow.
diff --git a/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png b/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..866ec86ddf52c3cb50931a4d30dbb41420d3dd37
GIT binary patch
literal 17180
zcmb8XWl&vF*QFc53GVK0f#B{MEV#S7I|PT|?jGFT-4E{W?(Wt*-~08wRbBUXb@PLw
zPMuBF-e;{f=NRLeCsa;G3=Rex=F_K7a1!Fezdn8XtO5L84g~@H#&=lN`t(U~M?zRY
z(KY=n16);610#s<`-P2*i#AGYHB>7X|M!YNYk|JL<%>>dUFX10n{&24+to$A?f@31
zJEp+G$d8Hdg%i97o*rlQ_e(VU2bCLML{NT&!qoeSU;>E58I(i~N%Y!%y}<~;%a^VC
ziq&tgPd_SjT7y;^ZF&a>DP~XZ&gW1@66t*zo9uQV60!9(Zay-CJs!@tWOrnFYy?6q
z!^s<JO~!<mrpe?xwee;v)bNz7tQPfISPsTB0$;B8F7MzvX`Tn82vl<#?4G8{UdQDu
zde$qzt=C%mJ3r!Y&;xXQ<@@eWQ-0W{aYN%}aBeML#VvL?uuT_8g^g#VKYsSUJ0_FK
z;DolZvC)1XNn&_?SzBpzUvc|*=ixid;`N%AnbCS!o-1wg`RH(7&l8JUjGePgs;Z-n
z+9DC{86Ew}b#|7<M*GU3;||8@EOetk5FQ#+Q9-Fz^4A$#?U>DGy>E_2Wpi&YKjZ!F
z#f6ub(|H*}?cQ;`Ti|5U-tiS%yUlfcWh$L3W9De4@N&cF!z}6Pa_egn0-ghZk&@k2
z4>*s@Xrs-BYg)^Q4!ck0gVf7Prw@Tl7H6QB$HV5;&R~r9y5A**0cy3^^)9rJO1Uy#
zXdJ0P!OxV*dJ7j-RBABUi3N7g!|&=#^}E4YuJ@-{A03WQh2Zt(`COoQYTtnf?5^b!
z<nb)7!5Nm5`4V}&at*F*bu=8uqgCi~JoOxbw-(dc?97^ilbS_Di-oeFI0eWv=Yz3~
zuOs$GlXVg#inKPN&^v>H>``Dr_su2~`0vs^!h=eU&eF_^FC?WI0a>m$193@Ks-5^H
zMMrhGcqcC|7vB=+Qf3PEw3?9#TFae`n!e$Hl$)BHIaFlg$YnP7hS0WkZ3@ey6X`TS
zbiw2iIJs>`YL=^20<B|0!SK2I;8M&M%a}@*Ws}u!e6&H8+U?s0t+rQw&P(GN8CC9c
zlDM3%PzIV}S9Mg@an%Md4dTFxdYobWv#74FPOQ~(Yinr~B@%{SlN>wQBAxO_vG`YT
zQ%CS1Ec!NFTEph%7RgXDBT3LV=Xs5RVg(aiK06o^(H8xQbFItk`<~Fa!lgeQ9tRD`
zVQ0P#Qi3AebNLo*noW-2Yb_65oj3b6cNnW}zrWcz4>VS8R9Q^$Hmv?I9R01$<8i1q
zn<?ojlfhe>xKv6z7`)hK<dA(P7s!8-K(7E{GNIFV+I&+r?F9~xyBLK^^($94%lq}^
zaEc_E)d?oG(R6rE{L2BSJ=ik>W<Xd{Wlyye&Nm#+WE!R#+Lzn&Kx(Cu?c1dr)fz{R
zNIb4TbEPUo?AWa40re+r*B??z<@{+4XT)c1-~!)c7h_pkjn6n$R~B`r{J^V96hBA8
z-@8k1ZW881;IPrx7>x|%&Xgp4J*WIkhT#`fvj@6ziOCgtYOP6NX@!~2ml}NWdA^m%
zK6}6G{2<cH6{h8dOvs)q)gfqoI2SvfE%whSgoP$mi@ZNho-UM~csX8eF10s#4aQlh
zv`9KxoPuW5?~^+z6u91{y3|x+|J+<e`?Ew9lu+B(-s%#=t!7{GHxiErbaTRH`dhz0
zEYGX8K|Gcny2bfqe8!~|6j(Oy`E<RNXg-}psfMi6>D}L4Yhnki?*{S@vyIOm3J1b_
zKSqtfm`ujQLv%-P(80PxKGz6`qU5Pu@GiD~e9}<a)ghHi5++hSp^z+l%K}Tfy#02X
zHv6bwYdD<CrnRs<jmYI}NvmD+O(KrGSJ#Ltu2lCZwm$@oM!D+7F{gl?KmmlhY*zdV
z{?u%4Y%vr~<iE~{&6#N;`xa27GI74TOYs06dZB^0;S_;g@46<LKvU~_?f(8{KG#h_
zJE|`^^5f1zw(W``Z+AG(qEWt81}~mU0X@jIe0*@S8oV$0fDX#<4|q4!x5EutmQDg&
zhj{#^9+5wrSw73PHVLVFcH4bhdu^{@DT7|!865F-r4@LkQ<-s-p&4AxSM^Gr;!jM`
zr|wSX^(MRCs~Q^gN?D8C!0Bk~R<aEUM<QtDVLFf+t=D!hMpK-?bSMZhP*Isl=gYLB
zxZQ59KS*Wjy!<r3pqh@Rvf!+$(5gtk4OpHPN~a~#jeLbfs2w`4aM|v&GX>Fl+@HRF
zz|_;@v8nY0sQm6qF!t%z`>zKso>-nx&mz`5JRvDPVWrWi|9pESUQkA7KE6Mc5|^aW
zsmBAm8G=Obei9dr_;P=?7sB}@Q(T-uDDXuX%-lUr$9dFLB{F@Z)g_6>pHw0af=d<y
zd3!)QRewBv2{(SZ)`U$o3cFw04Y{XE?ytDKQH+U*j}nJG$Ju(l0(h3Wo$a4&10tQ<
zX`_)C8W(GHAI|Y*r{nFJ!Kl+T9=GcN<ZU#Nl0u9J8Y+!)dcV!kra;?Re;9_|Y{(XE
z)o^^8-?+(OB;M(C4sTscUl>MIXA}Y7zyev#xCSB^iCRMfwbDSiS}mi=J*}nuXXF|~
zI$9XL)21uS(7mCUuRI=i`%eyYy&Bp40#=G?;)r~o?e{?Wx!cyR*`9u+lC$7cxx0$Q
ze8#7)qshDx9#qjjj<?Bn>KMXuH(yu-UvAF^*K#$r#I}R~T&pM^oSbrF>IH&@`|0N?
zR)uNrX4m9&p<6|H4xTKO_YcYEHD6<KyHbihCQ&W~wR_QwraUk_Bpxp2@VvMWi<O%7
zOuV`9rbEK<C&|d1qGR#u$CrGvdY!Pt6>Vy=weia>(R5lPMGYxa>HPcvB3yPk#H9MS
z{LH~X{LFrYP%%;8gh`<MbiW8&f(s!2{2fwG2;~QjSd9iI@clD!1Mm)V{yt?yux_xB
zMHoNA<WEQ%0*GKJg!(kE^1`J4!GywKd2JOmQ1U4Yl{$J`e&CCpHVoW6JTc5>zr61j
zyQgAwY|z0Jbn6zJY}T712cz)2F>*H2Q=X-=h`!qIz2o>HVoiSH&fs*`?UqiUMnkx~
z*?jlncD?)!d7Q#(V4S_h2~NcH{m)|TWC;W|%<ja*k(MBAgsIcfwGrnxB+c%_$+x*S
zGXr$Z6>6nT;6Wjj2?5IhC(OxWp=y#|qnhdGLpfmpWZtTt;Z%^4soR0MoBBny%y#dj
zAV<sf?zq|ar>!i|B4ZwhP*8Pt{62FLC!8%l5^I^xMDxr~IHRsfL)fR2$`8Gfed%=B
zv)$00bokKhCoy<}AVOhQQ6vp}ynqF!?7N<A_sJ@+73KGWYV}?iGV;HgW`dV;e+UOz
zt%y$-{&JhssV{cVYPQ}xU{1iDEa=spP@>Xnf<4_}=&JYyoh(!W-4#w;V+Q5g-T7MU
z-L!r%nk0(Q(csyshTcj89U?KL<j71-J|0^;$rm_6ojhadWNdqk!)upJZd-gN)FgVX
zU0Pyn(u62on4i=Q@!G9MBZ);ioxHfXxIJ`DD-G6Lvn+IHryNH)){=OVbVBFgYmS`p
z<^^uG<RHK1@f*q%()_CmjY8ImkH;$#2>8P&eG%9^*J@%+X4_A^UeAWM8}aF}4UA5I
za6s|&p~G?H_UQh^fpoWtd2dtRp`j=Py%OMN0}*(=FO)hJwDT2e2DUG=ETIk3V-G#(
z;FDWf&dWEjW_(}?j7FoBzlsQLgyO*c2-&{wF*x8eM>h)H6bJHS;g>9`-f*5R*UghJ
z&&DU6+q5Q*xp9C{1+Yk%qq%~aE)r?wzvWA!4~A^fYgQxA{wdO~|Kq&03DQvLh%YFT
zIa6<|WQ~M(-8m}O@;Qh+D^hF6hCv&Ra(6n}1o}mY5hk(7boWkzlI-M@q8QKji}knB
zmqZ>9cql<b1E%VH$cgcB8MRz*gpYv;9J>^M1HIq|Jp+xvA1AT(HEaCX4nn)iF^4x8
z3?xE9;fGVT7|P}MDBSLk6<V!9Za4b|<LPdmnx7SlAq}^Bz>}Gbu+@t|B+PP+Bp^EH
z%Sj|h$K!x4dLfPscmDZuEm)TaG)x|6+Qin%p5Q2g?^QlN0q7;3MNhH<SuJ<RIVMr(
zO&$*w&b-j25#@A|xIB^jbn2I{p(y0KSJU~6%^nErZC?H<Ka4R~8x(@?X>~2;tu}um
zkpv<1J~<q?W|o$;W%77{B$z2=Qo}R2Ty(o%ec-GqnD=<xZz+GK@%l#%@9as{pK4~x
z@G2uaK@QHVHtM0kP+^D+MxJrt*z$wa)QOEi@$Q{32$&()do-qnQ?D>+G=jiN%?mpN
z?zzsdgeZFwS*yiy3zDpCRi&c=2Q+A03g(|($E49{2|w2oZxH_Lev(4@bTOSc=-M?V
zU--yiNS*v+yGXvm@`}gnnR&>{>0&BwCP?A=d{VF&RBX1MttUi}0>cl!+wK1{6i<cn
z2ZNzI;cI?Vv3faYk6>`5OdCD8ZxPdSlOB)kg5zP~tF_PD9Y!jv#ds%;Vo|m*1zMKW
zT*=(6X9hN<6cac|JeksOqe`A6+#b*0^=jwFQ}R*bD4Y^azW7%@qj!(+x(mOc#yXG3
z{XXik<4;yr*5%tH*-Y+z;rMxTY&^((joh2Rbv&PQ>V!yU8(7r-41dwC_QV=b=NRjx
zf?N?_m|7-H^+QYn+E$fv^+3^MBFp8vVslRVZ8K8p`}_OF7N=6@X5eY*_k|!qA&{1V
z!=MY7D0-XKTO};Uj_}9Du$9RTEIvXbSBaYrr7;j}4@86kYRay5C6}M~Jsg)&vvqVs
z@37>a=Qm=ss+{iSPGOX9Vt#lFLuTiuF`JX@^W)anFh|qN%S$L`I-QPJTHC%P2A$h|
zGi2Uh$YN*JR3E3n6<U;v_~k2gA}V_)r>n!z7Rr_Tjwh-1d2S<#bViL3d(3(BUqjvb
z1*vV|Qh#=XJ%#gm-E)yk+Da^?Sy`1WzVR;wrZJoS0XnT-#P|#A?p(4eS{?jKolfH<
z*nG3XjT;R5eccqo<S%(00u4qZ=vH!mv?|EUUN84oj&P)PfMgVNP2sho_KABu67-%L
zwpRF7Av9k#2pdJ%Xomii(;7pG%Jc0x;zM@Rra!O_USQ0kMh3bjT{4k6aFSmFxA9`N
zf9!2%Ffzx<0+QKuGCGbSsE$C0*LJOL&u}Y&P6K``u<QxgI6GawZ>pW%bgQf6dd>DK
zlZ+Z_YBX8byet==;0bv9LYb+y3dyhj(i<a*`^c2XKEa|_x9$$b^nxDFK}*RPblTX4
zgi-1+27}U@*Hdi*=u_EQgaK%I>J8S`)|5qMGJdILzz&*3Sq22Dh`6cEYE$<{yjkL_
zQ>tu6%bo38s?Gx*GAc5r7R`A!a<I)t=i6%ZcyHzp6M9cM0hTqz_4ucx)Un;Hzxymk
zIr*|(%m>(%d(SRti_n;F@CdU$$*Ikx+T;GgMc3HA`BLZ(Zb5Zf%x2SIsiO15cpkz;
zoebb#B~&%)?YnCB&i}Q}jm$&#g`o3182gy4WX}dt(1=nT4<S*4i#RT~>nEDEwWSua
z2j3TvlIE7&MSL`&FBw5ORQV;%_mV|&iCC~SHk)~v3i`EHm*B84SmG~tf<sAsd>STd
zwZ?PaLt;;dQ$nk4ZgB*EPmgof3Os+KE|DV@K&r*vr->(j|IC+;YMFC1&)RUf(8xxw
z)6zEsS^P-`ZLjrb8mm=NWZ19zI1Sy>5>?ZzR_CDhi%N(;$q0Ddn1wPKhGFUI5Ne9L
z5oO)c_j%QrY}SiipGp24#v#UBio}CKZyQ|Y2CjaiNP<(VlvUynPkk|LCg600>J5fd
z+V)QCze!)VX|P=SYDJN|C*7g8AWu@|Hbd?lWx3oI`jACr00AYaIB@v6rVr#I>Sv*M
zRU3v3eO9H{izwev`YEm4DuI9I=<_?5thQCQ<>GgYbPg}<{xEdqFMRQII$A?aBsw}-
z>pV{nhl~a>V&xZ`x)WpfU%+8>|H707pda7z85Y65s7fW#jtlhop`$|~%qlWyRN%Ha
zo%uoBoov@%GsvW#q94D>{mlqyvCoI04L3CU#kUY5i2i<?QfqsAC_h^w4@WN)oa^-h
zug>=Q4mbNxwnQF;>CAw9f(gFk$$IZ*1M#o*x=ws*O4fcWCIZOzjD1!o>|cMyp#?%f
zm`0WXJgqLd#u3==GwbahkYZwDXp@vd-fz#~t?$N0#cFCRf{0*SF8!3_G}!H7uOTg!
zr{t3HP!<anaCTbiQS%sJ0*@Jte{g(*B>qmd_FxlEhd(Fq`aiA!ueZls{ih+E`q%vl
zKw)Dss{Ma;f&Uk+<L@D?n)er%WG`2QwvSH&hyl7eQ(sWX0~q#Ve!O^zeF{%re17hE
z-4;nE;P`e^Yoh0T12KBO5u8#;0a#Bx!lTKT(izwtOAcKWI=qjl44<=L_O{ZP(7Yjg
zSP&zY)>=b2Z0PmN)S+uBWWO*V(x7{ykY)>50X`KXZG6#fFJyAN7mjOYjk}j*qMyr1
z;-&d&K<qc-7G&~j7LP~7Y6z~{p&p-8^^M!3ErsG5>gZTaoG0VqFFF$P#885&L=SmP
z>!TN|K}*dJ-}&588Q}@Oyv8-gR=D1s!B*bmEy6@v2dG;bjHg7XzP)_T;+5w$b7eDw
zshlt0QYd@0IWs{5+rr4bB_Ak=2;nf`9F@)5m*545-G1rmzkK6zkMPH<S)@kcYfupD
zQ>}~kaR=lvL50C4&C<VKoQSc(N~BDiN6+~JACZr@gFy}EqGS$H3rgEEp_H(}OR7(`
z9XkXAVZ&t1*QGZI_{bi~D5cn(DJc3>tJ4jHHr7<STiu<6A^pxwM&8xP>SoDyRw6Tq
z>--2`)rSt#IzIs>#%DfnLg$hF@12g&Q@(_B8aHn*h=uA(&cZ$`EpWVvd;cD#Qpj2b
zoj$EL*oJ2F<=x-w1E5$Hh3kzZnum{PG!dLd35*lh<?P;_sIWU7leV?D6F~{?UVDKQ
z;bT+Q+Cr02C<sdo<SY$Jbd#4dxIBrqdEALfd41kTiEr-5aN3V<(+Xx@i$JZ7HjD})
zoP6>OVNmv`hm3VK7Fb(W;7XZ(gooL*YRH<z^ScuA??xqE`lV${$qK8F8w6i7mZ0&F
zfy%8H(+qe<>yFRTGg+AKS44mK2O9)AR&?52eXD9L4&d4AYW%CAOSr)WCI<`C$vn%-
zd><~54Bk6KKw!ukH#>v6&L`-dWt&=!^x?zp9uKx>$4X~52vHc8-A|8?n{%Z}%jE2I
z4j=DcXZBI}JUxJ~@{M&uBAH&RErj76FAG}QVO>QDw=SdA<pRR(=h2*7WY%}2Vuix}
zjXJtza<xK7TmoK9{Ee;qvz5g<GZdLj_U;$=kMp&Ol<w1qmxqfc?=$Qy1>wi%I~oED
zxdTz<tH7HBC9-Qx8_x0@z;Dg;Q3FR0*PDXvsq8CtFnNzGer=Rb;jh%|TdeZ!Y(JW7
z!erF_xj&ZSuhD3&Kel}L>anrzWsAiMcEF%Luq71CXv@!MI2?<XX~W&@W@BXKFH69;
z4Vt5Q%uStFt2G+DMmvAp_;??bKBLb_w>@2yPfklC(SChQ=CtDc*N%-nb$GcG*>rfi
zK3)>>Qg_!iRQDS;Qu#(69oLtdo=w8wu-S-2;&M#3s>q_(L}l?_doJIvHh=Ww$`Q5S
zq0=W<M*1LpY}AUm{LFlHSQ$%Yl}8K9f?8fwo;rURJ(_8wRiafyVm4d;2GqeW+Hi)z
zN#_=q3z&(8N;fVt$;7_D64v7Bq|g&OEsoFwk<NRd(EY}`TCvXeudaNK-ipI1r4}cp
z)XhykZ-FU4O!Be=!NG_)x65fY+*`xLg&m9tpLWN>io|tC=5_enN(0+#FN@<SmOwM!
zvlms{v&BCmv)BWXNVKX=9_U@&JhlS<Fk5dl<G5U`4n&@<xN#u2N+mJq4OZ~5yFJq~
z8G*pq_8O=1&F9S7-HyKvM5c8~d)J5x`O$VKG4NEmTzIeBq|TaE@t@HUl880uRw7qn
zO4h*>a0C&Hgqa^Gm1trWj;9M{dB4@a<b!flCh#e>3cihJc7`lD9O&i0z~Hi15$O5S
z$|I|nC?2*I$2NUzc!lGzB_71p9|NYL`~A7U0|vd`pOW8+R99<tb|e!%w}(U>UM~ZK
zo<9>D?oXGtPPu{2Yb8IC9aT>EVbNj3kW1~ZZ&8riv%gSTyOW~R3r9lwd)H#t6CE3|
zIGPfuWIh&sr8On>L&Q=^^mrQ9vRThhJ}2{~QF9jaL}t@}LBBi$4JG2bcLIag8?c!z
z`!=4g4-PI&CNhJXe9ZSh@@Y+$RcQAr#}l~N*jjr90f!P%M{Rj;L}l{)A@{m7Rzywa
zDfzs(z~|a<c#b1qMfp)aAaJHocB`Iq!+Oo*l6MH(X1b2b`E+qy`4reG(bP)jiz|(S
z>Fjp8`JhVS(AbjL<$^WQ<+`ILCqxykg91=Xhr<iH#)==J)>T8@uFXvx1)|cF14(i*
zIZHc>=vNS-vt>X6R;_WKVBnwS4a0gT%MaP;5uaDgTi3?a0;fb;wc7%V*9L8E?Zb^q
z1Pyy=WO73S(ok{RmFp`4ZVGl&jd_q)POG(Rt9}2aOYfw27SA2ZsWwk*Y3`d^gAH7S
zBkH$G?T+mWz~MD5jA@EZ+sm_Wmg|orHiBt)`c)|5f1()re8ZO!6BkP^HL#JHX%o%u
z1_LGN*w-kh_Z|siU_T5i_xZ8HCMF{cVQA*&_J&%4G8jeD7LLOe1paV%6(?Q<R^wcr
z>Uu-It;L5-AsOy^wcWk=S2`)Nb66*1aHCL|7@s8E`|2Ck+rVGK6V+P0YkvxPbZ<W}
z=x9a*QCwb6C_e93-I4Srw;Kq+LoHhy*cs&WcW8{y3KP!*Q$x(9;VlUi>t#%Rdm!LZ
z6^&xU`s!8f^_tx2HTM+<Yj-FG<)1MnVRStBqQebu9hh_hx_F%4A&bQ2{f4k-&|VHU
zI6KJ^t$V&+j}i!j0q?v-Eb_<xveM_n`*5igqobH`b(TtDa%vb_$q)0VzvsHUa*=%k
zDxLZ_4u`#g&2In6W+|11<l%>v7HJLb0&oAI<c|yPZ*nii8kP8w_#8d$)X#S~T{j=U
zJ~JLSR|6fNk=|fW`ckc~nObI`I$|yzh-^TB3C)4g<LXO?*$i<>4nqTVi}WM8Y~)b3
zFS6gw!FY~A@cpSlKGb7B?AD1Cr(rVe^W9mwa6SW*F=(`7O@qxQ^%fk4hK8n0r!(=Z
z4(cTgd<~OXiafQe2+x<H1i5@kCB-sDjv=R^gYuJy0(>-Ug#!KY)n-giL#P&~qtSPx
z!Fj7ogorC>K3J!BKM`9ytf2d|<&q6Ew~XBWKUbdiXp<cW<0)g^OeJ8VEiHnGxm+Do
zu`i+J_W2?r)pIyF0yh;`4@(Z4<zHxfE`rPJ(Wn*6Lf*eZ2qJn@Y%5YC+@K6zS$%wd
zjHiw$3i9UD*gG)jLBd4=%6`77qg0-O{5P-aJ2(eUCZNwM@@@8!otL}id`1I{84{ab
zeXTfMZHENhJbVWr+k=XU{NKBNh*Xb~D!+L2fo`%}d~#EcCXu0co-U08XZT~gUnquH
zWOH*eHJIitjxlvH)BPD@foGkF)(Puw|BcIt7*#e649E{{On;MrGVPzP$?!QUg@=Eo
zlkIiAzi|^zN9WL2)iexb{gLlW444;V!CFQiiNxH`@geG28N>)a_qco}Zt_EHPvJHr
zhP6s8Q)4vPrzMq$jW4`dt^O($%x=&$BDeegVl)2oK%tMCMW<Gmn4%CQVG6CDIiGhY
z^wo${Jb^f+Th{c~dO(2Fo@6q+o@X@BUXFzW4mwfUB&ALS4ht0X8<53#p1Kbc=TRC-
zo|t%qKmnnoAKgKfT#F*~PrJ16!|-#j`suZd5eiC9ni0mP&rd=<$qVvHE*I<iEAw=(
zd~iJOJI_wc=5kRt@XN}@J;YK+$@68}fqZCrstvoGP^FXRfK7NT>XODJ&2WGrz%djs
zl$WPKvCANbxYb>wiB)Ghg?#pIhCt+MN;#U$7_qaK#~9Y?a^lDJaBfj3vdakpiN(eD
z3i-R)ey?WPMrC&-DJ!nw*B2#e5cS&WvJdtY6Thfi;~`0M<o=ije~9P5tS4)TxTeCO
zYgRtqx6kUB+11@Wx88uJ&HD|sY43yFskBfD2y|kVNK4gvdoo&>-JN~4to8QWmTbVg
zhU@=>E*j)$re~mp1*E>SD+9GA6LN*lbeTHUZTZ^g6~E>7S8I$UiM*DXxn(SEM}mSR
z(Lgs>X6&w)pIWgfd!*Eyj(o6hTNQ`5hvJEN)R}GmP{?N5&cu3SP$y^2BwIXY`=P{h
z^5|r)a(5<pNjcrTz#+=cLs(6C^~3PWrf$fwyf7n(@H?ipkOZ}MRTwz{p%~awjiI2g
zr(2cf(lq~xX0vC?gWb&l96JK`&_SVxC4Bj5G@}=1JdFnS!FcA}c%G)Yql_Apk??u?
z5~FdQMbY&jmOfw&TB^;kPy<e-i6#8mYmN{^9^iy}(mLu)@rcal5*=pdlGg^|9H%Q3
z{v2NjhC1q);ZON7#FFa^3QT>k%WT(zwXQg+F`2MdRVo=;s5FfcRBZ=UrpSymn4NX)
z{any}X(;E`df=RP5b##!WN3kRe7jg}&Rb(`{!w(Q(O_zP$$2EGmO3`(1A^A7v@;OP
z3$7`<IJS(dsedhgb$+v?^4(7PQD_IahT|PlyjAVTL8RgVRBKO<&?t77mq><6k)C2o
zC&x9eQ-X_2O9mtJRzpSj_xsW~wTs{ZsnS;JjiP`6x4V6Wwti=^T8*x~5hee4=R!zp
zwCvx?8??RImqu-%?85S}S#62%euK=l$y#5w?DB_z-wa4DRj%xxw_35gSmznP%nrNB
z{;XPONF`Gm9FpojQy>F3XD2sF(rT`y+2NisQ}A#y6C7uK9LuJ|%k6p#mcnE*&?PXw
zSZ&ehj30`?!q{lF(VY_aa(9-Ou^{%;L^X)5<xe(}XfV7jvIUy`DXlCf2FYx>2=#)1
zFWCNkr)Og`xQCwl)R(!|Dqm|XEdwiS7P8!6H@j~&lka@Sx!B>=de4?GNv+ugm8D^N
z&L!05^O5QGR)<GT0TLD5tJnfUc}Yc$T(>ZEPd;|e2GN6Y8Q06r&--t!pJq%(1y2|4
z9<djdi?K~#6bl^jL?f{yGLDz!?opQso|IXlZPtKjShLd&79BnCwjz#xIF?MTUoxSt
zQ=#DJa67`OItNhO>!p^|-xYK77nMuRM_AhTe=Ix2ufN9RX~t06>5s!2%NIreiPqjf
zG6E$kA-v+tFv_rbzQE&M9c~G=UENBdN+2MSMg0wq{B{2`lN+3Yo7?tqvYUU9A0a9t
z0#2QSy{6MOdBoPpscjH|0lwk?aPpbrF@BycRlw29x7m14Uk*a%T3%edt`m*Gaj>oi
zvNJ;C@MLw^v{#3b_&v!`BB~6JodM}D`9${kO!18gOh(j-mGb^?Ryl#Gf8$9aUaR@M
zJqOEIo{5E1LaDS`_Vb3Re?n7?#iI$I%-GdiuR$zSXc<_9fEli})F*#8V+wPs*6-)o
zn`o)GKvFKCA*!ygH;N%`R9LWHlg|+ZA5CZEQM_S6D~zy0I|m87tzRE7WcSVROm>J{
z%BHhdz&QgA0c^ebi`GW;d22Qxdq@fuQkj>epVVNhkoeo~8m;5VCH(NXoVx`=X|;Oc
z@iGwEWHc*u&3GiMuqk9L={>Yc6?$i>tPfWz`E(vSZ+5$PsT8gx>Zw_+R)=nirzL>+
zNHUSO|3LO3X`d4-S9ZhO6Bo?LP5b+Yj^$cgiqA~@WVr`IMfL~qu}!e*{-!ni^=565
zT2|xzF}O?49>Z#-UNNjv61=xGQ)tP7$k+CY`vT<8@(7|>XD#v7iPo8rlApc^Cc>&s
zrM%Uyna?jU<9z*9l?`4nk)F>Mfvj_Xx(=sDlqbU^BRMU;G8Z`huoE2h>O*a$)l~8)
zWR)hfQpe$jJI(z3D3(Tr{_F!ad8!Ntw<Qrl039}z!itaCwFu^CW2aG%aHwjT#A=0h
zbI;fcH7nmiTo!}KpQ?lYybyNi0Q!8%g!w-b;x{%M@Rr@NhgdBHrkqY@%dIX9)XHTc
zKTIZc_bvGv1;rn<<X#KaiR-l1eyiyl32R=l;TNrD@_O|Q^IWW6!OD3(U8~^~!N7Au
zw5UkMQx<24=Kh2L)mwNA(3(#v6+O<V=O0KqWSB1fCY6lGghi+8E=&2*Wb&n17z+8?
z`RrYwc-rWa;${VWXw+Xq@8|ot2uQk0x0|@*bJ2c<u9~t9R*A0@G&NKDuenF3leq<Q
z_1S3d5W^G+#X!W%<YF2P+gGS=G67<-c_}%^v*naPi{62;)%5(b)|>q?5#b!K#M(Mx
z*BoO;vQ*}Pg4bfTN-0;8+s~CIst1q(0enSH=5r-uyt?WI*?%4tQc!Z5FTM#Z&~LiU
zbg<}4dT%&et?1vUa}+5q{n3Em-GPVg&hXKmBL<ae@&j6GGqc@2&m$@m7dLNX_89rS
zsv2nsPq+LvV3s@Ic+P2#bULu?$66P-(XrR#CXvfDy!bk|+tu-&6d;>LUWjrc#3e5u
z&9&A}yHE{7p@QReS<I$?zD(MvcJfB+%%LU+65WW&_(-LdnMgMq0|>#8BrW^l-DDhv
z%%ODNJQ%dL>HfhG5__`4`Kp>BTyEp1W@Jj;daD)ma#&xW+Q7n?2S(qO<7|56RS;MO
zXL~R2CKmmw$A+a-OSf^Xu(1`CJZGwDI`U+ZONmQ6y6C(i%$;<bW<kk$lc+;o`FMX}
zm<-N6A&McBQua1UOPvE;1)<>kGuP)R0`JdNo^CgGC`r~$E+7&9se1rZ$gy0q_iYeC
zKT}5ABJ0FB^9j3_MpKzGmbRtlMwOEu(Bg8bA>>sW8@t*WAR?ah7f(DbR-gb*1bgo)
zeK3P3G4xPHhX){HbB{UC&MsnY#w^<5#bW~}{xDmC=+z1WZ-&1kgm&eNzI0UN9X$h0
z|23vN^LVZ+K3={s$f81;_Kb*1aaW&YEyjSLTlj1C!+%iV{owC#Y_W$Wtoi&iRRhtQ
zi-5qkbVZ5;3WaT@ABh<J&22o@uV))JSnKtcP%<g>o!cXY)s;S*R_{cc9${!VFJvR3
z1lp&P;>OSIyZmTCl%}PMYq3!Lwfj+6s@myz5uYp!W9~(>&6TL{4&obbA1^hehezbB
zYzik{${d`OH-jJTXBeiYBFI)<-&c00Yf>PoV_FO#+nBG}qp32VlT@YVyLhxSVAR_-
z{40|Iv6Zj!a86y76!M@|YV1^JwE&N#_ueCddudwq#e25j;xQ!V^Z@qIA;PjqpVRYE
zu4uzEmBoV|k~NJLunt*Dh$@hO15qnPGk3Gtmvw|jMTLw$n<p5Tb9fbPmX+cjJDp}p
zHEW+6ccL*07&PkeykLah`4%SSQVSlN^;g}rdGI$K%t<dYmcK9bl-q+*6Fd_06<Yp&
zPdC*Fb>44THpH<aQhQ9Ei*j|&ru`KuAZ2Ax%N=an6z}lb%wKsD_;v6;d2sULX4y58
zqce%9B>crDH;bwYDx%k9ulHxI717W&5I*Co%4X{|og14K_HhJln=JvJWB?zC{btvy
zu4d?)K!~IqpT==UwegudlzbMulBJ%vE?>vlB6JoE8q(bNKSolSMA(Yt+>`FHBPxgi
z8+dskbZ~Lvc`kuUwk&zxQ_p}R>HHyj^eoa#GW|W&L3RVMq60UAa(Xohg-LpzBpZt-
zF*L?B$UF4-Op$*gNedu)bL`9iIIm=Ac-PVz{fS$S4ki%t9eeG1p}Ht2bMe`nbJyqb
zpW%EMu;lR}@Q6YxtSGEw`L;35cysxi5K6GFXNSz^DaZg9P9YokC;sWlJWU9|`%QZX
zqrT@=C+`d>>8ByfBLR}i2V<4i?_w7g5KqVsK4aH8-I=4g#uHxcRT;ADZOm=Rqh{QS
zA9~OLOp(jEfAGpA{<zQZyXcQML1^c@UsNDCoM9;DU?d==h&Xk<FF*Ob#M4I<NoK>F
z@y>gj#gDy-sRFiq$h7C%AI#RbU-WJoD}*J;;+E|v^E(%q7Rq0BI{1NWsZso6Z99Ar
zvH<awMun0~?Mt`v1$<MX40Dv52T6fKp%l0+MbiEF;n%aX&5)w-?uX^JmWSPo<alw$
zc(;7OQ2!|%?sTH-z5mTqAwNjv1|3KgP&F=EoZiY-3T2TjHb4EtK3PV&bx4#e@n1WW
zYJWgJ<}n3{S!VyWpE17V8elY;5DZ5f`DY3nJzjmSK093`yrt5D(2guenuE4?#m({q
zzcP3Qd*^d*ZLq8W9@(pXhmY8)vh(+iTAU6lg{_xuGI{5}3|f9Xf|br6U19co@^sU=
zSo)FJ63yxV_)?nh9<RwW8-1#+=pCUbJ1@_0mvh4p=n2r(oR3>fxO_oTn;!<v--K{S
zY5+?f(3w!=J0~iAj%nddfpogwr<9&E`MnQX9!;$(jsgr@2DZr;D`ATtb^c3vVy|g$
zq*lvah3;^V<gs<EeuS6=wP|cf4*6XAqiK*oqA<9}T9xTulNnJuh7|}ok<6=&c)bKi
zB~XwYmKA|0;e?G&nf4Kt20ja!%+(Tn!*Zz!2O^f#_u}InQe53o0MXB1^z``}%k>U>
zKMKF+^xi5ti*JI?wD#kDhJIh6>3dtyx4=d#G-0tG@&;_htv=0qWJcwRTL&svHvqBe
zUVJ2%hRDKZ*)%eN%&OT#%z(~XJidPCK~OgkWtqCO`0kNyUSJ~2vEq*vfm=Wv4BWf`
zG6TNn&O11%Y=LL>(N-J_jpxllu<RA`t0;|rU)KeHU%&Fz(~XH{v%T62m-48pcoHw(
zuY#XwC3kE#%K?C;yVY0x0|@nOCByc|QVE-{Cz=3=(goyI?fJyCxYYP-ffBz52JgiG
z<pD{dnXiT_DcFs&HGENBBS*0R&mo^KAu$Lr!)^CR&HjnmEwbD_04FkIl%&1^bYFIY
zmhM)OYBAG5MAq+*n2UMw$qwZ*Rd|5hiBX#St*3Wimh$>kLYyWX^bPCj5eAhOO2Hl|
zk4+3Z{{(FpZTeR{vA?)L8p0!b2F)!k0>z-y9tpr02FJHp?C!hY_W?)chF-hrx7a%m
zRtT5N`6j^lx}Ur}KW+j{O_R%ocJzyQ7BA0$Y;wqKv4TaRn}^5FbG^mu6*RHS>(URC
zO;$H#PEMzu>K%$|uLEnqRYT9<;@AREWY_-?#ltJa9iC4;of~FfeBawttm8(K=!;di
zCc5XG7V;fhDrrsN9LEiZ-A`_18~#HRA48u3o~GT=wn+OK1lxoh$7J({;?zp1N(Cj`
zvn=1U%N{6aC(pp=Y1)4zbX*OCbL00QJ(S-Sa8Q=|rtUJv`vmeP&}a}T?9<X_3Qyp#
z=^0pr@p`>nU8MXdpgB1OPUDLCij5+@uVgX^PanrD=K7<HcD|`kDwSFy7QxEE2~K0b
zJLDUTK#;322rt=YbMs28R--?k-V8Dy%WlM1^AnykpNnCaKz;|BFg#fF(cr=Z3tTSu
zZ(kvAds7@R7Lk1b`cmVh_F!D7**>xDyd>dVwZoDMmZF@`>-i_@FQn<-VECJsI-3c$
zOxj81dDh3jUqV$Xw7W6(T82{TxDqRE53DM+YO!hN8pLlUP(2(kf`H_*`f=dxgdrTC
z2LcF>yT@Tw;Tg2K#V86RY3qbJ_g;t_r83*xu0-)rGGNuNE44d&5Xd^1efCP`0G<OW
zvV_p&LYwEQ$;-=2zDAMiXBH!@kI0qF7&sB9p5n;;>~Nf=KnA-Vu0mncRt0ibkKUo9
zrIA&Hg2}iZ0OKvT82xa*SjO7s6oNnyJY}!8I~<Zf<#P=wj38iRowQF(Or-rDS$<k=
zIwgp%5GEL?8vQF@5^vj^n_GJmU^5o0%wXa1dHuq%ew27|ozU(4zmb6&VS8J<?e?x&
zLvAsxv*iZ9X)EOY#g0)Dor_5hgHj2zXap|Sb{qfR=4rT%PN&{NrWfrt7Lzf5WhT!p
zt$7L5e<0$KgynrY@2`DPI9%v}4z<016<J>h-5FGRvhgK^8t5(3Z<SLEH590~It!7;
zVaxRjj;yM!MYwzZsJr<6c(s$Ke6~#OnxGYWAF7*FXnZG8|1%8T8JC}A3YW<;c^-K#
zkamj2Vu7l35+V$p_P0_#jICqUpbzKUqo02OtMzIVuA)>7F`0V3y{<*7CKNU(@#ZA;
z*I!fl_X<m+F?6#Q$0OGd*(_e@IxQIDX(oUn=*gGDU=(b|NTh;g|LfKwH+eCc<6osR
zD0~jSta31}l_0grx0>x~|L%Bt*jFAoTRday#-oA;AH%6ct%S9z=PRARitO<~pxCi_
zxRfN1m`g2+wgQ;-JIfIor;TgvE(DxT=g5Y2A6vbUm!4->yA830;*kJ$NWu;PhxH@o
zl-g}zKoDVS`}E&5u;fp8-QnrIM}WN7Xi5F8CJyKr)fXR_565A^dH+IJc-3zVbn`YX
zjlOndl<aI?`?TDim!I(7&4K}GvAMcbOEf;S2f)_egYnk~I4s4lUd<B;9s>Rq7J-rS
z4%~Tz;Uf8<gnyi9ML-Z7*66!~0AhIXa1baIg(6aWGRQLw086Je1~g3Gcf_Iqfw`>P
zm?EiWXjmMxq5M_NRSM298FR9e05HZtA90h>9y|XK;L5o9sGX03v?QO+!?#u!--ate
zu_or1pK;`H#9#0(aL5*K_8GeD_Ux;E%~t`|jHYt;*i2xgFK8>X(<i;j>4e+|@byU^
zA>4W8PIM!#iP(S~jmzbty9F-I_IcFoT$%je0b~A~mfe1Lbhx7n2no&7W&rzrUc>ls
zlIYZ(y-YaB&ty*VI}N-3U}U)_ywi7!{coi5$~0<IyEA|JHTj1KR`cUfUestb)V2VT
zo6O|`A4?`%0Y+6jkF5pZzo2-k($rG`09KfJ1x6_r8*Lb3$)tmP4#tdJM1U*;5ec8q
zJA5inbo(Z0Vknb`i(9)*7naj_G`V|xh770*rmKIc_D5490IQc+`_&b*^0f-c{R|s}
zt;vuO?IUQl+n~fEaYeaGJTp7HT=tkG5@|Vr%79Ofvhq*a>1?c<E;kV|l@@scGZS<E
z6@1o$`ee!FcpBBF@D5l=CGV9$vZxDCnBSC%B~O6QI7<usbFpGEQ!7VNahnKu&`@lC
z_T-}7j#?d6uI!-rO>wKb?-%g9mIAVntWSUg+k?u+#x2(|38WkPe@khIB~i#?ce(Q4
zI|&jrb}qQxzIzRfA|8MTxVuF|wYEg=u0zZYE-ks)hP+UOxz3kyU~<^Ar*qbc^#p`&
zZZ2Zq;V=WO=s+)}O*Ax6%DYkSg4zB78gTz7Ebjk8VRXVV<2xw!8&;kG$gVI1o6TyM
zANa@i=~C_E>k&$nwb(EuK+EBq;>PHNPH8W+yX|n|1MQPefVYG3F+)3bvS#x~5YRxX
zF4cOl1;OEsnq`_+(O`eRnC_QyeYuB=ve{;JQjZ^I*GH`;%Ta>h2Dn^;W6OH5z0nrO
zl$z_*&`lUWL~>KbHo%U&>3VN}xfOYX@ZM3Kxh#qgI=SAHD$**fkZ01z`I`4=Ih+eu
zI(a=lcU`v*Y!bnGZX=PQIIhC`I0jj3Rh5dqJjW+Yx2hBDxe|TAjwGTN8>XU3Y}~Em
z@ViQ3*W)&seM|_)`>|X!Sf%&o&TO`dh}5sT!pKULfRUeHqCLleA}*Xs0NN81l<Li!
z{+@?25GB<6>~z(80*Q-@=Y}O&DWYtx*hL8s2avo<rYw{0dUiaBDC9EwQpsS65JZL5
zCR8V%A_8&nozGT%1^glGz3JhYM|IcR+=MifYF0QK%>4gK#1YjnI;IQW`rB_{^(L8?
zoxMsDU%$#Fn)(4;w`inpcmAw8bHR;2B!VkGT%D|esRBfjPQ-8}>XR}&F8l94Xf#1D
zq7rGWcs0gjppTjCWoAqcBXp?BMQq}_o#tN+tBG`Hgitww5T%8(ic8~_q{I`lUGubX
z*&BjZ768}DXmg&PHG(BBc1QEwY@zaEqqEe{D)jm8q_?}9NX|X!QqbSL)BWjJnp8N<
z%KS16I{0;tp1-@h|8Irb5u-&pfJMR^OJmu(-B@dJ-s)~1W~m0eZS4vG0A6fxqWO?z
zp)m5N)F?pxgGwDjpTT8@!lT{kow1O&5$_uFUTFNQJe5UmG4~Uvi~Y8y2{vW7;fu4C
ze3_|UJiU@%nr3QRQ0k}*9+&RwA|rRFXLC(Sab3^>xKlcpi>27u@tj<duJ>%I2BgVE
zIzaqUjtDDh;Ls<D!kv`z_lTTvOLoujvveOOTa{5e^vJ+<<Eiiib2JkNB1ZgsO1ZA`
z+EKI-iFBz~cJswruo&Z62udl3g5^m*%MDu=Ma}bo_px)skj%%BHFLJo2#1fMB?Ni|
zY-dg}&{4s#u7Hu+->Sg%Ye$<V>CC@x(hu}TPuVB3D~r`$Y0mfwIC!;y+kfWNlta9G
z+m%G6g-j-ey5k|^Ak#8t>UU1`4^duGPN41qGK({A5WJvTtQpRKGs>1#n9Jj9tu9-W
zpvX!Fyu6A{Ab<^7*l5!GI+6F17C9`ZQXGB06Ox!j-!c)k&k25422Sb`iR<2pP~u^8
zh-f17DSNn5jPAg4$$^ks7a){z`o_oMR~nwso<;Az>+El;_PM>Ea=iayA<cKqNei9s
zl}lWUNNcngg+iv%0>(SUk}9+~jGh5F1M|l`gGTRDSQ1B`5Wjfau)Q-L4}?@wYiZg!
z58o$&YTI{DTGgiLhY#_mykI!I+zheYSZ)!RGL6od#zy=6*37Ci6~AN_3sziiM=)UO
zP7es)+B{z7H|=7@r|hCf{OyC$evN@5@m6knkr9#S&r5JAzh0=<UkC=m<Y~3Cx&qU8
z1O^mj1*p1<bw)VA2^94C?dAG5I&uV&6fu6pYT>*+IXAak(Ob!?e5%pL2bae?Hp&60
zaB@kZL0j~_i-T-H_SZ3g>l0<IHPt2}Wx3Sg|HG!vVn&@tw53d=75)62=DYi;Dy@r7
zAOZoOuI|S>6kWV&Tx#2ha+&a&`yKly2Nj9~9HT^hZbM9XJdS~-_>;Nfe05FPYL5xt
z(X38HpcHnWYjMF@r!g4<Oeo4fq}up0WUJTqeU&5g;FuF;B9o_on{4E}-L&DUVg)Jn
z#P8^f*cx>O;^{eyh0^TFPu}p#<(kD>K@|(Y$VrvmjPdKs#h1IXvI9w0O25`95_N3>
zq(qa5ni{zRBsc)cX9PgP3l$+CVNgu@%dgmVsNZU_KwqA14n<if3~>FAIboEJpD{|Y
z7w6|IgBav%z?3W0{ipLsD09a8s0OL3Nuhs3xL1{zSvB$DH#2Sbg+?seZQGXM3UwR+
zJY$Z<!g;#G=|$(q0O0{qI^|S!5z<#kxNThRx;N{!*4jIXkzZC)2%|t^!(Hk51OrN?
z2`W{oJS>-`|7NZ&pQuaCg^VUNDB-qzXq<O@8b6#g{p@(O95J&E23Vo}|4hkLW-E}}
z@bmr2gqgSXdYi9sD6(A&i`8<tLQbQ-tuQW`F<XTW>{XLru2Wh=n`J06S^dm)u`Z=_
zsZOVFi?xd?p`Zr2VZPN$0~+97Pdz_Sepl}h6yc>%PZ~dJcV`FMOteE+eD0=+hp$iP
z_6Oq;z?f!^ITXkFm+725lu9Y1pXF4@G=AzbYBh#?_W9|KcZwqshb4)5H)DC8AAyP@
zwn~M8`M$J{Fz`xp)+{YSa5yYIlnaa@7<{}Fg|xHrepDvo>6Aco4oJhH7kH@roCjbY
za7ex)eS27x2LUf$Bf@^~h)H*okW_Jj%g6T%z-eb`m%RK2b0QF{Yx5(aoxoOW&}wyB
z*hl!wlr1^Y+w1RRGMXGSBO-zpeq5_nZ=O(-xNN7O3p7Smjg1Gvu^o-iY{Q`ypc3Fj
zVDFKdA&*$>$uq|CI`lnRn3#|R(hQb`NYrAl9P4cakUhzv=&sFPOA&fd@afKA)a&=%
zDd@y0p?deR7Q%%or~4T=AwNN7_@^WZT91e0Vc&MyEWT=u-=5Na51)u71SNByuV%#j
zU#OJJ*iE1t>Z|polBQ!6)|u9pwZAG!)}PEyy%(I(tG6k>yo}zM;F^+Jq<{{?TAF=c
zFJTt!zyt6tkzvlBOSZE;0`3Junf+-{{E!2ugBt{lihMQE4a<*M%{slGdgwBKGh5Ck
z6Ns8SD{#o~;i`AG?QhMI0Zh7kVrO9F79t?=-XkmHzA+cTtYad7tC=j;KQ2if0aCIZ
zP=VxJNyr6519K%1MY4O`hyQU<6s$jH`DUC^U<sP7jQ119{!=0urpey&G8Yjy`9W$Z
zfHdoci=0EnAQW(*W-;3YtpZ`#A2%mMK<z_K@n_4lLPW6(P)Z_equ8COY89k)z%bg*
z5jHV(T^}%7B|q8YNAL`+Vl$hceoXxpjxGn|qlR!*L$j|`6^{P$`oojRQwtxSp0D$T
z%SmO)z+}lWgjty;romHFtMgHrRw4k2=1T@T&sc7a>r{oLS;phJgqPB2Fp=i!LG#8M
z&1<mC4Mb*rk{Y>w?TWo%ZW|OCIT7^JHzthWMEaFtN+@NuvCeb4lLwh49>Wc;lD>;Q
zMFy}{zVn5U_AldFtCqapr<m%E&ir5ZSfN4JImT_|epZwX*%~=9?9oct+AR-<#i`48
z&DZ}A?{4lr!>I85%mYFb5HPxK9S0`R;^Kde&=LwHROcAL)3iKP6ZQN8j-Yju@_6(9
z>4UHm)8cDFX{cgaZ*Wm35@1+2C)-C<le!6?$Ndo63K%3LrmX|J6V66pfprQ@WG|-8
zzDA?Ay$LL<1UQ+XruI$xZ!ECG2n8#vxm_?%m51Bd+fO%HiF5t12Wo9YQ7*F3A0tWl
zxL$L(E^^*&leh=n{*X=jL??-<=I1mjSJIU@kJJ2nB9$!1OvZIIVGBU65JO}@u2^B&
zKPZ{-w_=5i3EC<pdLlLT4VaCdR$t~wzybKug;mX2PjS<)>nMG|(CR<S37r8n#O?VA
zMXOYS)*^d^t?+!G{cJv?A@RA{>jnLSAvc=wyW6xdy0X0+oq{)rG&IikLyl7xM#fw5
zi=elxePc9boa_(6-J5pLqu9*J3?3Kd=0K(@8WQe*yIk2V@IY-Qa2-`DliMvGRlTQ3
zKTVhg8khwyL(|l1p~;H+2hXR4Oa6AD>10&<Y<PBezIULxz#Iw}L#QX{`AZrT8Ba(c
zD-pE84ba??z{KIyd5%Bi&imVkpC??jpA<0M*$oU&GwQv%&K9YKmAB5C{knnUvRYR3
z{(dMx+<|jh9kkCr%yz<Q4#sM;+$UWpoyJzJltN?>#DCHtC-B#bT3o0d>U^VfB4$TA
zjg>e8hTKpNp?i(1v0A;^w<w9<q%iW`=>BVrv?)l~KncMfe^h!5h*K_YJYJL+ONn=O
zC%gcjGpsJc$;cmXmw?x`Yas#@Ii97isixQ!h5z%~?pOa$Ylv$S1s`KAW&{Rsonna&
z?={YfS@B0=Nk6bFj(z?G|8d{*mBNWvEtz9={QuzkdoPjdga@<UwMHZOrd)Ftvzt4P
zz|hwP&oLvvycfVqA!_@zM3L0C^*on}P=v2fB&HscHr1QY^#sCVxMKeowXX});&hQi
zo1b+0HSfI%)iOKnJg_7mZmvmqt4Cy!Ks@FY8stt0D~oZRHYF{Br1VVwK}m;!>MRM?
zQe5yIylMzROaR)?Q&*vI;`hj8&SJJ7_^y8*c@+_XifLqAYAlbUT)4H&F@BsHxY>}G
z;Jg6;P@#Btf*m#XN6#Jyq*zh0B(DPJ#<*6Ceoug12nrWGWO_*B#X4~fv{Xv;-VUen
zw^lsLY~k)kz_Tz&Pi3__Ml}SbizprT#xNO`@Sprr$SC7IP7xHemY@6*N8V}{<(s>>
zGZ?W~A%r$}N5+;wzbi$S1ME7~BqX(zfx{iiK~BJtsK2nIZ2_FQ#*1txl%EPlKYENX
zQ<=@X1f-}5S_Z!g5;qwx*^@PVoy^w8L%aPs+Ahkxz{Cya@?R_VPAyF4e+~*1l&Cqm
zJ)Gm=;^8SF7AC`JgA*XFmYIvwSZD~l6rnSe_^|+!<eLTUgu-IGypg!!`b*P?TDzT7
zXRIP?p+Jy?v)xjMU(Nvd@>~W0X!W1a#_2oeSdmU$2O(6?MR>Rg%|9s$e8B@CVFf3}
t;Q{>YzlKHs7lHbJHevezchLvrI`w7&lUg1=@U0i0Bt&F{D+G1@{s(nG?`Z%4

literal 0
HcmV?d00001

diff --git a/documentation/zensical.toml b/documentation/zensical.toml
index e69af2918..a2b056c92 100644
--- a/documentation/zensical.toml
+++ b/documentation/zensical.toml
@@ -36,7 +36,8 @@ nav = [
             {"Queue Management" = "Organizers/Running_a_benchmark/Queue-Management.md"},
             {"Compute Worker Management & Setup" = "Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md"},
             {"Compute Worker Management with Podman" = "Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md"},
-            {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"}
+            {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"},
+            {"Human in the loop" = "Organizers/Running_a_benchmark/Competition-HITL.md"}
         ]}
     ]},
     {"Developers" = [

From 98bfa89fc4f6b33fd7f60f6dd7e9c079b192623e Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 21 Jul 2026 13:30:27 +0200
Subject: [PATCH 08/17] HITL documentation reviwed

---
 .../Running_a_benchmark/Competition-HITL.md   | 74 ++-----------------
 1 file changed, 8 insertions(+), 66 deletions(-)

diff --git a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
index b45049032..5acdf4c07 100644
--- a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
+++ b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
@@ -19,10 +19,10 @@ This mechanism is intended for competitions where the scoring results must be re
 
 The HITL feature guarantees:
 
-* the submission is fully executed before any result is published;
-* no score appears on the leaderboard before validation;
-* no detailed results are available before validation;
-* no output archive is sent back to the Site Worker before validation;
+* the submission is fully executed before any result is published.
+* no score appears on the leaderboard before validation.
+* no detailed results are available before validation.
+* no output archive is sent back to instance.
 * the operator can approve or reject the submission directly from the Compute Worker.
 
 ---
@@ -31,8 +31,6 @@ The HITL feature guarantees:
 
 The HITL feature applies **only to private Compute Workers**.
 
-Public Compute Workers **must never execute HITL jobs**, as waiting for manual validation would permanently block shared workers.
-
 ---
 
 ## Activation
@@ -41,10 +39,6 @@ Public Compute Workers **must never execute HITL jobs**, as waiting for manual v
 
 Each competition exposes the following option:
 
-```
-Enable Human-in-the-Loop
-```
-
 When enabled, every submission routed to a private Compute Worker is executed in HITL mode.
 ![image1](_attachments/doc_HITL_1.png)
 
@@ -163,9 +157,9 @@ This needs to be done inside of the compute worker host machine (not inside of t
 The Compute Worker immediately resumes execution.
 The following artifacts are published:
 
-* detailed results;
-* scores;
-* output archive;
+* detailed results.
+* scores.
+* output archive.
 * logs.
 
 The submission status becomes "Finished"
@@ -186,27 +180,7 @@ No scoring results are published.
 
 If no decision is received within 24 hours, the submission automatically fails.
 The Compute Worker reports a timeout error.
-
----
-
-## Publication policy
-
-The fundamental principle of HITL is:
-
-> **Nothing leaves the Compute Worker before validation.**
-
-When HITL is enabled:
-
-| Artifact                                | Before validation | After approval |
-| --------------------------------------- | ----------------- | -------------- |
-| scores.json                             | ❌                 | ✅              |
-| leaderboard score                       | ❌                 | ✅              |
-| detailed_results.html                   | ❌                 | ✅              |
-| detailed results websocket notification | ❌                 | ✅              |
-| output archive                          | ❌                 | ✅              |
-| execution logs                          | Local only        | ✅              |
-
-The Compute Worker keeps every generated artifact locally until approval.
+The compute worker is enable to run another submission while waiting for HITL approval.
 
 ---
 
@@ -234,14 +208,6 @@ flowchart TD
 
 ---
 
-## Output behaviour
-
-Prediction files, scoring outputs and logs are generated normally.
-However, publication is postponed until validation succeeds.
-This guarantees that no evaluation result becomes visible before manual approval.
-
----
-
 ## Error handling
 
 The following situations are handled explicitly.
@@ -254,27 +220,3 @@ The following situations are handled explicitly.
 | Operator rejects submission                          | Submission fails                |
 | Validation timeout                                   | Submission fails                |
 | Approval received                                    | Results are published normally  |
-
----
-
-## Design principles
-
-The HITL implementation follows four principles:
-
-1. **Backward compatibility**
-
-   * Existing deployments require no configuration changes.
-
-2. **Safety**
-
-   * Configuration mismatches are detected before execution.
-
-3. **Isolation**
-
-   * HITL is supported only on private Compute Workers.
-
-4. **Atomic publication**
-
-   * No artifact is published before manual approval.
-
-These principles ensure that sensitive competitions can safely introduce manual validation without impacting the standard Codabench execution workflow.

From 8512c44b9a980a91b435bffaaa8de94a8ed77f62 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 21 Jul 2026 13:50:45 +0200
Subject: [PATCH 09/17] HITL in cw documentation

---
 .../Running_a_benchmark/Compute-Worker-Management---Setup.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
index 5735af914..fc0576c1c 100644
--- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
+++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
@@ -59,7 +59,7 @@ HOST_DIRECTORY=/codabench
 CONTAINER_ENGINE_EXECUTABLE=docker
 #USE_GPU=True
 #GPU_DEVICE=nvidia.com/gpu=all
-
+#HUMAN_IN_THE_LOOP=False
 #######################################################################
 #                       Network                                       #
 #######################################################################
@@ -72,6 +72,8 @@ By default, the competition container created by the compute worker has access t
 
 If the VM hosting the compute worker is behind a proxy, and you want to allow the competition container to access internet, you will also need to set the proxy for the competition container to use, in which case you can use `COMPETITION_CONTAINER_HTTP_PROXY`
 
+To control the scoring output before sending it to the instance, you can use the human in the loop feature the check the scoring file (json or txt) before sending anything to the instance, see: [HITL documentation](https://docs.codabench.org/latest/Organizers/Running_a_benchmark/Competition-HITL/)
+
 !!! note
     - The broker URL is a unique identifier of the job queue that the worker should listen to. To create a queue or obtain the broker URL of an existing queue, you can refer to [Queue Management](Queue-Management.md) docs page.
 
@@ -204,6 +206,7 @@ The folder `$HOST_DIRECTORY/data`, usually `/codabench/data`, is shared between
 
 
 !!! tip "If you simply wish to set up some compute workers to increase the computing power of your benchmark, you don't need to scroll this page any further."
+
 ---
 
 ## Building compute worker

From 4be6f88c6b27f26e2a05ddc338086d3e20ccd0f8 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Thu, 23 Jul 2026 11:46:09 +0200
Subject: [PATCH 10/17] site worker modified to return submission status failed
 if hitl enabled on public CW

---
 src/apps/competitions/tasks.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 35bbebd32..3f43d69ea 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -162,6 +162,25 @@ def _send_to_compute_worker(submission, is_scoring):
         and submission.queue is not None
     )
 
+    if (
+        submission.phase.competition.enable_human_in_the_loop
+        and submission.queue is None
+    ):
+        submission.status = Submission.FAILED
+        submission.status_details = (
+            "This competition requires Human-in-the-Loop (HITL), "
+            "but the submission was routed to a public compute worker. "
+            "HITL is only supported on private compute workers."
+        )
+        submission.save(update_fields=["status", "status_details"])
+
+        logger.error(
+            "Submission %s rejected: HITL requires a private compute worker.",
+            submission.id,
+        )
+
+        return
+
     run_args = {
         "user_pk": submission.owner.pk,
         "submissions_api_url": settings.SUBMISSIONS_API_URL,

From ad73d9c7a727f13f23de027503567efe1a1316e7 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Thu, 23 Jul 2026 17:33:38 +0200
Subject: [PATCH 11/17] in progress: detailed results check and validation

---
 compute_worker/compute_worker.py | 73 ++++++++++++--------------------
 src/apps/competitions/tasks.py   |  1 -
 2 files changed, 26 insertions(+), 48 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 84fcf1825..d2f388a35 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -332,7 +332,12 @@ def run_wrapper(run_args):
             if run.human_in_the_loop:
                 run._update_status(SubmissionStatus.AWAITING_VALIDATION)
                 run.wait_for_human_validation()
-                run.send_final_detailed_results()
+                if run.pending_detailed_results:
+                    asyncio.run(
+                        run.send_detailed_results(
+                            run.pending_detailed_results
+                        )
+                    )
                 run.push_scores()
                 run.push_output()
             else:
@@ -515,6 +520,7 @@ def __init__(self, run_args):
         self.reference_data = run_args.get("reference_data")
         self.ingestion_only_during_scoring = run_args.get("ingestion_only_during_scoring")
         self.detailed_results_url = run_args.get("detailed_results_url")
+        self.pending_detailed_results = None
 
         self.ingestion_program_exit_code = None
         self.ingestion_program_elapsed_time = None
@@ -563,7 +569,10 @@ async def watch_detailed_results(self):
                 new_time = os.path.getmtime(file_path)
                 if new_time != last_modified_time:
                     last_modified_time = new_time
-                    await self.send_detailed_results(file_path)
+                    if self.human_in_the_loop:
+                        self.pending_detailed_results = file_path
+                    else:
+                        await self.send_detailed_results(file_path)
             else:
                 logger.info(time.time() - start)
                 if time.time() - start > expiration_seconds:
@@ -576,7 +585,10 @@ async def watch_detailed_results(self):
         else:
             # make sure we always send the final version of the file
             if file_path:
-                await self.send_detailed_results(file_path)
+                if self.human_in_the_loop:
+                    self.pending_detailed_results = file_path
+                else:
+                    await self.send_detailed_results(file_path)
 
     def push_logs(self):
         """Upload any collected logs, even in case of crash.
@@ -638,49 +650,6 @@ async def send_detailed_results(self, file_path):
                 except Exception as e:
                     logger.exception(e)
 
-    def send_final_detailed_results(self):
-        if not self.detailed_results_url:
-            return
-
-        file_path = self.get_detailed_results_file_path()
-
-        if not file_path:
-            logger.info("No detailed_results.html found")
-            return
-
-        logger.info(
-            f"Uploading final detailed results {file_path} - {self.detailed_results_url}"
-        )
-
-        self._put_file(
-            self.detailed_results_url,
-            file=file_path,
-            content_type="text/html",
-        )
-
-        websocket_url = f"{self.websocket_url}?kind=detailed_results"
-
-        try:
-            websocket = asyncio.run(
-                asyncio.wait_for(
-                    websockets.connect(websocket_url),
-                    timeout=30.0,
-                )
-            )
-
-            asyncio.run(
-                websocket.send(
-                    json.dumps(
-                        {
-                            "kind": "detailed_result_update",
-                        }
-                    )
-                )
-            )
-
-        except Exception as e:
-            logger.exception(e)
-
     def _get_stdout_stderr_file_names(self, run_args):
         # run_args should be the run_args argument passed to __init__ from the run_wrapper.
         if not self.is_scoring:
@@ -1569,7 +1538,17 @@ def wait_for_human_validation(self):
         logger.info("=" * 60)
         logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
         logger.info("Inspect the scores file:")
-        logger.info(f"  cat {scores_path}")
+        logger.info(f"cat {scores_path}")
+
+        if self.detailed_results_url:
+            detailed_results = self.get_detailed_results_file_path()
+
+            if detailed_results and os.path.exists(detailed_results):
+                logger.info("")
+                logger.info("Detailed results")
+                logger.info(f"  {detailed_results}")
+                logger.info("  (Open this file in your browser to review the HTML report)")
+
         logger.info(f"To approve : touch {approved_host}")
         logger.info(f"To reject  : touch {rejected_host}")
         logger.info("=" * 60)
diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 3f43d69ea..220e93677 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -178,7 +178,6 @@ def _send_to_compute_worker(submission, is_scoring):
             "Submission %s rejected: HITL requires a private compute worker.",
             submission.id,
         )
-
         return
 
     run_args = {

From 3b361306e72324749805e451e8e5df50640e5bb2 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Fri, 24 Jul 2026 16:45:01 +0200
Subject: [PATCH 12/17] feature OK / detailed result added to validation
 process

---
 compute_worker/compute_worker.py | 92 ++++++++++++++++++++++++--------
 1 file changed, 71 insertions(+), 21 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index d2f388a35..e1eb54c29 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -3,6 +3,10 @@
 import hashlib
 import json
 import os
+import functools
+import http.server
+import socketserver
+import threading
 import traceback
 import shutil
 import signal
@@ -521,6 +525,8 @@ def __init__(self, run_args):
         self.ingestion_only_during_scoring = run_args.get("ingestion_only_during_scoring")
         self.detailed_results_url = run_args.get("detailed_results_url")
         self.pending_detailed_results = None
+        self.hitl_http_server = None
+        self.hitl_http_thread = None
 
         self.ingestion_program_exit_code = None
         self.ingestion_program_elapsed_time = None
@@ -548,12 +554,6 @@ async def watch_detailed_results(self):
         """Watches files alongside scoring + program containers, currently only used
         for detailed_results.html"""
 
-        if self.human_in_the_loop:
-            logger.info(
-                "HITL enabled: skipping detailed_results streaming"
-            )
-            return
-
         if not self.detailed_results_url:
             return
         file_path = self.get_detailed_results_file_path()
@@ -618,6 +618,40 @@ def get_detailed_results_file_path(self):
             if html_files:
                 return html_files[0]
 
+    def start_hitl_http_server(self):
+        if not self.pending_detailed_results:
+            return
+        root = os.path.dirname(self.pending_detailed_results)
+        logger.info(
+            "Starting temporary HTTP server for HITL review (%s)",
+            root,
+        )
+        handler = functools.partial(
+            http.server.SimpleHTTPRequestHandler,
+            directory=root,
+        )
+        self.hitl_http_server = socketserver.TCPServer(
+            ("127.0.0.1", 8765),
+            handler,
+        )
+        self.hitl_http_thread = threading.Thread(
+            target=self.hitl_http_server.serve_forever,
+            daemon=True,
+        )
+        self.hitl_http_thread.start()
+
+    def stop_hitl_http_server(self):
+        if self.hitl_http_server is None:
+            return
+
+        logger.info("Stopping HITL HTTP server")
+
+        self.hitl_http_server.shutdown()
+        self.hitl_http_server.server_close()
+
+        self.hitl_http_server = None
+        self.hitl_http_thread = None
+
     async def send_detailed_results(self, file_path):
         logger.info(
             f"Updating detailed results {file_path} - {self.detailed_results_url}"
@@ -1535,44 +1569,59 @@ def wait_for_human_validation(self):
         approved_host = os.path.join(host_output_dir, "hitl_approved")
         rejected_host = os.path.join(host_output_dir, "hitl_rejected")
 
+        detailed_results = None
+        if self.detailed_results_url:
+            detailed_results = self.get_detailed_results_file_path()
+            if detailed_results:
+                self.pending_detailed_results = detailed_results
+                self.start_hitl_http_server()
+
         logger.info("=" * 60)
         logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
         logger.info("Inspect the scores file:")
         logger.info(f"cat {scores_path}")
 
-        if self.detailed_results_url:
-            detailed_results = self.get_detailed_results_file_path()
-
-            if detailed_results and os.path.exists(detailed_results):
-                logger.info("")
-                logger.info("Detailed results")
-                logger.info(f"  {detailed_results}")
-                logger.info("  (Open this file in your browser to review the HTML report)")
+        if detailed_results and os.path.exists(detailed_results):
+            logger.info("")
+            logger.info("Inspect the detailed results:")
+            logger.info("")
+            logger.info("Create an SSH tunnel from your workstation:")
+            logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
+            logger.info("")
+            logger.info("Then open in your browser:")
+            logger.info(
+                "http://127.0.0.1:8765/%s",
+                os.path.basename(detailed_results),
+            )
 
+        logger.info("")
         logger.info(f"To approve : touch {approved_host}")
         logger.info(f"To reject  : touch {rejected_host}")
         logger.info("=" * 60)
+        logger.info("Waiting for human validation...")
 
         poll_interval = 3
         max_wait = 60 * 60 * 24
-
         elapsed = 0
 
-        logger.info(
-            "Waiting for human validation..."
-        )
         while elapsed < max_wait:
             if os.path.exists(approved_container):
-                logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
+                logger.info(
+                    f"HITL: submission {self.submission_id} approved, sending results."
+                )
+                self.stop_hitl_http_server()
                 return
+
             if os.path.exists(rejected_container):
+                self.stop_hitl_http_server()
                 raise SubmissionException(
-                    f"HITL: scores rejected by the compute node operator "
-                    f"(submission {self.submission_id})"
+                    f"HITL: submission {self.submission_id} rejected by the compute node operator."
                 )
+
             time.sleep(poll_interval)
             elapsed += poll_interval
 
+        self.stop_hitl_http_server()
         raise SubmissionException(
             f"HITL: 24h timeout reached without validation "
             f"(submission {self.submission_id})"
@@ -1650,6 +1699,7 @@ def push_output(self):
             self._put_dir(self.scoring_result, self.output_dir)
 
     def clean_up(self):
+        self.stop_hitl_http_server()
         if Settings.CODALAB_IGNORE_CLEANUP_STEP:
             logger.warning(
                 f"CODALAB_IGNORE_CLEANUP_STEP mode enabled, ignoring clean up of: {self.root_dir}"

From 0c4eeb94b899aee0b2a10dfc9fb3a35e7fe171ee Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 27 Jul 2026 14:43:00 +0200
Subject: [PATCH 13/17] hitl added to competiton admin

---
 src/apps/competitions/admin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py
index 69bc44229..f9a5c0e1e 100644
--- a/src/apps/competitions/admin.py
+++ b/src/apps/competitions/admin.py
@@ -212,6 +212,7 @@ class CompetitionExpansion(admin.ModelAdmin):
                     "submissions_count",
                     "participants_count",
                     "created_when",
+                    "enable_human_in_the_loop"
                 ]
             },
         ),

From 174d05cb237605f020630b4a62f8e2bda970aa1b Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 27 Jul 2026 14:45:06 +0200
Subject: [PATCH 14/17] admin: moving htil in checkbox section

---
 src/apps/competitions/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py
index f9a5c0e1e..6f2d32bde 100644
--- a/src/apps/competitions/admin.py
+++ b/src/apps/competitions/admin.py
@@ -212,7 +212,6 @@ class CompetitionExpansion(admin.ModelAdmin):
                     "submissions_count",
                     "participants_count",
                     "created_when",
-                    "enable_human_in_the_loop"
                 ]
             },
         ),
@@ -234,6 +233,7 @@ class CompetitionExpansion(admin.ModelAdmin):
                     "can_participants_make_submissions_public",
                     "is_featured",
                     "forum_enabled",
+                    "enable_human_in_the_loop"
                 ]
             },
         ),

From e20361bf25de2a7a0704890012816df9e1ec87f5 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 27 Jul 2026 15:41:31 +0200
Subject: [PATCH 15/17] options added for detailed result check and predictions
 not sent if sub rejected

---
 compute_worker/compute_worker.py | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index e1eb54c29..22c637881 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -335,13 +335,18 @@ def run_wrapper(run_args):
         if run.is_scoring:
             if run.human_in_the_loop:
                 run._update_status(SubmissionStatus.AWAITING_VALIDATION)
-                run.wait_for_human_validation()
+                if not run.wait_for_human_validation():
+                    raise SubmissionException(
+                        f"HITL: submission {run.submission_id} rejected by the compute node operator."
+                    )
+
                 if run.pending_detailed_results:
                     asyncio.run(
                         run.send_detailed_results(
                             run.pending_detailed_results
                         )
                     )
+
                 run.push_scores()
                 run.push_output()
             else:
@@ -1582,17 +1587,26 @@ def wait_for_human_validation(self):
         logger.info(f"cat {scores_path}")
 
         if detailed_results and os.path.exists(detailed_results):
-            logger.info("")
             logger.info("Inspect the detailed results:")
-            logger.info("")
+            logger.info("HTML report location:")
+            logger.info(f"{detailed_results}")
+            logger.info("Option 1 (recommended): Preview without copying the file")
             logger.info("Create an SSH tunnel from your workstation:")
             logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
-            logger.info("")
             logger.info("Then open in your browser:")
             logger.info(
                 "http://127.0.0.1:8765/%s",
                 os.path.basename(detailed_results),
             )
+            host_detailed_results = self._get_host_path(detailed_results)
+            logger.info("HTML report location:")
+            logger.info("%s", host_detailed_results)
+            logger.info("")
+            logger.info("Option 2: Copy the HTML report")
+            logger.info(
+                "cat %s",
+                host_detailed_results,
+            )
 
         logger.info("")
         logger.info(f"To approve : touch {approved_host}")
@@ -1610,20 +1624,18 @@ def wait_for_human_validation(self):
                     f"HITL: submission {self.submission_id} approved, sending results."
                 )
                 self.stop_hitl_http_server()
-                return
+                return True
 
             if os.path.exists(rejected_container):
                 self.stop_hitl_http_server()
-                raise SubmissionException(
-                    f"HITL: submission {self.submission_id} rejected by the compute node operator."
-                )
+                return False
 
             time.sleep(poll_interval)
             elapsed += poll_interval
 
         self.stop_hitl_http_server()
         raise SubmissionException(
-            f"HITL: 24h timeout reached without validation "
+            f"HITL: 24h timeout reached without validation"
             f"(submission {self.submission_id})"
         )
 

From dc3a1744e4a57d525978d18f7a6c7b9ef583fe99 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 28 Jul 2026 10:11:33 +0200
Subject: [PATCH 16/17] cw logs conf modified to display colors (private cw
 compatible)

---
 compute_worker/compute_worker.py | 114 +++++++++++++++++++++++--------
 1 file changed, 85 insertions(+), 29 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 22c637881..fbff4ace7 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -34,11 +34,41 @@
 from billiard.exceptions import SoftTimeLimitExceeded
 
 from logs_loguru import configure_logging, colorize_run_args
-
 logger = logging.getLogger(__name__)
 
 sys.path.append("/app/src/settings/")
 
+# Colors for logs (method below)
+RESET = "\033[0m"
+BOLD = "\033[1m"
+
+RED = "\033[31m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+CYAN = "\033[36m"
+
+
+def colorize(text, color="", bold=False):
+    """
+    Colorize any log message using ANSI escape sequences.
+    Compatible with logging, Loguru, Docker and SSH terminals, public and private CWs.
+    """
+    prefix = ""
+
+    if bold:
+        prefix += BOLD
+
+    if color == "red":
+        prefix += RED
+    elif color == "green":
+        prefix += GREEN
+    elif color == "yellow":
+        prefix += YELLOW
+    elif color == "cyan":
+        prefix += CYAN
+
+    return f"{prefix}{text}{RESET}"
+
 
 # -----------------------------------------------
 # Settings
@@ -1581,38 +1611,59 @@ def wait_for_human_validation(self):
                 self.pending_detailed_results = detailed_results
                 self.start_hitl_http_server()
 
-        logger.info("=" * 60)
-        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
-        logger.info("Inspect the scores file:")
-        logger.info(f"cat {scores_path}")
+        logger.info(colorize("=" * 60, "cyan", bold=True))
+        logger.info(colorize(f"HUMAN IN THE LOOP — submission {self.submission_id}", "cyan", bold=True))
+
+        logger.info(colorize("Inspect the scores file:", bold=True))
+        logger.info(colorize(f"cat {scores_path}", "green"))
 
         if detailed_results and os.path.exists(detailed_results):
-            logger.info("Inspect the detailed results:")
-            logger.info("HTML report location:")
-            logger.info(f"{detailed_results}")
-            logger.info("Option 1 (recommended): Preview without copying the file")
+            host_detailed_results = self._get_host_path(detailed_results)
+
+            logger.info("")
+            logger.info(
+                colorize(
+                    "Option 1 (recommended): Preview without copying the file",
+                    "yellow",
+                )
+            )
+
             logger.info("Create an SSH tunnel from your workstation:")
-            logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
+            logger.info(
+                colorize(
+                    "ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>",
+                    "green",
+                )
+            )
+
             logger.info("Then open in your browser:")
             logger.info(
-                "http://127.0.0.1:8765/%s",
-                os.path.basename(detailed_results),
+                colorize(
+                    f"http://127.0.0.1:8765/{os.path.basename(detailed_results)}",
+                    "green",
+                )
             )
-            host_detailed_results = self._get_host_path(detailed_results)
+
+            logger.info("")
             logger.info("HTML report location:")
-            logger.info("%s", host_detailed_results)
+            logger.info(colorize(host_detailed_results, "green"))
+
             logger.info("")
-            logger.info("Option 2: Copy the HTML report")
             logger.info(
-                "cat %s",
-                host_detailed_results,
+                colorize(
+                    "Option 2: Display the HTML directly in the terminal",
+                    "yellow",
+                )
             )
+            logger.info(colorize(f"cat {host_detailed_results}", "green"))
 
         logger.info("")
-        logger.info(f"To approve : touch {approved_host}")
-        logger.info(f"To reject  : touch {rejected_host}")
-        logger.info("=" * 60)
-        logger.info("Waiting for human validation...")
+        logger.info(colorize("Validation commands:", bold=True))
+        logger.info(colorize(f"Approve : touch {approved_host}", "green"))
+        logger.info(colorize(f"Reject  : touch {rejected_host}", "red"))
+
+        logger.info(colorize("=" * 60, "cyan", bold=True))
+        logger.info(colorize("Waiting for human validation...", "yellow"))
 
         poll_interval = 3
         max_wait = 60 * 60 * 24
@@ -1621,21 +1672,26 @@ def wait_for_human_validation(self):
         while elapsed < max_wait:
             if os.path.exists(approved_container):
                 logger.info(
-                    f"HITL: submission {self.submission_id} approved, sending results."
+                    colorize(
+                        f"HITL: submission {self.submission_id} approved, sending results.",
+                        "green",
+                        bold=True,
+                    )
                 )
-                self.stop_hitl_http_server()
-                return True
-
-            if os.path.exists(rejected_container):
-                self.stop_hitl_http_server()
-                return False
 
+                logger.info(
+                    colorize(
+                        f"HITL: submission {self.submission_id} rejected by the compute node operator.",
+                        "red",
+                        bold=True,
+                    )
+                )
             time.sleep(poll_interval)
             elapsed += poll_interval
 
         self.stop_hitl_http_server()
         raise SubmissionException(
-            f"HITL: 24h timeout reached without validation"
+            f"HITL: 24h timeout reached without validation "
             f"(submission {self.submission_id})"
         )
 

From 56fdf1faf9f8248bb846e320c3aac9943cedea43 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 28 Jul 2026 11:12:31 +0200
Subject: [PATCH 17/17] cw code for hitl clean

---
 compute_worker/compute_worker.py | 116 ++++++++-----------------------
 1 file changed, 28 insertions(+), 88 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index fbff4ace7..b3a1590dc 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -34,41 +34,11 @@
 from billiard.exceptions import SoftTimeLimitExceeded
 
 from logs_loguru import configure_logging, colorize_run_args
+
 logger = logging.getLogger(__name__)
 
 sys.path.append("/app/src/settings/")
 
-# Colors for logs (method below)
-RESET = "\033[0m"
-BOLD = "\033[1m"
-
-RED = "\033[31m"
-GREEN = "\033[32m"
-YELLOW = "\033[33m"
-CYAN = "\033[36m"
-
-
-def colorize(text, color="", bold=False):
-    """
-    Colorize any log message using ANSI escape sequences.
-    Compatible with logging, Loguru, Docker and SSH terminals, public and private CWs.
-    """
-    prefix = ""
-
-    if bold:
-        prefix += BOLD
-
-    if color == "red":
-        prefix += RED
-    elif color == "green":
-        prefix += GREEN
-    elif color == "yellow":
-        prefix += YELLOW
-    elif color == "cyan":
-        prefix += CYAN
-
-    return f"{prefix}{text}{RESET}"
-
 
 # -----------------------------------------------
 # Settings
@@ -1611,59 +1581,34 @@ def wait_for_human_validation(self):
                 self.pending_detailed_results = detailed_results
                 self.start_hitl_http_server()
 
-        logger.info(colorize("=" * 60, "cyan", bold=True))
-        logger.info(colorize(f"HUMAN IN THE LOOP — submission {self.submission_id}", "cyan", bold=True))
-
-        logger.info(colorize("Inspect the scores file:", bold=True))
-        logger.info(colorize(f"cat {scores_path}", "green"))
+        logger.info("=" * 60)
+        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
+        logger.info("Inspect scoring:")
+        logger.info(f"cat {scores_path}")
+        logger.info("")
 
         if detailed_results and os.path.exists(detailed_results):
-            host_detailed_results = self._get_host_path(detailed_results)
-
-            logger.info("")
-            logger.info(
-                colorize(
-                    "Option 1 (recommended): Preview without copying the file",
-                    "yellow",
-                )
-            )
-
+            logger.info("Inspect detailed results:")
+            logger.info("Option 1: Preview without copying the file")
             logger.info("Create an SSH tunnel from your workstation:")
+            logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
+            logger.info("Then open in your browser:",)
             logger.info(
-                colorize(
-                    "ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>",
-                    "green",
-                )
-            )
-
-            logger.info("Then open in your browser:")
-            logger.info(
-                colorize(
-                    f"http://127.0.0.1:8765/{os.path.basename(detailed_results)}",
-                    "green",
-                )
+                "http://127.0.0.1:8765/%s",
+                os.path.basename(detailed_results),
             )
-
-            logger.info("")
-            logger.info("HTML report location:")
-            logger.info(colorize(host_detailed_results, "green"))
-
-            logger.info("")
+            host_detailed_results = self._get_host_path(detailed_results)
+            logger.info("Option 2: Copy the HTML report")
             logger.info(
-                colorize(
-                    "Option 2: Display the HTML directly in the terminal",
-                    "yellow",
-                )
+                "cat %s",
+                host_detailed_results,
             )
-            logger.info(colorize(f"cat {host_detailed_results}", "green"))
 
         logger.info("")
-        logger.info(colorize("Validation commands:", bold=True))
-        logger.info(colorize(f"Approve : touch {approved_host}", "green"))
-        logger.info(colorize(f"Reject  : touch {rejected_host}", "red"))
-
-        logger.info(colorize("=" * 60, "cyan", bold=True))
-        logger.info(colorize("Waiting for human validation...", "yellow"))
+        logger.info(f"To approve : touch {approved_host}")
+        logger.info(f"To reject  : touch {rejected_host}")
+        logger.info("=" * 60)
+        logger.info("Waiting for human validation...")
 
         poll_interval = 3
         max_wait = 60 * 60 * 24
@@ -1672,26 +1617,21 @@ def wait_for_human_validation(self):
         while elapsed < max_wait:
             if os.path.exists(approved_container):
                 logger.info(
-                    colorize(
-                        f"HITL: submission {self.submission_id} approved, sending results.",
-                        "green",
-                        bold=True,
-                    )
+                    f"HITL: submission {self.submission_id} approved, sending results."
                 )
+                self.stop_hitl_http_server()
+                return True
+
+            if os.path.exists(rejected_container):
+                self.stop_hitl_http_server()
+                return False
 
-                logger.info(
-                    colorize(
-                        f"HITL: submission {self.submission_id} rejected by the compute node operator.",
-                        "red",
-                        bold=True,
-                    )
-                )
             time.sleep(poll_interval)
             elapsed += poll_interval
 
         self.stop_hitl_http_server()
         raise SubmissionException(
-            f"HITL: 24h timeout reached without validation "
+            f"HITL: 24h timeout reached without validation"
             f"(submission {self.submission_id})"
         )