diff --git a/src/kernelbot/api/main.py b/src/kernelbot/api/main.py index 0afb819c..7644faea 100644 --- a/src/kernelbot/api/main.py +++ b/src/kernelbot/api/main.py @@ -1079,8 +1079,16 @@ async def get_user_submission( status_code=403, detail="Not authorized to view this submission" ) - # RunItem is a TypedDict (already a dict), select fields to expose + # RunItem is a TypedDict (already a dict), select fields to expose. + # Detailed results are limited to public test/benchmark runs because + # private results can contain secret validation specs and errors. run_fields = ("start_time", "end_time", "mode", "secret", "runner", "score", "passed") + runs = [] + for run in submission["runs"]: + response_run = {key: run[key] for key in run_fields} + if not run["secret"] and run["mode"] in ("test", "benchmark"): + response_run["result"] = run["result"] + runs.append(response_run) runner_queue = await get_submission_runner_queue_status(submission) return { "id": submission["submission_id"], @@ -1091,7 +1099,7 @@ async def get_user_submission( "submission_time": submission["submission_time"], "done": submission["done"], "code": submission["code"], - "runs": [{k: r[k] for k in run_fields} for r in submission["runs"]], + "runs": runs, "job": { "status": submission.get("job_status"), "error": submission.get("job_error"), diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index b8adfba8..485f1ce5 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -117,6 +117,161 @@ def test_get_runner_queue(self, test_client, mock_backend): } +class TestUserSubmissions: + def test_submission_details_require_authentication(self, test_client): + response = test_client.get("/user/submissions/42") + + assert response.status_code == 400 + + def test_submission_details_return_not_found(self, test_client, mock_backend): + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + mock_backend.db.validate_identity = MagicMock( + return_value={"user_id": "123", "user_name": "test-user"} + ) + mock_backend.db.get_submission_by_id = MagicMock(return_value=None) + + response = test_client.get( + "/user/submissions/42", + headers={"X-Popcorn-Cli-Id": "cli-token"}, + ) + + assert response.status_code == 404 + + def test_submission_details_reject_other_owner(self, test_client, mock_backend): + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + mock_backend.db.validate_identity = MagicMock( + return_value={"user_id": "123", "user_name": "test-user"} + ) + mock_backend.db.get_submission_by_id = MagicMock(return_value={"user_id": "456"}) + + response = test_client.get( + "/user/submissions/42", + headers={"X-Popcorn-Cli-Id": "cli-token"}, + ) + + assert response.status_code == 403 + + def test_submission_details_include_stored_run_results(self, test_client, mock_backend): + """Polling clients receive the per-row data stored for completed runs.""" + mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db) + mock_backend.db.__exit__ = MagicMock(return_value=None) + mock_backend.db.validate_identity = MagicMock( + return_value={"user_id": "123", "user_name": "test-user"} + ) + benchmark_result = { + "benchmark-count": "2", + "benchmark.0.status": "pass", + "benchmark.0.spec": "shape-0", + "benchmark.0.mean": "1.5", + "benchmark.1.status": "pass", + "benchmark.1.spec": "shape-1", + "benchmark.1.mean": "2.5", + } + test_result = { + "test-count": "2", + "test.0.status": "pass", + "test.0.spec": "case-0", + "test.0.message": "correct", + "test.1.status": "fail", + "test.1.spec": "case-1", + "test.1.error": "mismatch", + } + mock_backend.db.get_submission_by_id = MagicMock( + return_value={ + "submission_id": 42, + "leaderboard_id": 7, + "leaderboard_name": "test-leaderboard", + "file_name": "submission.py", + "user_id": "123", + "submission_time": "2026-07-06T12:00:00Z", + "done": True, + "code": "print('ok')", + "runs": [ + { + "start_time": "2026-07-06T12:00:00Z", + "end_time": "2026-07-06T12:01:00Z", + "mode": "test", + "secret": False, + "runner": "B200", + "score": None, + "passed": False, + "result": test_result, + }, + { + "start_time": "2026-07-06T12:00:00Z", + "end_time": "2026-07-06T12:01:00Z", + "mode": "benchmark", + "secret": False, + "runner": "B200", + "score": None, + "passed": True, + "result": benchmark_result, + }, + { + "start_time": "2026-07-06T12:00:00Z", + "end_time": "2026-07-06T12:01:00Z", + "mode": "benchmark", + "secret": True, + "runner": "B200", + "score": None, + "passed": True, + "result": {"secret-result": "must not be exposed"}, + }, + { + "start_time": "2026-07-06T12:00:00Z", + "end_time": "2026-07-06T12:01:00Z", + "mode": "leaderboard", + "secret": False, + "runner": "B200", + "score": 1.5, + "passed": True, + "result": {"leaderboard-result": "not part of this contract"}, + }, + { + "start_time": "2026-07-06T12:00:00Z", + "end_time": "2026-07-06T12:01:00Z", + "mode": "profile0", + "secret": False, + "runner": "B200", + "score": None, + "passed": True, + "result": {"profile-report": "PROFILE_SENTINEL"}, + }, + ], + "job_status": "succeeded", + "job_error": None, + "job_last_heartbeat": "2026-07-06T12:01:00Z", + } + ) + mock_backend.get_runner_queue_status = AsyncMock( + return_value=RunnerQueueStatus( + runner="Modal", + gpu="B200", + queued_jobs=0, + available_runners=1, + ) + ) + + response = test_client.get( + "/user/submissions/42", + headers={"X-Popcorn-Cli-Id": "cli-token"}, + ) + + assert response.status_code == 200 + runs = response.json()["runs"] + assert runs[0]["result"] == test_result + assert runs[1]["result"] == benchmark_result + assert "result" not in runs[2] + assert runs[2]["secret"] is True + assert runs[2]["passed"] is True + assert "result" not in runs[3] + assert "result" not in runs[4] + assert "must not be exposed" not in response.text + assert "PROFILE_SENTINEL" not in response.text + + class TestAdminStats: """Test admin stats endpoint.""" diff --git a/tests/test_background_submission_manager.py b/tests/test_background_submission_manager.py index 3ff2a05b..c0fd6374 100644 --- a/tests/test_background_submission_manager.py +++ b/tests/test_background_submission_manager.py @@ -111,6 +111,63 @@ async def fake_submit_full(req, mode, reporter, sub_id, skip_precheck=False): await manager.stop() +@pytest.mark.asyncio +async def test_accepted_job_survives_request_task_cancellation(mock_backend): + """Disconnecting an accepted request must not cancel its background job.""" + db_context = mock_backend.db + db_context.upsert_submission_job_status = mock.Mock(side_effect=lambda *args, **kwargs: args[0]) + db_context.update_heartbeat_if_active = mock.Mock() + started = asyncio.Event() + release = asyncio.Event() + backend_cancelled = False + + async def fake_submit_full(req, mode, reporter, sub_id, skip_precheck=False): + nonlocal backend_cancelled + started.set() + try: + await release.wait() + except asyncio.CancelledError: + backend_cancelled = True + raise + return None, None + + mock_backend.submit_full = fake_submit_full + manager = BackgroundSubmissionManager( + mock_backend, min_workers=1, max_workers=1, idle_seconds=0.1 + ) + await manager.start() + request_accepted = asyncio.Event() + + async def simulated_request(): + await manager.enqueue(get_req(1), SubmissionMode.BENCHMARK, sub_id=42) + request_accepted.set() + await asyncio.Event().wait() + + request_task = asyncio.create_task(simulated_request()) + try: + await asyncio.wait_for(request_accepted.wait(), timeout=1) + await asyncio.wait_for(started.wait(), timeout=1) + + request_task.cancel() + with pytest.raises(asyncio.CancelledError): + await request_task + + assert not backend_cancelled + release.set() + await asyncio.wait_for(manager.queue.join(), timeout=1) + + assert not backend_cancelled + assert ( + mock.call(42, status="succeeded", last_heartbeat=mock.ANY) + in db_context.upsert_submission_job_status.call_args_list + ) + finally: + release.set() + if not request_task.done(): + request_task.cancel() + await manager.stop() + + @pytest.mark.asyncio async def test_leaderboard_secret_failure_marks_job_failed(mock_backend): db_context = mock_backend.db