From d6b035d03ae2d1b22173bd6a3b3a21d22e6a483d Mon Sep 17 00:00:00 2001 From: kaihere14 Date: Tue, 17 Nov 2026 01:38:23 +0530 Subject: [PATCH 1/2] [FIX] Handle HttpError from GCP VM creation in start_test The Compute API raises HttpError when it rejects an instance insert (bad config, missing image, quota, rate limit, server error) instead of returning an error inside the operation. start_test() only checked the returned operation, so the exception escaped to gcp_instance(), which logs it and leaves the test pending. A permanent error was retried on every cron run and the test never failed or reported back to GitHub. Catch HttpError around create_instance() and convert it to the same error shape a failed operation has, so the existing retry rules and messages apply. 429 and 5xx responses, quota and rate limit errors are retried; anything else marks the test as failed. --- mod_ci/controllers.py | 67 ++++++++- tests/test_ci/test_controllers.py | 234 ++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+), 1 deletion(-) diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index 666f4598..bc9f885d 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -25,6 +25,7 @@ from github import (Auth, Commit, Github, GithubException, GithubObject, Repository) from google.oauth2 import service_account +from googleapiclient.errors import HttpError from lxml import etree from markdown2 import markdown from pymysql.err import IntegrityError @@ -154,6 +155,8 @@ def safe_db_commit(db, operation_description: str = "database operation") -> boo 'RESOURCE_NOT_FOUND': "Required GCP resource not found. Please contact the administrator.", 'RESOURCE_ALREADY_EXISTS': "A VM with this name already exists. Please contact the administrator.", 'TIMEOUT': "GCP operation timed out. The test will be retried automatically.", + 'RATE_LIMIT_EXCEEDED': "GCP API rate limit reached. The test will be retried automatically.", + 'BACKEND_ERROR': "GCP returned a temporary server error. The test will be retried automatically.", } # GCP error codes that are transient and should be retried automatically. @@ -162,8 +165,62 @@ def safe_db_commit(db, operation_description: str = "database operation") -> boo 'ZONE_RESOURCE_POOL_EXHAUSTED', 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS', 'QUOTA_EXCEEDED', + 'RATE_LIMIT_EXCEEDED', + 'BACKEND_ERROR', } +# When the Compute API rejects a request outright, the client library raises HttpError instead +# of returning an operation. Its body names the failure with a camelCase "reason" rather than +# the upper-case "code" an operation carries. These are the reasons that have an equivalent +# code above; any other reason is kept as it is. +GCP_HTTP_ERROR_REASONS = { + 'quotaExceeded': 'QUOTA_EXCEEDED', + 'rateLimitExceeded': 'RATE_LIMIT_EXCEEDED', + 'userRateLimitExceeded': 'RATE_LIMIT_EXCEEDED', + 'backendError': 'BACKEND_ERROR', + 'internalError': 'BACKEND_ERROR', + 'notFound': 'RESOURCE_NOT_FOUND', + 'alreadyExists': 'RESOURCE_ALREADY_EXISTS', +} + +# HTTP statuses that mean "try again later", whatever reason the body gives. +GCP_RETRYABLE_HTTP_STATUSES = {429, 500, 502, 503, 504} + + +def gcp_error_from_http_error(error: HttpError) -> Dict: + """ + Convert an HttpError raised by a Compute API call into a GCP error response. + + The result has the same shape as a failed operation, so parse_gcp_error() and + is_retryable_gcp_error() treat both kinds of failure the same way. + + :param error: The error raised by the googleapiclient request + :return: A dictionary of the form {'error': {'errors': [{'code': ..., 'message': ...}]}} + """ + status = getattr(error.resp, 'status', None) + reason = None + message = str(error) + try: + body = json.loads(error.content.decode('utf-8')) + except (AttributeError, UnicodeDecodeError, ValueError): + body = None + error_body = body.get('error') if isinstance(body, dict) else None + if isinstance(error_body, dict): + message = error_body.get('message') or message + errors = error_body.get('errors') + if isinstance(errors, list) and errors and isinstance(errors[0], dict): + reason = errors[0].get('reason') + message = errors[0].get('message') or message + + code = GCP_HTTP_ERROR_REASONS.get(reason, reason) if reason else f'HTTP_{status}' + if status in GCP_RETRYABLE_HTTP_STATUSES and code not in GCP_RETRYABLE_ERRORS: + code = 'RATE_LIMIT_EXCEEDED' if status == 429 else 'BACKEND_ERROR' + + return { + 'error': {'errors': [{'code': code, 'message': message}]}, + 'httpErrorStatusCode': status, + } + def get_gcp_error_code(result: Dict) -> Optional[str]: """ @@ -1233,7 +1290,15 @@ def start_test(compute, app, db, repository: Repository.Repository, test, bot_to zone = config.get('ZONE', '') project_id = config.get('PROJECT_NAME', '') - operation = create_instance(compute, project_id, zone, test, full_url) + try: + operation = create_instance(compute, project_id, zone, test, full_url) + except HttpError as e: + # A request the Compute API rejects (bad config, missing image, quota, rate limit, + # server error) raises instead of returning an error in the operation. Left uncaught, + # it reached gcp_instance(), which retries every test on every cron run, so a + # permanent error kept the test pending forever and never reported a failure. + log.error(f"Test {test.id}: Compute API rejected VM creation: {e}") + operation = gcp_error_from_http_error(e) # Check if the create_instance call itself returned an error (synchronous failure) if 'error' in operation: diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index a9a69bab..32215b6a 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -5,7 +5,9 @@ from unittest import mock from unittest.mock import MagicMock +import httplib2 from flask import g +from googleapiclient.errors import HttpError from mod_auth.models import Role from mod_ci.controllers import (Workflow_builds, get_info_for_pr_comment, @@ -3951,6 +3953,105 @@ def test_parse_gcp_error_error_not_a_dict(self): mock_log.error.assert_called_once() +def make_http_error(status, reason=None, message='Request failed', content=None): + """Build the HttpError googleapiclient raises when the Compute API rejects a request.""" + if content is None: + error = {'code': status, 'message': message} + if reason is not None: + error['errors'] = [{'message': message, 'domain': 'global', 'reason': reason}] + content = json.dumps({'error': error}).encode('utf-8') + return HttpError(httplib2.Response({'status': status}), content) + + +class TestGcpErrorFromHttpError(unittest.TestCase): + """Tests for gcp_error_from_http_error, which maps an HttpError onto a GCP error response.""" + + def test_quota_exceeded_is_retryable(self): + """A 403 quotaExceeded maps to QUOTA_EXCEEDED, which is retried.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error) + + result = gcp_error_from_http_error( + make_http_error(403, 'quotaExceeded', "Quota 'CPUS' exceeded. Limit: 24.0 in region us-central1.")) + + self.assertEqual(get_gcp_error_code(result), 'QUOTA_EXCEEDED') + self.assertTrue(is_retryable_gcp_error(result)) + self.assertEqual(result['httpErrorStatusCode'], 403) + self.assertEqual(result['error']['errors'][0]['message'], + "Quota 'CPUS' exceeded. Limit: 24.0 in region us-central1.") + + def test_rate_limit_is_retryable(self): + """A 403 rateLimitExceeded maps to RATE_LIMIT_EXCEEDED, which is retried.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error) + + result = gcp_error_from_http_error(make_http_error(403, 'rateLimitExceeded')) + + self.assertEqual(get_gcp_error_code(result), 'RATE_LIMIT_EXCEEDED') + self.assertTrue(is_retryable_gcp_error(result)) + + def test_not_found_is_permanent(self): + """A 404 notFound (for example a missing image family) is not retried.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error) + + result = gcp_error_from_http_error(make_http_error(404, 'notFound', "The resource was not found")) + + self.assertEqual(get_gcp_error_code(result), 'RESOURCE_NOT_FOUND') + self.assertFalse(is_retryable_gcp_error(result)) + + def test_unknown_reason_is_kept_and_permanent(self): + """A reason with no known equivalent is passed through as the code and is not retried.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error, + parse_gcp_error) + + result = gcp_error_from_http_error(make_http_error(400, 'invalid', "Invalid value for field 'machineType'")) + + self.assertEqual(get_gcp_error_code(result), 'invalid') + self.assertFalse(is_retryable_gcp_error(result)) + message = parse_gcp_error(result, log=MagicMock()) + self.assertEqual(message, "VM creation failed (invalid). Please contact the administrator.") + self.assertNotIn('machineType', message) + + def test_server_error_is_retryable_whatever_the_reason(self): + """A 5xx is retried even when its reason has no known equivalent.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error) + + result = gcp_error_from_http_error(make_http_error(503, 'serviceUnavailable')) + + self.assertEqual(get_gcp_error_code(result), 'BACKEND_ERROR') + self.assertTrue(is_retryable_gcp_error(result)) + + def test_too_many_requests_without_json_body(self): + """A 429 with a body that is not JSON is still recognised as a retryable rate limit.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error) + + result = gcp_error_from_http_error(make_http_error(429, content=b'Too Many Requests')) + + self.assertEqual(get_gcp_error_code(result), 'RATE_LIMIT_EXCEEDED') + self.assertTrue(is_retryable_gcp_error(result)) + + def test_permanent_error_without_json_body(self): + """A 4xx with no JSON body falls back to an HTTP_ code and is not retried.""" + from mod_ci.controllers import (gcp_error_from_http_error, + get_gcp_error_code, + is_retryable_gcp_error) + + result = gcp_error_from_http_error(make_http_error(400, content=b'Bad Request')) + + self.assertEqual(get_gcp_error_code(result), 'HTTP_400') + self.assertFalse(is_retryable_gcp_error(result)) + + class TestDiagnoseMissingArtifact(BaseTestCase): """Test the _diagnose_missing_artifact function and retry behavior.""" @@ -4811,3 +4912,136 @@ def test_start_test_operation_timeout_creates_db_record( start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token") mock_g.db.add.assert_called_once() + + @mock.patch('run.log') + @mock.patch('mod_ci.controllers.mark_test_failed') + @mock.patch('mod_ci.controllers.wait_for_operation') + @mock.patch('mod_ci.controllers.create_instance') + @mock.patch('mod_ci.controllers.find_artifact_for_commit') + @mock.patch('mod_ci.controllers.requests.get') + @mock.patch('mod_ci.controllers.zipfile.ZipFile') + @mock.patch('builtins.open', new_callable=mock.mock_open()) + @mock.patch('mod_ci.controllers.g') + @mock.patch('mod_ci.controllers.TestProgress') + @mock.patch('mod_ci.controllers.GcpInstance') + def test_start_test_http_error_quota_exceeded_retries( + self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, + mock_zipfile, mock_requests_get, mock_find_artifact, + mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log): + """A quota HttpError raised by create_instance leaves the test pending for retry. + + The Compute API raises HttpError for a rejected insert instead of returning an + error in the operation, so start_test must catch it rather than let it escape. + """ + from mod_ci.controllers import start_test + + self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, + mock_find_artifact, mock_requests_get, mock_zipfile) + mock_create_instance.side_effect = make_http_error(403, 'quotaExceeded', "Quota 'CPUS' exceeded.") + + start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token") + + mock_create_instance.assert_called_once() + mock_mark_failed.assert_not_called() + mock_wait_for_operation.assert_not_called() + mock_g.db.add.assert_not_called() + + @mock.patch('run.log') + @mock.patch('mod_ci.controllers.mark_test_failed') + @mock.patch('mod_ci.controllers.wait_for_operation') + @mock.patch('mod_ci.controllers.create_instance') + @mock.patch('mod_ci.controllers.find_artifact_for_commit') + @mock.patch('mod_ci.controllers.requests.get') + @mock.patch('mod_ci.controllers.zipfile.ZipFile') + @mock.patch('builtins.open', new_callable=mock.mock_open()) + @mock.patch('mod_ci.controllers.g') + @mock.patch('mod_ci.controllers.TestProgress') + @mock.patch('mod_ci.controllers.GcpInstance') + def test_start_test_http_error_server_error_retries( + self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, + mock_zipfile, mock_requests_get, mock_find_artifact, + mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log): + """A 503 HttpError raised by create_instance leaves the test pending for retry.""" + from mod_ci.controllers import start_test + + self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, + mock_find_artifact, mock_requests_get, mock_zipfile) + mock_create_instance.side_effect = make_http_error(503, 'backendError', 'Backend Error') + + start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token") + + mock_mark_failed.assert_not_called() + mock_wait_for_operation.assert_not_called() + mock_g.db.add.assert_not_called() + + @mock.patch('run.log') + @mock.patch('mod_ci.controllers.mark_test_failed') + @mock.patch('mod_ci.controllers.wait_for_operation') + @mock.patch('mod_ci.controllers.create_instance') + @mock.patch('mod_ci.controllers.find_artifact_for_commit') + @mock.patch('mod_ci.controllers.requests.get') + @mock.patch('mod_ci.controllers.zipfile.ZipFile') + @mock.patch('builtins.open', new_callable=mock.mock_open()) + @mock.patch('mod_ci.controllers.g') + @mock.patch('mod_ci.controllers.TestProgress') + @mock.patch('mod_ci.controllers.GcpInstance') + def test_start_test_http_error_permanent_marks_failed( + self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, + mock_zipfile, mock_requests_get, mock_find_artifact, + mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log): + """A permanent HttpError raised by create_instance marks the test as failed. + + Before the fix this escaped to gcp_instance(), which logged it and retried the + test on every cron run, so it never left the queue and never reported a failure. + """ + from mod_ci.controllers import start_test + + self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, + mock_find_artifact, mock_requests_get, mock_zipfile) + mock_create_instance.side_effect = make_http_error( + 404, 'notFound', "The resource 'projects/test/global/images/family/ci-linux' was not found") + test = Test.query.first() + + start_test(MagicMock(), self.app, mock_g.db, MagicMock(), test, "token") + + mock_mark_failed.assert_called_once() + self.assertIs(mock_mark_failed.call_args.args[1], test) + self.assertEqual(mock_mark_failed.call_args.args[3], + "Required GCP resource not found. Please contact the administrator.") + mock_wait_for_operation.assert_not_called() + mock_g.db.add.assert_not_called() + + @mock.patch('run.log') + @mock.patch('mod_ci.controllers.mark_test_failed') + @mock.patch('mod_ci.controllers.wait_for_operation') + @mock.patch('mod_ci.controllers.find_artifact_for_commit') + @mock.patch('mod_ci.controllers.requests.get') + @mock.patch('mod_ci.controllers.zipfile.ZipFile') + @mock.patch('builtins.open', new_callable=mock.mock_open()) + @mock.patch('mod_ci.controllers.g') + @mock.patch('mod_ci.controllers.TestProgress') + @mock.patch('mod_ci.controllers.GcpInstance') + def test_start_test_catches_http_error_from_instances_insert( + self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, + mock_zipfile, mock_requests_get, mock_find_artifact, + mock_wait_for_operation, mock_mark_failed, mock_log): + """An HttpError from compute.instances().insert().execute() is handled in start_test. + + create_instance is not mocked here, so the error comes from the real call path. + """ + from mod_ci.controllers import start_test + + self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, + mock_find_artifact, mock_requests_get, mock_zipfile) + compute = MagicMock() + compute.instances.return_value.insert.return_value.execute.side_effect = make_http_error( + 400, 'invalid', "Invalid value for field 'resource.machineType'") + + start_test(compute, self.app, mock_g.db, MagicMock(), Test.query.first(), "token") + + compute.instances.return_value.insert.return_value.execute.assert_called_once() + mock_mark_failed.assert_called_once() + self.assertEqual(mock_mark_failed.call_args.args[3], + "VM creation failed (invalid). Please contact the administrator.") + mock_wait_for_operation.assert_not_called() + mock_g.db.add.assert_not_called() From f146e3f4792e0ff0274a7fbc27ac781ecfcd10f4 Mon Sep 17 00:00:00 2001 From: kaihere14 Date: Tue, 17 Nov 2026 01:46:42 +0530 Subject: [PATCH 2/2] test(ci): table-drive the HttpError tests to remove duplication The four start_test HttpError tests repeated the same decorator stack and setup, which SonarCloud counted as duplicated new code. They now share one helper that patches with an ExitStack and raises from compute.instances().insert().execute(), and the cases run as subtests. The gcp_error_from_http_error unit tests are a single table as well. --- tests/test_ci/test_controllers.py | 275 ++++++++---------------------- 1 file changed, 73 insertions(+), 202 deletions(-) diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index 32215b6a..1062298f 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -1,3 +1,4 @@ +import contextlib import json import os import unittest @@ -10,8 +11,10 @@ from googleapiclient.errors import HttpError from mod_auth.models import Role -from mod_ci.controllers import (Workflow_builds, get_info_for_pr_comment, - is_valid_commit_hash, mark_test_failed, +from mod_ci.controllers import (Workflow_builds, gcp_error_from_http_error, + get_gcp_error_code, get_info_for_pr_comment, + is_retryable_gcp_error, is_valid_commit_hash, + mark_test_failed, parse_gcp_error, parse_git_commit_from_log_file, parse_git_commit_from_log_stream, parse_git_commit_from_logs, @@ -3966,90 +3969,34 @@ def make_http_error(status, reason=None, message='Request failed', content=None) class TestGcpErrorFromHttpError(unittest.TestCase): """Tests for gcp_error_from_http_error, which maps an HttpError onto a GCP error response.""" - def test_quota_exceeded_is_retryable(self): - """A 403 quotaExceeded maps to QUOTA_EXCEEDED, which is retried.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error) - + def test_maps_reason_and_status_to_error_code(self): + """Known reasons map to the platform's codes, and 429 or 5xx is always retryable.""" + cases = [ + # status, reason, body (None builds a Compute-style JSON body), code, retryable + (403, 'quotaExceeded', None, 'QUOTA_EXCEEDED', True), + (403, 'rateLimitExceeded', None, 'RATE_LIMIT_EXCEEDED', True), + (404, 'notFound', None, 'RESOURCE_NOT_FOUND', False), + (409, 'alreadyExists', None, 'RESOURCE_ALREADY_EXISTS', False), + (400, 'invalid', None, 'invalid', False), + (503, 'serviceUnavailable', None, 'BACKEND_ERROR', True), + (429, None, b'Too Many Requests', 'RATE_LIMIT_EXCEEDED', True), + (400, None, b'Bad Request', 'HTTP_400', False), + ] + for status, reason, content, code, retryable in cases: + with self.subTest(status=status, reason=reason): + result = gcp_error_from_http_error(make_http_error(status, reason, content=content)) + self.assertEqual(get_gcp_error_code(result), code) + self.assertEqual(is_retryable_gcp_error(result), retryable) + self.assertEqual(result['httpErrorStatusCode'], status) + + def test_keeps_gcp_message_out_of_user_message(self): + """The GCP message is kept for the log, while the user sees a generic message.""" result = gcp_error_from_http_error( - make_http_error(403, 'quotaExceeded', "Quota 'CPUS' exceeded. Limit: 24.0 in region us-central1.")) - - self.assertEqual(get_gcp_error_code(result), 'QUOTA_EXCEEDED') - self.assertTrue(is_retryable_gcp_error(result)) - self.assertEqual(result['httpErrorStatusCode'], 403) - self.assertEqual(result['error']['errors'][0]['message'], - "Quota 'CPUS' exceeded. Limit: 24.0 in region us-central1.") - - def test_rate_limit_is_retryable(self): - """A 403 rateLimitExceeded maps to RATE_LIMIT_EXCEEDED, which is retried.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error) - - result = gcp_error_from_http_error(make_http_error(403, 'rateLimitExceeded')) - - self.assertEqual(get_gcp_error_code(result), 'RATE_LIMIT_EXCEEDED') - self.assertTrue(is_retryable_gcp_error(result)) - - def test_not_found_is_permanent(self): - """A 404 notFound (for example a missing image family) is not retried.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error) - - result = gcp_error_from_http_error(make_http_error(404, 'notFound', "The resource was not found")) - - self.assertEqual(get_gcp_error_code(result), 'RESOURCE_NOT_FOUND') - self.assertFalse(is_retryable_gcp_error(result)) - - def test_unknown_reason_is_kept_and_permanent(self): - """A reason with no known equivalent is passed through as the code and is not retried.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error, - parse_gcp_error) - - result = gcp_error_from_http_error(make_http_error(400, 'invalid', "Invalid value for field 'machineType'")) - - self.assertEqual(get_gcp_error_code(result), 'invalid') - self.assertFalse(is_retryable_gcp_error(result)) - message = parse_gcp_error(result, log=MagicMock()) - self.assertEqual(message, "VM creation failed (invalid). Please contact the administrator.") - self.assertNotIn('machineType', message) - - def test_server_error_is_retryable_whatever_the_reason(self): - """A 5xx is retried even when its reason has no known equivalent.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error) - - result = gcp_error_from_http_error(make_http_error(503, 'serviceUnavailable')) + make_http_error(400, 'invalid', "Invalid value for field 'machineType'")) - self.assertEqual(get_gcp_error_code(result), 'BACKEND_ERROR') - self.assertTrue(is_retryable_gcp_error(result)) - - def test_too_many_requests_without_json_body(self): - """A 429 with a body that is not JSON is still recognised as a retryable rate limit.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error) - - result = gcp_error_from_http_error(make_http_error(429, content=b'Too Many Requests')) - - self.assertEqual(get_gcp_error_code(result), 'RATE_LIMIT_EXCEEDED') - self.assertTrue(is_retryable_gcp_error(result)) - - def test_permanent_error_without_json_body(self): - """A 4xx with no JSON body falls back to an HTTP_ code and is not retried.""" - from mod_ci.controllers import (gcp_error_from_http_error, - get_gcp_error_code, - is_retryable_gcp_error) - - result = gcp_error_from_http_error(make_http_error(400, content=b'Bad Request')) - - self.assertEqual(get_gcp_error_code(result), 'HTTP_400') - self.assertFalse(is_retryable_gcp_error(result)) + self.assertEqual(result['error']['errors'][0]['message'], "Invalid value for field 'machineType'") + self.assertEqual(parse_gcp_error(result, log=MagicMock()), + "VM creation failed (invalid). Please contact the administrator.") class TestDiagnoseMissingArtifact(BaseTestCase): @@ -4913,135 +4860,59 @@ def test_start_test_operation_timeout_creates_db_record( mock_g.db.add.assert_called_once() - @mock.patch('run.log') - @mock.patch('mod_ci.controllers.mark_test_failed') - @mock.patch('mod_ci.controllers.wait_for_operation') - @mock.patch('mod_ci.controllers.create_instance') - @mock.patch('mod_ci.controllers.find_artifact_for_commit') - @mock.patch('mod_ci.controllers.requests.get') - @mock.patch('mod_ci.controllers.zipfile.ZipFile') - @mock.patch('builtins.open', new_callable=mock.mock_open()) - @mock.patch('mod_ci.controllers.g') - @mock.patch('mod_ci.controllers.TestProgress') - @mock.patch('mod_ci.controllers.GcpInstance') - def test_start_test_http_error_quota_exceeded_retries( - self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, - mock_zipfile, mock_requests_get, mock_find_artifact, - mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log): - """A quota HttpError raised by create_instance leaves the test pending for retry. + def _start_test_when_insert_raises(self, error): + """Run start_test with compute.instances().insert().execute() raising the given error. - The Compute API raises HttpError for a rejected insert instead of returning an - error in the operation, so start_test must catch it rather than let it escape. + Returns the patched mocks, keyed by the name of what they replace, and the test that was started. """ from mod_ci.controllers import start_test - self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, - mock_find_artifact, mock_requests_get, mock_zipfile) - mock_create_instance.side_effect = make_http_error(403, 'quotaExceeded', "Quota 'CPUS' exceeded.") - - start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token") - - mock_create_instance.assert_called_once() - mock_mark_failed.assert_not_called() - mock_wait_for_operation.assert_not_called() - mock_g.db.add.assert_not_called() + targets = ['run.log', 'mod_ci.controllers.mark_test_failed', 'mod_ci.controllers.wait_for_operation', + 'mod_ci.controllers.find_artifact_for_commit', 'mod_ci.controllers.requests.get', + 'mod_ci.controllers.zipfile.ZipFile', 'mod_ci.controllers.g', + 'mod_ci.controllers.TestProgress', 'mod_ci.controllers.GcpInstance'] + with contextlib.ExitStack() as stack: + mocks = {target.rsplit('.', 1)[1]: stack.enter_context(mock.patch(target)) for target in targets} + stack.enter_context(mock.patch('builtins.open', new_callable=mock.mock_open())) + self._setup_start_test_mocks(mocks['g'], mocks['GcpInstance'], mocks['TestProgress'], + mocks['find_artifact_for_commit'], mocks['get'], mocks['ZipFile']) + compute = MagicMock() + compute.instances.return_value.insert.return_value.execute.side_effect = error + test = Test.query.first() + + start_test(compute, self.app, mocks['g'].db, MagicMock(), test, "token") - @mock.patch('run.log') - @mock.patch('mod_ci.controllers.mark_test_failed') - @mock.patch('mod_ci.controllers.wait_for_operation') - @mock.patch('mod_ci.controllers.create_instance') - @mock.patch('mod_ci.controllers.find_artifact_for_commit') - @mock.patch('mod_ci.controllers.requests.get') - @mock.patch('mod_ci.controllers.zipfile.ZipFile') - @mock.patch('builtins.open', new_callable=mock.mock_open()) - @mock.patch('mod_ci.controllers.g') - @mock.patch('mod_ci.controllers.TestProgress') - @mock.patch('mod_ci.controllers.GcpInstance') - def test_start_test_http_error_server_error_retries( - self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, - mock_zipfile, mock_requests_get, mock_find_artifact, - mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log): - """A 503 HttpError raised by create_instance leaves the test pending for retry.""" - from mod_ci.controllers import start_test + compute.instances.return_value.insert.return_value.execute.assert_called_once() + mocks['wait_for_operation'].assert_not_called() + mocks['g'].db.add.assert_not_called() + return mocks, test - self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, - mock_find_artifact, mock_requests_get, mock_zipfile) - mock_create_instance.side_effect = make_http_error(503, 'backendError', 'Backend Error') + def test_start_test_retries_transient_http_error(self): + """Quota, rate limit and server errors raised by the VM insert leave the test pending for retry. - start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token") + The Compute API raises HttpError for a rejected insert instead of returning an + error in the operation, so start_test must catch it rather than let it escape. + """ + for status, reason in [(403, 'quotaExceeded'), (403, 'rateLimitExceeded'), (503, 'backendError')]: + with self.subTest(status=status, reason=reason): + mocks, _ = self._start_test_when_insert_raises(make_http_error(status, reason)) - mock_mark_failed.assert_not_called() - mock_wait_for_operation.assert_not_called() - mock_g.db.add.assert_not_called() + mocks['mark_test_failed'].assert_not_called() - @mock.patch('run.log') - @mock.patch('mod_ci.controllers.mark_test_failed') - @mock.patch('mod_ci.controllers.wait_for_operation') - @mock.patch('mod_ci.controllers.create_instance') - @mock.patch('mod_ci.controllers.find_artifact_for_commit') - @mock.patch('mod_ci.controllers.requests.get') - @mock.patch('mod_ci.controllers.zipfile.ZipFile') - @mock.patch('builtins.open', new_callable=mock.mock_open()) - @mock.patch('mod_ci.controllers.g') - @mock.patch('mod_ci.controllers.TestProgress') - @mock.patch('mod_ci.controllers.GcpInstance') - def test_start_test_http_error_permanent_marks_failed( - self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, - mock_zipfile, mock_requests_get, mock_find_artifact, - mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log): - """A permanent HttpError raised by create_instance marks the test as failed. + def test_start_test_fails_test_on_permanent_http_error(self): + """A permanent error raised by the VM insert marks the test as failed. Before the fix this escaped to gcp_instance(), which logged it and retried the test on every cron run, so it never left the queue and never reported a failure. """ - from mod_ci.controllers import start_test - - self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, - mock_find_artifact, mock_requests_get, mock_zipfile) - mock_create_instance.side_effect = make_http_error( - 404, 'notFound', "The resource 'projects/test/global/images/family/ci-linux' was not found") - test = Test.query.first() - - start_test(MagicMock(), self.app, mock_g.db, MagicMock(), test, "token") - - mock_mark_failed.assert_called_once() - self.assertIs(mock_mark_failed.call_args.args[1], test) - self.assertEqual(mock_mark_failed.call_args.args[3], - "Required GCP resource not found. Please contact the administrator.") - mock_wait_for_operation.assert_not_called() - mock_g.db.add.assert_not_called() - - @mock.patch('run.log') - @mock.patch('mod_ci.controllers.mark_test_failed') - @mock.patch('mod_ci.controllers.wait_for_operation') - @mock.patch('mod_ci.controllers.find_artifact_for_commit') - @mock.patch('mod_ci.controllers.requests.get') - @mock.patch('mod_ci.controllers.zipfile.ZipFile') - @mock.patch('builtins.open', new_callable=mock.mock_open()) - @mock.patch('mod_ci.controllers.g') - @mock.patch('mod_ci.controllers.TestProgress') - @mock.patch('mod_ci.controllers.GcpInstance') - def test_start_test_catches_http_error_from_instances_insert( - self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file, - mock_zipfile, mock_requests_get, mock_find_artifact, - mock_wait_for_operation, mock_mark_failed, mock_log): - """An HttpError from compute.instances().insert().execute() is handled in start_test. - - create_instance is not mocked here, so the error comes from the real call path. - """ - from mod_ci.controllers import start_test - - self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress, - mock_find_artifact, mock_requests_get, mock_zipfile) - compute = MagicMock() - compute.instances.return_value.insert.return_value.execute.side_effect = make_http_error( - 400, 'invalid', "Invalid value for field 'resource.machineType'") - - start_test(compute, self.app, mock_g.db, MagicMock(), Test.query.first(), "token") + cases = [ + (404, 'notFound', "Required GCP resource not found. Please contact the administrator."), + (400, 'invalid', "VM creation failed (invalid). Please contact the administrator."), + ] + for status, reason, expected_message in cases: + with self.subTest(status=status, reason=reason): + mocks, test = self._start_test_when_insert_raises(make_http_error(status, reason)) - compute.instances.return_value.insert.return_value.execute.assert_called_once() - mock_mark_failed.assert_called_once() - self.assertEqual(mock_mark_failed.call_args.args[3], - "VM creation failed (invalid). Please contact the administrator.") - mock_wait_for_operation.assert_not_called() - mock_g.db.add.assert_not_called() + mocks['mark_test_failed'].assert_called_once() + self.assertIs(mocks['mark_test_failed'].call_args.args[1], test) + self.assertEqual(mocks['mark_test_failed'].call_args.args[3], expected_message)