Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 66 additions & 1 deletion mod_ci/controllers.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from github import (Auth, Commit, Github, GithubException, GithubObject,
Repository)
from google.oauth2 import service_account
from googleapiclient.errors import HttpError
from lxml import etree
from markdown2 import markdown
from pymysql.err import IntegrityError
Expand Down Expand Up @@ -154,6 +155,8 @@
'RESOURCE_NOT_FOUND': "Required GCP resource not found. Please contact the administrator.",
'RESOURCE_ALREADY_EXISTS': "A VM with this name already exists. Please contact the administrator.",
'TIMEOUT': "GCP operation timed out. The test will be retried automatically.",
'RATE_LIMIT_EXCEEDED': "GCP API rate limit reached. The test will be retried automatically.",
'BACKEND_ERROR': "GCP returned a temporary server error. The test will be retried automatically.",
}

# GCP error codes that are transient and should be retried automatically.
Expand All @@ -162,8 +165,62 @@
'ZONE_RESOURCE_POOL_EXHAUSTED',
'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS',
'QUOTA_EXCEEDED',
'RATE_LIMIT_EXCEEDED',
'BACKEND_ERROR',
}

# When the Compute API rejects a request outright, the client library raises HttpError instead
# of returning an operation. Its body names the failure with a camelCase "reason" rather than
# the upper-case "code" an operation carries. These are the reasons that have an equivalent
# code above; any other reason is kept as it is.
GCP_HTTP_ERROR_REASONS = {
'quotaExceeded': 'QUOTA_EXCEEDED',
'rateLimitExceeded': 'RATE_LIMIT_EXCEEDED',
'userRateLimitExceeded': 'RATE_LIMIT_EXCEEDED',
'backendError': 'BACKEND_ERROR',
'internalError': 'BACKEND_ERROR',
'notFound': 'RESOURCE_NOT_FOUND',
'alreadyExists': 'RESOURCE_ALREADY_EXISTS',
}

# HTTP statuses that mean "try again later", whatever reason the body gives.
GCP_RETRYABLE_HTTP_STATUSES = {429, 500, 502, 503, 504}


def gcp_error_from_http_error(error: HttpError) -> Dict:
"""
Convert an HttpError raised by a Compute API call into a GCP error response.

The result has the same shape as a failed operation, so parse_gcp_error() and
is_retryable_gcp_error() treat both kinds of failure the same way.

:param error: The error raised by the googleapiclient request
:return: A dictionary of the form {'error': {'errors': [{'code': ..., 'message': ...}]}}
"""
status = getattr(error.resp, 'status', None)
reason = None
message = str(error)
try:
body = json.loads(error.content.decode('utf-8'))
except (AttributeError, UnicodeDecodeError, ValueError):

Check warning on line 205 in mod_ci/controllers.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this redundant Exception class; it derives from another which is already caught.

See more on https://sonarcloud.io/project/issues?id=CCExtractor_sample-platform&issues=AaHl-pYebAHkmNKSPfMI&open=AaHl-pYebAHkmNKSPfMI&pullRequest=1279
body = None
error_body = body.get('error') if isinstance(body, dict) else None
if isinstance(error_body, dict):
message = error_body.get('message') or message
errors = error_body.get('errors')
if isinstance(errors, list) and errors and isinstance(errors[0], dict):
reason = errors[0].get('reason')
message = errors[0].get('message') or message

code = GCP_HTTP_ERROR_REASONS.get(reason, reason) if reason else f'HTTP_{status}'
if status in GCP_RETRYABLE_HTTP_STATUSES and code not in GCP_RETRYABLE_ERRORS:
code = 'RATE_LIMIT_EXCEEDED' if status == 429 else 'BACKEND_ERROR'

return {
'error': {'errors': [{'code': code, 'message': message}]},
'httpErrorStatusCode': status,
}


def get_gcp_error_code(result: Dict) -> Optional[str]:
"""
Expand Down Expand Up @@ -1233,7 +1290,15 @@

zone = config.get('ZONE', '')
project_id = config.get('PROJECT_NAME', '')
operation = create_instance(compute, project_id, zone, test, full_url)
try:
operation = create_instance(compute, project_id, zone, test, full_url)
except HttpError as e:
# A request the Compute API rejects (bad config, missing image, quota, rate limit,
# server error) raises instead of returning an error in the operation. Left uncaught,
# it reached gcp_instance(), which retries every test on every cron run, so a
# permanent error kept the test pending forever and never reported a failure.
log.error(f"Test {test.id}: Compute API rejected VM creation: {e}")
operation = gcp_error_from_http_error(e)

# Check if the create_instance call itself returned an error (synchronous failure)
if 'error' in operation:
Expand Down
109 changes: 107 additions & 2 deletions tests/test_ci/test_controllers.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
import contextlib
import json
import os
import unittest
from importlib import reload
from unittest import mock
from unittest.mock import MagicMock

import httplib2
from flask import g
from googleapiclient.errors import HttpError

from mod_auth.models import Role
from mod_ci.controllers import (Workflow_builds, get_info_for_pr_comment,
is_valid_commit_hash, mark_test_failed,
from mod_ci.controllers import (Workflow_builds, gcp_error_from_http_error,
get_gcp_error_code, get_info_for_pr_comment,
is_retryable_gcp_error, is_valid_commit_hash,
mark_test_failed, parse_gcp_error,
parse_git_commit_from_log_file,
parse_git_commit_from_log_stream,
parse_git_commit_from_logs,
Expand Down Expand Up @@ -3951,6 +3956,49 @@ def test_parse_gcp_error_error_not_a_dict(self):
mock_log.error.assert_called_once()


def make_http_error(status, reason=None, message='Request failed', content=None):
"""Build the HttpError googleapiclient raises when the Compute API rejects a request."""
if content is None:
error = {'code': status, 'message': message}
if reason is not None:
error['errors'] = [{'message': message, 'domain': 'global', 'reason': reason}]
content = json.dumps({'error': error}).encode('utf-8')
return HttpError(httplib2.Response({'status': status}), content)


class TestGcpErrorFromHttpError(unittest.TestCase):
"""Tests for gcp_error_from_http_error, which maps an HttpError onto a GCP error response."""

def test_maps_reason_and_status_to_error_code(self):
"""Known reasons map to the platform's codes, and 429 or 5xx is always retryable."""
cases = [
# status, reason, body (None builds a Compute-style JSON body), code, retryable
(403, 'quotaExceeded', None, 'QUOTA_EXCEEDED', True),
(403, 'rateLimitExceeded', None, 'RATE_LIMIT_EXCEEDED', True),
(404, 'notFound', None, 'RESOURCE_NOT_FOUND', False),
(409, 'alreadyExists', None, 'RESOURCE_ALREADY_EXISTS', False),
(400, 'invalid', None, 'invalid', False),
(503, 'serviceUnavailable', None, 'BACKEND_ERROR', True),
(429, None, b'<html>Too Many Requests</html>', 'RATE_LIMIT_EXCEEDED', True),
(400, None, b'Bad Request', 'HTTP_400', False),
]
for status, reason, content, code, retryable in cases:
with self.subTest(status=status, reason=reason):
result = gcp_error_from_http_error(make_http_error(status, reason, content=content))
self.assertEqual(get_gcp_error_code(result), code)
self.assertEqual(is_retryable_gcp_error(result), retryable)
self.assertEqual(result['httpErrorStatusCode'], status)

def test_keeps_gcp_message_out_of_user_message(self):
"""The GCP message is kept for the log, while the user sees a generic message."""
result = gcp_error_from_http_error(
make_http_error(400, 'invalid', "Invalid value for field 'machineType'"))

self.assertEqual(result['error']['errors'][0]['message'], "Invalid value for field 'machineType'")
self.assertEqual(parse_gcp_error(result, log=MagicMock()),
"VM creation failed (invalid). Please contact the administrator.")


class TestDiagnoseMissingArtifact(BaseTestCase):
"""Test the _diagnose_missing_artifact function and retry behavior."""

Expand Down Expand Up @@ -4811,3 +4859,60 @@ def test_start_test_operation_timeout_creates_db_record(
start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token")

mock_g.db.add.assert_called_once()

def _start_test_when_insert_raises(self, error):
"""Run start_test with compute.instances().insert().execute() raising the given error.

Returns the patched mocks, keyed by the name of what they replace, and the test that was started.
"""
from mod_ci.controllers import start_test

targets = ['run.log', 'mod_ci.controllers.mark_test_failed', 'mod_ci.controllers.wait_for_operation',
'mod_ci.controllers.find_artifact_for_commit', 'mod_ci.controllers.requests.get',
'mod_ci.controllers.zipfile.ZipFile', 'mod_ci.controllers.g',
'mod_ci.controllers.TestProgress', 'mod_ci.controllers.GcpInstance']
with contextlib.ExitStack() as stack:
mocks = {target.rsplit('.', 1)[1]: stack.enter_context(mock.patch(target)) for target in targets}
stack.enter_context(mock.patch('builtins.open', new_callable=mock.mock_open()))
self._setup_start_test_mocks(mocks['g'], mocks['GcpInstance'], mocks['TestProgress'],
mocks['find_artifact_for_commit'], mocks['get'], mocks['ZipFile'])
compute = MagicMock()
compute.instances.return_value.insert.return_value.execute.side_effect = error
test = Test.query.first()

start_test(compute, self.app, mocks['g'].db, MagicMock(), test, "token")

compute.instances.return_value.insert.return_value.execute.assert_called_once()
mocks['wait_for_operation'].assert_not_called()
mocks['g'].db.add.assert_not_called()
return mocks, test

def test_start_test_retries_transient_http_error(self):
"""Quota, rate limit and server errors raised by the VM insert leave the test pending for retry.

The Compute API raises HttpError for a rejected insert instead of returning an
error in the operation, so start_test must catch it rather than let it escape.
"""
for status, reason in [(403, 'quotaExceeded'), (403, 'rateLimitExceeded'), (503, 'backendError')]:
with self.subTest(status=status, reason=reason):
mocks, _ = self._start_test_when_insert_raises(make_http_error(status, reason))

mocks['mark_test_failed'].assert_not_called()

def test_start_test_fails_test_on_permanent_http_error(self):
"""A permanent error raised by the VM insert marks the test as failed.

Before the fix this escaped to gcp_instance(), which logged it and retried the
test on every cron run, so it never left the queue and never reported a failure.
"""
cases = [
(404, 'notFound', "Required GCP resource not found. Please contact the administrator."),
(400, 'invalid', "VM creation failed (invalid). Please contact the administrator."),
]
for status, reason, expected_message in cases:
with self.subTest(status=status, reason=reason):
mocks, test = self._start_test_when_insert_raises(make_http_error(status, reason))

mocks['mark_test_failed'].assert_called_once()
self.assertIs(mocks['mark_test_failed'].call_args.args[1], test)
self.assertEqual(mocks['mark_test_failed'].call_args.args[3], expected_message)