diff --git a/bugbug/sklearn_utils.py b/bugbug/sklearn_utils.py new file mode 100644 index 0000000000..fbb41ce0c6 --- /dev/null +++ b/bugbug/sklearn_utils.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import numpy as np +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.compose import ColumnTransformer +from sklearn.preprocessing import OrdinalEncoder + + +class StructuredColumnTransformer(ColumnTransformer): + def _hstack(self, Xs, n_samples): + result = super()._hstack(Xs, n_samples=n_samples) + + transformer_names = (name for name, transformer, column in self.transformers_) + types = [] + for f, transformer_name in zip(Xs, transformer_names): + types.append((transformer_name, result.dtype, (f.shape[1],))) + + return result.view(np.dtype(types)) + + +class DictExtractor(BaseEstimator, TransformerMixin): + def __init__(self, key): + self.key = key + + def fit(self, x, y=None): + return self + + def transform(self, data): + return np.array([elem[self.key] for elem in data]).reshape(-1, 1) + + +class MergeText(BaseEstimator, TransformerMixin): + def __init__(self, cols): + self.cols = cols + + def fit(self, X, y=None): + return self + + def transform(self, X): + return X[self.cols].apply(lambda row: " ".join(row), axis=1) + + +class MissingOrdinalEncoder(OrdinalEncoder): + """Ordinal encoder that ignores missing values encountered after training. + + Workaround for issue: scikit-learn/scikit-learn#11997 + """ + + def fit(self, X, y=None): + self._categories = self.categories + self._fit(X, handle_unknown="ignore") + return self + + def transform(self, X): + X_int, _ = self._transform(X, handle_unknown="ignore") + return X_int.astype(self.dtype, copy=False) diff --git a/bugbug/utils.py b/bugbug/utils.py index b3b927e48c..e55e352181 100644 --- a/bugbug/utils.py +++ b/bugbug/utils.py @@ -25,17 +25,12 @@ import dateutil.parser import libmozdata import lmdb -import numpy as np import psutil import requests -import scipy import taskcluster import tenacity import zstandard from requests.packages.urllib3.util.retry import Retry -from sklearn.base import BaseEstimator, TransformerMixin -from sklearn.compose import ColumnTransformer -from sklearn.preprocessing import OrdinalEncoder from bugbug import get_bugbug_version @@ -67,62 +62,14 @@ def numpy_to_dict(array): def to_array(val): + import scipy.sparse + if isinstance(val, scipy.sparse.csr_matrix): return val.toarray() return val -class StructuredColumnTransformer(ColumnTransformer): - def _hstack(self, Xs, n_samples): - result = super()._hstack(Xs, n_samples=n_samples) - - transformer_names = (name for name, transformer, column in self.transformers_) - types = [] - for f, transformer_name in zip(Xs, transformer_names): - types.append((transformer_name, result.dtype, (f.shape[1],))) - - return result.view(np.dtype(types)) - - -class DictExtractor(BaseEstimator, TransformerMixin): - def __init__(self, key): - self.key = key - - def fit(self, x, y=None): - return self - - def transform(self, data): - return np.array([elem[self.key] for elem in data]).reshape(-1, 1) - - -class MergeText(BaseEstimator, TransformerMixin): - def __init__(self, cols): - self.cols = cols - - def fit(self, X, y=None): - return self - - def transform(self, X): - return X[self.cols].apply(lambda row: " ".join(row), axis=1) - - -class MissingOrdinalEncoder(OrdinalEncoder): - """Ordinal encoder that ignores missing values encountered after training. - - Workaround for issue: scikit-learn/scikit-learn#11997 - """ - - def fit(self, X, y=None): - self._categories = self.categories - self._fit(X, handle_unknown="ignore") - return self - - def transform(self, X): - X_int, _ = self._transform(X, handle_unknown="ignore") - return X_int.astype(self.dtype, copy=False) - - def get_taskcluster_options() -> dict: """Get the Taskcluster setup options according to current environment. diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py index 3a9057c465..372457a77c 100644 --- a/http_service/bugbug_http/app.py +++ b/http_service/bugbug_http/app.py @@ -32,12 +32,7 @@ from sentry_sdk.integrations.flask import FlaskIntegration from bugbug import bugzilla, get_bugbug_version, utils -from bugbug.model import CommentModel -from bugbug.models import get_model_class -from bugbug_http.models import ( - COMMIT_MODELS, - MODELS_NAMES, - PUSH_CLASSIFIERS, +from bugbug_http.jobs import ( classify_broken_site_report, classify_bug, classify_comment, @@ -47,6 +42,12 @@ schedule_tests, schedule_tests_from_patch, ) +from bugbug_http.model_names import ( + COMMENT_MODELS, + COMMIT_MODELS, + MODELS_NAMES, + PUSH_CLASSIFIERS, +) from bugbug_http.sentry import setup_sentry if os.environ.get("SENTRY_DSN"): @@ -208,17 +209,31 @@ class Schedules(Schema): @dataclass(init=False, frozen=True) class JobInfo: - func: Callable[..., str] + """A job, running one of the functions of bugbug_http.models in the RQ workers. + + The function is given by its name (see bugbug_http.jobs), as the web + processes don't import bugbug_http.models (it loads the ML libraries), or + as the function itself (in bugbug_http.models). + """ + + func_name: str args: Sequence[Any] = field(default_factory=list) - def __init__(self, func, *args): + def __init__(self, func: str | Callable[..., str], *args): # Custom __init__ is needed to support *args, and object.__setattr__ is # needed for 'frozen=True'. - object.__setattr__(self, "func", func) + object.__setattr__( + self, "func_name", func if isinstance(func, str) else func.__name__ + ) object.__setattr__(self, "args", args) def __str__(self): - return f"{self.func.__name__}:{'_'.join(map(str, self.args))}" + return f"{self.func_name}:{'_'.join(map(str, self.args))}" + + @property + def func_path(self) -> str: + """The path of the job function, which the RQ workers import.""" + return f"bugbug_http.models.{self.func_name}" @property def mapping_key(self): @@ -264,7 +279,7 @@ def schedule_job( job_id = init_job(job, job_id) q.enqueue( - job.func, + job.func_path, *job.args, job_id=job_id, job_timeout=timeout, @@ -278,7 +293,7 @@ def prepare_queue_job( ) -> Queue: job_id = init_job(job, job_id) return Queue.prepare_data( - job.func, + job.func_path, args=job.args, job_id=job_id, timeout=timeout, @@ -443,9 +458,7 @@ def get_comments_last_change_time(comment_ids: Sequence[int]) -> dict[int, str]: def is_comment_model(model_name: str) -> bool: - return model_name in MODELS_NAMES and issubclass( - get_model_class(model_name), CommentModel - ) + return model_name in COMMENT_MODELS def get_github_issues_update_time( diff --git a/http_service/bugbug_http/download_models.py b/http_service/bugbug_http/download_models.py index 5e2d421807..3c20143207 100644 --- a/http_service/bugbug_http/download_models.py +++ b/http_service/bugbug_http/download_models.py @@ -7,7 +7,8 @@ from bugbug import utils from bugbug_http import ALLOW_MISSING_MODELS -from bugbug_http.models import MODEL_CACHE, MODELS_NAMES +from bugbug_http.model_names import MODELS_NAMES +from bugbug_http.models import MODEL_CACHE LOGGER = logging.getLogger() diff --git a/http_service/bugbug_http/jobs.py b/http_service/bugbug_http/jobs.py new file mode 100644 index 0000000000..93cb532979 --- /dev/null +++ b/http_service/bugbug_http/jobs.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""The names of the job functions of bugbug_http.models, which the RQ workers run. + +The web processes refer to the job functions with these names rather than importing +bugbug_http.models, as it loads the ML libraries. Imported by name, a misspelled one fails like a +misspelled function would: as an undefined name for the linter, or an ImportError. +""" + +classify_broken_site_report = "classify_broken_site_report" +classify_bug = "classify_bug" +classify_comment = "classify_comment" +classify_commits = "classify_commits" +classify_issue = "classify_issue" +classify_perf_regression = "classify_perf_regression" +get_config_specific_groups = "get_config_specific_groups" +schedule_tests = "schedule_tests" +schedule_tests_from_patch = "schedule_tests_from_patch" diff --git a/http_service/bugbug_http/model_names.py b/http_service/bugbug_http/model_names.py new file mode 100644 index 0000000000..f6ce3a50ba --- /dev/null +++ b/http_service/bugbug_http/model_names.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""The models served by the HTTP service. + +They are listed here rather than in bugbug_http.models, which the web processes don't import as it +loads the ML libraries. +""" + +from bugbug_http import jobs + +MODELS_NAMES = [ + "defectenhancementtask", + "component", + "componentspecific", + "invalidcompatibilityreport", + "needsdiagnosis", + "regression", + "stepstoreproduce", + "spambug", + "testlabelselect", + "testgroupselect", + "testconfigselectvalue", + "accessibility", + "performancebug", + "worksforme", + "fenixcomponent", + "perfregressionpredictor", + "regressor", +] + +# The models in MODELS_NAMES which classify comments (subclasses of bugbug.model.CommentModel). +COMMENT_MODELS: list[str] = [] + +# Models that classify a push (branch + revision) rather than a bug or an issue, keyed by the model +# name accepted by the push prediction endpoint, with their job function (see bugbug_http.jobs). +PUSH_CLASSIFIERS = { + "perfregressionpredictor": jobs.classify_perf_regression, +} + +# Models that classify single commits (see classify_commits). +COMMIT_MODELS = ["regressor"] diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py index 1511cb3e16..778f82e04c 100644 --- a/http_service/bugbug_http/models.py +++ b/http_service/bugbug_http/models.py @@ -34,26 +34,6 @@ logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger() -MODELS_NAMES = [ - "defectenhancementtask", - "component", - "componentspecific", - "invalidcompatibilityreport", - "needsdiagnosis", - "regression", - "stepstoreproduce", - "spambug", - "testlabelselect", - "testgroupselect", - "testconfigselectvalue", - "accessibility", - "performancebug", - "worksforme", - "fenixcomponent", - "perfregressionpredictor", - "regressor", -] - DEFAULT_EXPIRATION_TTL = 7 * 24 * 3600 # A week TREEHERDER_API_URL = "https://treeherder.mozilla.org/api" @@ -470,16 +450,6 @@ def classify_perf_regression(branch: str, rev: str) -> str: return "OK" -# Models that classify a push (branch + revision) rather than a bug or an -# issue, keyed by the model name accepted by the push prediction endpoint. -PUSH_CLASSIFIERS = { - PERF_REGRESSION_PREDICTOR: classify_perf_regression, -} - -# Models that classify single commits (see classify_commits). -COMMIT_MODELS = ["regressor"] - - def _get_ignored_reason(commit: repository.CommitDict) -> str | None: if commit["backsout"]: return "backout" diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py index 9bb8b9e62e..da196a0ef5 100644 --- a/http_service/tests/conftest.py +++ b/http_service/tests/conftest.py @@ -22,6 +22,7 @@ import zstandard from _pytest.monkeypatch import MonkeyPatch from rq.exceptions import NoSuchJobError +from rq.utils import import_attribute import bugbug.models import bugbug.models.testconfigselect @@ -98,10 +99,13 @@ def __init__(self, *args, **kwargs): def enqueue( self, func, *args, job_id=None, job_timeout=None, ttl=None, failure_ttl=None ): - pass + # The web processes refer to the job functions by path: check that the RQ workers + # would find them. + assert callable(import_attribute(func)) def enqueue_many(self, job_datas, pipeline=None): - pass + for job_data in job_datas: + assert callable(import_attribute(job_data.func)) class JobMock: """Mock class to mimic rq.job.Job.""" diff --git a/http_service/tests/test_web_imports.py b/http_service/tests/test_web_imports.py new file mode 100644 index 0000000000..3f2ac267e2 --- /dev/null +++ b/http_service/tests/test_web_imports.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import subprocess +import sys + +from bugbug.model import CommentModel +from bugbug.models import get_model_class +from bugbug_http import jobs, model_names, models + +# bugbug_http.models, which only the RQ workers run, and the libraries only they need, which would +# add a couple hundred MB to each web process. +WORKER_MODULES = [ + "bugbug_http.models", + "numpy", + "pandas", + "scipy", + "sklearn", + "xgboost", + "imblearn", +] + + +def test_web_does_not_import_worker_modules() -> None: + # In a new interpreter, as the tests themselves import them. + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys, bugbug_http.app; " + f"print(' '.join(m for m in {WORKER_MODULES!r} if m in sys.modules))", + ], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.split() == [] + + +def test_model_names() -> None: + assert [ + model_name + for model_name in model_names.MODELS_NAMES + if issubclass(get_model_class(model_name), CommentModel) + ] == model_names.COMMENT_MODELS + + assert model_names.PUSH_CLASSIFIERS.keys() <= set(model_names.MODELS_NAMES) + + +def test_job_names() -> None: + names = { + name: value for name, value in vars(jobs).items() if not name.startswith("_") + } + assert names + for name, value in names.items(): + assert value == name + assert callable(getattr(models, name)) diff --git a/tests/test_sklearn_utils.py b/tests/test_sklearn_utils.py new file mode 100644 index 0000000000..d74a1e7ae5 --- /dev/null +++ b/tests/test_sklearn_utils.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.feature_extraction.text import CountVectorizer + +from bugbug import sklearn_utils + + +def test_StructuredColumnTransformer() -> None: + transformers = [ + ("feat1_transformed", CountVectorizer(), "feat1"), + ("feat2_transformed", CountVectorizer(), "feat2"), + ] + + df = pd.DataFrame( + [ + { + "feat1": "First", + "feat2": "Second", + }, + { + "feat1": "Third", + "feat2": "Fourth", + }, + ] + ) + + np.testing.assert_array_equal( + ColumnTransformer(transformers).fit_transform(df), + np.array([[1, 0, 0, 1], [0, 1, 1, 0]]), + ) + + np.testing.assert_array_equal( + sklearn_utils.StructuredColumnTransformer(transformers).fit_transform(df), + np.array( + [[([1, 0], [0, 1])], [([0, 1], [1, 0])]], + dtype=[ + ("feat1_transformed", " None: assert result is None -def test_StructuredColumnTransformer() -> None: - transformers = [ - ("feat1_transformed", CountVectorizer(), "feat1"), - ("feat2_transformed", CountVectorizer(), "feat2"), - ] - - df = pd.DataFrame( - [ - { - "feat1": "First", - "feat2": "Second", - }, - { - "feat1": "Third", - "feat2": "Fourth", - }, - ] - ) - - np.testing.assert_array_equal( - ColumnTransformer(transformers).fit_transform(df), - np.array([[1, 0, 0, 1], [0, 1, 1, 0]]), - ) - - np.testing.assert_array_equal( - utils.StructuredColumnTransformer(transformers).fit_transform(df), - np.array( - [[([1, 0], [0, 1])], [([0, 1], [1, 0])]], - dtype=[ - ("feat1_transformed", " None: from datetime import timedelta, timezone