Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions bugbug/sklearn_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OrdinalEncoder


class StructuredColumnTransformer(ColumnTransformer):
def _hstack(self, Xs, n_samples):
result = super()._hstack(Xs, n_samples=n_samples)

transformer_names = (name for name, transformer, column in self.transformers_)
types = []
for f, transformer_name in zip(Xs, transformer_names):
types.append((transformer_name, result.dtype, (f.shape[1],)))

return result.view(np.dtype(types))


class DictExtractor(BaseEstimator, TransformerMixin):
def __init__(self, key):
self.key = key

def fit(self, x, y=None):
return self

def transform(self, data):
return np.array([elem[self.key] for elem in data]).reshape(-1, 1)


class MergeText(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols = cols

def fit(self, X, y=None):
return self

def transform(self, X):
return X[self.cols].apply(lambda row: " ".join(row), axis=1)


class MissingOrdinalEncoder(OrdinalEncoder):
"""Ordinal encoder that ignores missing values encountered after training.

Workaround for issue: scikit-learn/scikit-learn#11997
"""

def fit(self, X, y=None):
self._categories = self.categories
self._fit(X, handle_unknown="ignore")
return self

def transform(self, X):
X_int, _ = self._transform(X, handle_unknown="ignore")
return X_int.astype(self.dtype, copy=False)
57 changes: 2 additions & 55 deletions bugbug/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,12 @@
import dateutil.parser
import libmozdata
import lmdb
import numpy as np
import psutil
import requests
import scipy
import taskcluster
import tenacity
import zstandard
from requests.packages.urllib3.util.retry import Retry
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OrdinalEncoder

from bugbug import get_bugbug_version

Expand Down Expand Up @@ -67,62 +62,14 @@ def numpy_to_dict(array):


def to_array(val):
import scipy.sparse

if isinstance(val, scipy.sparse.csr_matrix):
return val.toarray()

return val


class StructuredColumnTransformer(ColumnTransformer):
def _hstack(self, Xs, n_samples):
result = super()._hstack(Xs, n_samples=n_samples)

transformer_names = (name for name, transformer, column in self.transformers_)
types = []
for f, transformer_name in zip(Xs, transformer_names):
types.append((transformer_name, result.dtype, (f.shape[1],)))

return result.view(np.dtype(types))


class DictExtractor(BaseEstimator, TransformerMixin):
def __init__(self, key):
self.key = key

def fit(self, x, y=None):
return self

def transform(self, data):
return np.array([elem[self.key] for elem in data]).reshape(-1, 1)


class MergeText(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols = cols

def fit(self, X, y=None):
return self

def transform(self, X):
return X[self.cols].apply(lambda row: " ".join(row), axis=1)


class MissingOrdinalEncoder(OrdinalEncoder):
"""Ordinal encoder that ignores missing values encountered after training.

Workaround for issue: scikit-learn/scikit-learn#11997
"""

def fit(self, X, y=None):
self._categories = self.categories
self._fit(X, handle_unknown="ignore")
return self

def transform(self, X):
X_int, _ = self._transform(X, handle_unknown="ignore")
return X_int.astype(self.dtype, copy=False)


def get_taskcluster_options() -> dict:
"""Get the Taskcluster setup options according to current environment.

Expand Down
43 changes: 28 additions & 15 deletions http_service/bugbug_http/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,7 @@
from sentry_sdk.integrations.flask import FlaskIntegration

from bugbug import bugzilla, get_bugbug_version, utils
from bugbug.model import CommentModel
from bugbug.models import get_model_class
from bugbug_http.models import (
COMMIT_MODELS,
MODELS_NAMES,
PUSH_CLASSIFIERS,
from bugbug_http.jobs import (
classify_broken_site_report,
classify_bug,
classify_comment,
Expand All @@ -47,6 +42,12 @@
schedule_tests,
schedule_tests_from_patch,
)
from bugbug_http.model_names import (
COMMENT_MODELS,
COMMIT_MODELS,
MODELS_NAMES,
PUSH_CLASSIFIERS,
)
from bugbug_http.sentry import setup_sentry

if os.environ.get("SENTRY_DSN"):
Expand Down Expand Up @@ -208,17 +209,31 @@ class Schedules(Schema):

@dataclass(init=False, frozen=True)
class JobInfo:
func: Callable[..., str]
"""A job, running one of the functions of bugbug_http.models in the RQ workers.

The function is given by its name (see bugbug_http.jobs), as the web
processes don't import bugbug_http.models (it loads the ML libraries), or
as the function itself (in bugbug_http.models).
"""

func_name: str
args: Sequence[Any] = field(default_factory=list)

def __init__(self, func, *args):
def __init__(self, func: str | Callable[..., str], *args):
# Custom __init__ is needed to support *args, and object.__setattr__ is
# needed for 'frozen=True'.
object.__setattr__(self, "func", func)
object.__setattr__(
self, "func_name", func if isinstance(func, str) else func.__name__
)
object.__setattr__(self, "args", args)

def __str__(self):
return f"{self.func.__name__}:{'_'.join(map(str, self.args))}"
return f"{self.func_name}:{'_'.join(map(str, self.args))}"

@property
def func_path(self) -> str:
"""The path of the job function, which the RQ workers import."""
return f"bugbug_http.models.{self.func_name}"

@property
def mapping_key(self):
Expand Down Expand Up @@ -264,7 +279,7 @@ def schedule_job(
job_id = init_job(job, job_id)

q.enqueue(
job.func,
job.func_path,
*job.args,
job_id=job_id,
job_timeout=timeout,
Expand All @@ -278,7 +293,7 @@ def prepare_queue_job(
) -> Queue:
job_id = init_job(job, job_id)
return Queue.prepare_data(
job.func,
job.func_path,
args=job.args,
job_id=job_id,
timeout=timeout,
Expand Down Expand Up @@ -443,9 +458,7 @@ def get_comments_last_change_time(comment_ids: Sequence[int]) -> dict[int, str]:


def is_comment_model(model_name: str) -> bool:
return model_name in MODELS_NAMES and issubclass(
get_model_class(model_name), CommentModel
)
return model_name in COMMENT_MODELS


def get_github_issues_update_time(
Expand Down
3 changes: 2 additions & 1 deletion http_service/bugbug_http/download_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

from bugbug import utils
from bugbug_http import ALLOW_MISSING_MODELS
from bugbug_http.models import MODEL_CACHE, MODELS_NAMES
from bugbug_http.model_names import MODELS_NAMES
from bugbug_http.models import MODEL_CACHE

LOGGER = logging.getLogger()

Expand Down
21 changes: 21 additions & 0 deletions http_service/bugbug_http/jobs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

"""The names of the job functions of bugbug_http.models, which the RQ workers run.

The web processes refer to the job functions with these names rather than importing
bugbug_http.models, as it loads the ML libraries. Imported by name, a misspelled one fails like a
misspelled function would: as an undefined name for the linter, or an ImportError.
"""

classify_broken_site_report = "classify_broken_site_report"
classify_bug = "classify_bug"
classify_comment = "classify_comment"
classify_commits = "classify_commits"
classify_issue = "classify_issue"
classify_perf_regression = "classify_perf_regression"
get_config_specific_groups = "get_config_specific_groups"
schedule_tests = "schedule_tests"
schedule_tests_from_patch = "schedule_tests_from_patch"
44 changes: 44 additions & 0 deletions http_service/bugbug_http/model_names.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

"""The models served by the HTTP service.

They are listed here rather than in bugbug_http.models, which the web processes don't import as it
loads the ML libraries.
"""

from bugbug_http import jobs

MODELS_NAMES = [
"defectenhancementtask",
"component",
"componentspecific",
"invalidcompatibilityreport",
"needsdiagnosis",
"regression",
"stepstoreproduce",
"spambug",
"testlabelselect",
"testgroupselect",
"testconfigselectvalue",
"accessibility",
"performancebug",
"worksforme",
"fenixcomponent",
"perfregressionpredictor",
"regressor",
]

# The models in MODELS_NAMES which classify comments (subclasses of bugbug.model.CommentModel).
COMMENT_MODELS: list[str] = []

# Models that classify a push (branch + revision) rather than a bug or an issue, keyed by the model
# name accepted by the push prediction endpoint, with their job function (see bugbug_http.jobs).
PUSH_CLASSIFIERS = {
"perfregressionpredictor": jobs.classify_perf_regression,
}

# Models that classify single commits (see classify_commits).
COMMIT_MODELS = ["regressor"]
30 changes: 0 additions & 30 deletions http_service/bugbug_http/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,26 +34,6 @@
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger()

MODELS_NAMES = [
"defectenhancementtask",
"component",
"componentspecific",
"invalidcompatibilityreport",
"needsdiagnosis",
"regression",
"stepstoreproduce",
"spambug",
"testlabelselect",
"testgroupselect",
"testconfigselectvalue",
"accessibility",
"performancebug",
"worksforme",
"fenixcomponent",
"perfregressionpredictor",
"regressor",
]

DEFAULT_EXPIRATION_TTL = 7 * 24 * 3600 # A week

TREEHERDER_API_URL = "https://treeherder.mozilla.org/api"
Expand Down Expand Up @@ -470,16 +450,6 @@ def classify_perf_regression(branch: str, rev: str) -> str:
return "OK"


# Models that classify a push (branch + revision) rather than a bug or an
# issue, keyed by the model name accepted by the push prediction endpoint.
PUSH_CLASSIFIERS = {
PERF_REGRESSION_PREDICTOR: classify_perf_regression,
}

# Models that classify single commits (see classify_commits).
COMMIT_MODELS = ["regressor"]


def _get_ignored_reason(commit: repository.CommitDict) -> str | None:
if commit["backsout"]:
return "backout"
Expand Down
8 changes: 6 additions & 2 deletions http_service/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import zstandard
from _pytest.monkeypatch import MonkeyPatch
from rq.exceptions import NoSuchJobError
from rq.utils import import_attribute

import bugbug.models
import bugbug.models.testconfigselect
Expand Down Expand Up @@ -98,10 +99,13 @@ def __init__(self, *args, **kwargs):
def enqueue(
self, func, *args, job_id=None, job_timeout=None, ttl=None, failure_ttl=None
):
pass
# The web processes refer to the job functions by path: check that the RQ workers
# would find them.
assert callable(import_attribute(func))

def enqueue_many(self, job_datas, pipeline=None):
pass
for job_data in job_datas:
assert callable(import_attribute(job_data.func))

class JobMock:
"""Mock class to mimic rq.job.Job."""
Expand Down
Loading