From 620adaf31c20ceb1e99372b36d333e34681225ab Mon Sep 17 00:00:00 2001 From: Veydop <37590446+Veydop@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:50:44 +0530 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20refactored=20apply=20custom=20dedupl?= =?UTF-8?q?ication=20rules=20so=20that=20deduplicatio=E2=80=A6=20(#6794)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- keep/api/core/db.py | 10 ++ keep/api/tasks/process_event_task.py | 19 ++- keep/providers/base/base_provider.py | 69 ++++++++-- tests/deduplication/test_deduplications.py | 142 ++++++++++++++++++++- 4 files changed, 224 insertions(+), 16 deletions(-) diff --git a/keep/api/core/db.py b/keep/api/core/db.py index 6cda3595a6..1e89b9f8ae 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -103,6 +103,9 @@ "assignee", ] KEEP_AUDIT_EVENTS_ENABLED = config("KEEP_AUDIT_EVENTS_ENABLED", cast=bool, default=True) +KEEP_CUSTOM_DEDUPLICATION_ENABLED = config( + "KEEP_CUSTOM_DEDUPLICATION_ENABLED", cast=bool, default=True +) INTERVAL_WORKFLOWS_RELAUNCH_TIMEOUT = timedelta(minutes=60) WORKFLOWS_TIMEOUT = timedelta(minutes=120) @@ -2584,6 +2587,13 @@ def get_deduplication_rule_by_id(tenant_id, rule_id: str): def get_custom_deduplication_rule(tenant_id, provider_id, provider_type): + # check the custom deduplication flag here so every caller behaves the same + if not KEEP_CUSTOM_DEDUPLICATION_ENABLED: + return None + # alerts ingested without a provider are attributed to the "keep" provider. + # normalizing here so all callers resolve the same rule row + if not provider_type: + provider_type = "keep" with Session(engine) as session: rule = session.exec( select(AlertDeduplicationRule) diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index d74362e92b..501455688b 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -768,13 +768,30 @@ def process_event( if isinstance(event, dict): if not event.get("name"): event["name"] = event.get("id", "unknown alert name") - event = [AlertDto(**event)] + # format through the "keep" provider instead of building the AlertDto + # directly, so that a custom deduplication rule is applied here too + event = ProvidersFactory.get_provider_class("keep").format_alert( + tenant_id=tenant_id, + event=event, + provider_id=provider_id, + provider_type=provider_type, + ) raw_event = [raw_event] # Prepare the event for the digest if isinstance(event, AlertDto): event = [event] raw_event = [raw_event] + # an alert that arrives already parsed skips provider formatting, and + # with it the custom deduplication rule - apply it explicitly + event = ProvidersFactory.get_provider_class( + "keep" + ).apply_custom_deduplication_rule( + event, + tenant_id=tenant_id, + provider_id=provider_id, + provider_type=provider_type, + ) with tracer.start_as_current_span("process_event_internal_preparation"): __internal_prepartion(event, fingerprint, api_key_name) diff --git a/keep/providers/base/base_provider.py b/keep/providers/base/base_provider.py index 87e9e5ae3e..e7a09a49ad 100644 --- a/keep/providers/base/base_provider.py +++ b/keep/providers/base/base_provider.py @@ -469,13 +469,6 @@ def format_alert( ) return None logger.debug("Alert formatted") - # after the provider calculated the default fingerprint - # check if there is a custom deduplication rule and apply - custom_deduplication_rule = get_custom_deduplication_rule( - tenant_id=tenant_id, - provider_id=provider_id, - provider_type=provider_type, - ) if not isinstance(formatted_alert, list): formatted_alert.providerId = provider_id @@ -487,12 +480,50 @@ def format_alert( alert.providerId = provider_id alert.providerType = provider_type - # if there is no custom deduplication rule, return the formatted alert + # after the provider calculated the default fingerprint + # check if there is a custom deduplication rule and apply + return cls.apply_custom_deduplication_rule( + formatted_alert, + tenant_id=tenant_id, + provider_id=provider_id, + provider_type=provider_type, + ) + + @classmethod + def apply_custom_deduplication_rule( + cls, + alerts: list[AlertDto], + tenant_id: str, + provider_id: str | None, + provider_type: str | None, + ) -> list[AlertDto]: + """ + Override the fingerprint of already-formatted alerts with a custom deduplication rule. + + Alerts that reach Keep already parsed as AlertDto never go through + _format_alert, so this has to be callable on its own - otherwise a configured + rule is silently ignored for them. + + Args: + alerts (list[AlertDto]): The alerts to apply the rule to, in place. + tenant_id (str): The tenant id. + provider_id (str | None): The provider id, if any. + provider_type (str | None): The provider type, if any. + + Returns: + list[AlertDto]: The same alerts, with fingerprints overridden if a rule exists. + """ + logger = logging.getLogger(__name__) + custom_deduplication_rule = get_custom_deduplication_rule( + tenant_id=tenant_id, + provider_id=provider_id, + provider_type=provider_type, + ) + # if there is no custom deduplication rule, keep the provider's fingerprint if not custom_deduplication_rule: - return formatted_alert - # if there is a custom deduplication rule, apply it - # apply the custom deduplication rule to calculate the fingerprint - for alert in formatted_alert: + return alerts + + for alert in alerts: logger.info( "Applying custom deduplication rule", extra={ @@ -504,7 +535,7 @@ def format_alert( alert.fingerprint = cls.get_alert_fingerprint( alert, custom_deduplication_rule.fingerprint_fields ) - return formatted_alert + return alerts @staticmethod def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str: @@ -518,10 +549,12 @@ def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str Returns: str: hexdigest of the fingerprint or the event.name if no fingerprint_fields were given. """ + logger = logging.getLogger(__name__) if not fingerprint_fields: return alert.name fingerprint = hashlib.sha256() event_dict = alert.dict() + matched_fields = [] for fingerprint_field in fingerprint_fields: keys = fingerprint_field.split(".") fingerprint_field_value = event_dict @@ -535,6 +568,16 @@ def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str fingerprint_field_value = json.dumps(fingerprint_field_value) if fingerprint_field_value is not None: fingerprint.update(str(fingerprint_field_value).encode()) + matched_fields.append(fingerprint_field) + if not matched_fields: + logger.warning( + "None of the fingerprint fields were found on the alert - " + "all alerts will share the same fingerprint", + extra={ + "fingerprint_fields": fingerprint_fields, + "alert_name": alert.name, + }, + ) return fingerprint.hexdigest() def get_alerts_configuration(self, alert_id: Optional[str] = None): diff --git a/tests/deduplication/test_deduplications.py b/tests/deduplication/test_deduplications.py index 508b8b543c..eaa89083d8 100644 --- a/tests/deduplication/test_deduplications.py +++ b/tests/deduplication/test_deduplications.py @@ -1,3 +1,4 @@ +import hashlib import logging import random import time @@ -8,11 +9,18 @@ import pytz from sqlalchemy import text -from keep.api.core.db import get_last_alerts +from keep.api.core import db as db_module +from keep.api.core.db import get_custom_deduplication_rule, get_last_alerts from keep.api.core.dependencies import SINGLE_TENANT_UUID -from keep.api.models.alert import DeduplicationRuleDto, AlertStatus +from keep.api.models.alert import ( + AlertDto, + AlertSeverity, + AlertStatus, + DeduplicationRuleDto, +) from keep.api.models.db.alert import AlertDeduplicationRule, AlertDeduplicationEvent, Alert from keep.api.utils.enrichment_helpers import convert_db_alerts_to_dto_alerts +from keep.providers.base.base_provider import BaseProvider from keep.providers.providers_factory import ProvidersFactory from tests.fixtures.client import client, setup_api_key, test_app # noqa @@ -1018,3 +1026,133 @@ def test_sort_keys_deduplication_fix(db_session, client, test_app): assert prometheus_rule is not None assert prometheus_rule.get("ingested") == 2 assert prometheus_rule.get("dedup_ratio") == 50.0 # 1 out of 2 was deduplicated + + +def _dedup_test_alert(**kwargs) -> AlertDto: + payload = { + "id": "test-id", + "name": "test alert", + "status": AlertStatus.FIRING.value, + "severity": AlertSeverity.CRITICAL.value, + "lastReceived": "2024-01-01T00:00:00.000Z", + "source": ["keep"], + } + payload.update(kwargs) + return AlertDto(**payload) + + +def _add_keep_dedup_rule(db_session): + db_session.exec(text("DELETE FROM alertdeduplicationrule")) + rule = AlertDeduplicationRule( + name="catch all rule", + description="test", + tenant_id=SINGLE_TENANT_UUID, + provider_type="keep", + provider_id=None, + fingerprint_fields=["service"], + full_deduplication=False, + ignore_fields=[], + last_updated_by="test", + created_by="test", + ) + db_session.add(rule) + db_session.commit() + db_session.refresh(rule) + return rule + + +def test_fingerprint_warns_when_no_field_resolves(caplog): + # when none of the configured fields exist on the alert, every alert collapses + # into the same (empty) digest - that must not happen silently + logger_name = "keep.providers.base.base_provider" + with caplog.at_level(logging.WARNING, logger=logger_name): + first = BaseProvider.get_alert_fingerprint( + _dedup_test_alert(name="alert one"), ["nonexistent_field"] + ) + second = BaseProvider.get_alert_fingerprint( + _dedup_test_alert(name="alert two"), ["nonexistent_field"] + ) + + assert first == second + assert "all alerts will share the same fingerprint" in caplog.text + + # a field that does resolve produces a distinct fingerprint and no warning + caplog.clear() + with caplog.at_level(logging.WARNING, logger=logger_name): + resolved = BaseProvider.get_alert_fingerprint( + _dedup_test_alert(service="api"), ["service"] + ) + + assert resolved != first + assert "all alerts will share the same fingerprint" not in caplog.text + + +def test_custom_rule_lookup_normalizes_missing_provider_type(db_session): + # rules for provider-less alerts are stored under provider_type="keep", but + # callers may pass None - both must resolve the same row + rule = _add_keep_dedup_rule(db_session) + + via_none = get_custom_deduplication_rule(SINGLE_TENANT_UUID, None, None) + via_keep = get_custom_deduplication_rule(SINGLE_TENANT_UUID, None, "keep") + + assert via_none is not None + assert via_none.id == rule.id == via_keep.id + + +def test_custom_rule_lookup_respects_disabled_flag(db_session, monkeypatch): + # the kill switch must apply to every caller, not just get_deduplication_rules + _add_keep_dedup_rule(db_session) + + monkeypatch.setattr(db_module, "KEEP_CUSTOM_DEDUPLICATION_ENABLED", False) + assert get_custom_deduplication_rule(SINGLE_TENANT_UUID, None, None) is None + + monkeypatch.setattr(db_module, "KEEP_CUSTOM_DEDUPLICATION_ENABLED", True) + assert get_custom_deduplication_rule(SINGLE_TENANT_UUID, None, None) is not None + + +@pytest.mark.timeout(30) +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + }, + ], + indirect=True, +) +def test_custom_rule_applies_to_provider_less_alerts(db_session, client, test_app): + # alerts pushed to the generic /alerts/event endpoint arrive already parsed as + # an AlertDto and never go through a provider's _format_alert - the catch-all + # deduplication rule must still be applied to them + db_session.exec(text("DELETE FROM alertdeduplicationrule")) + rule = AlertDeduplicationRule( + name="catch all rule", + description="test", + tenant_id=SINGLE_TENANT_UUID, + provider_type="keep", + provider_id=None, + fingerprint_fields=["service"], + full_deduplication=False, + ignore_fields=[], + last_updated_by="test", + created_by="test", + ) + db_session.add(rule) + db_session.commit() + + # same service, different names: without the custom rule each alert would be + # fingerprinted as sha256(name) and the two would never be correlated + for name in ["first alert", "second alert"]: + client.post( + "/alerts/event", + json={"name": name, "service": "billing-api", "source": ["nagios"]}, + headers={"x-api-key": "some-api-key"}, + ) + time.sleep(0.1) + + wait_for_alerts(client, 1) + + alerts = client.get("/alerts", headers={"x-api-key": "some-api-key"}).json() + assert len(alerts) == 1 + # the fingerprint is derived from the rule's field, not from the alert name + assert alerts[0]["fingerprint"] == hashlib.sha256(b"billing-api").hexdigest() From f08e3c434c68f7127308574f29739135e919e228 Mon Sep 17 00:00:00 2001 From: Veydop <37590446+Veydop@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:09:23 +0530 Subject: [PATCH 2/3] fix: ingest array payloads posted to provider endpoints (#6798) Signed-off-by: Veydop <37590446+Veydop@users.noreply.github.com> --- keep/api/tasks/process_event_task.py | 27 ++++++-- tests/deduplication/test_deduplications.py | 74 ++++++++++++++++++++++ 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index 501455688b..5816a989ed 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -721,18 +721,31 @@ def process_event( if isinstance(event, list): event_list = [] + already_formatted = [] for event_item in event: if not isinstance(event_item, AlertDto): - event_list.append( - provider_class.format_alert( - tenant_id=tenant_id, - event=event_item, - provider_id=provider_id, - provider_type=provider_type, - ) + # format_alert returns a list, so extend rather than + # append - appending nests it and breaks downstream + formatted_event_item = provider_class.format_alert( + tenant_id=tenant_id, + event=event_item, + provider_id=provider_id, + provider_type=provider_type, ) + if formatted_event_item: + event_list.extend(formatted_event_item) else: + # already parsed, so it skipped format_alert and with it + # the custom deduplication rule - collect and apply once + already_formatted.append(event_item) event_list.append(event_item) + if already_formatted: + provider_class.apply_custom_deduplication_rule( + already_formatted, + tenant_id=tenant_id, + provider_id=provider_id, + provider_type=provider_type, + ) event = event_list else: event = provider_class.format_alert( diff --git a/tests/deduplication/test_deduplications.py b/tests/deduplication/test_deduplications.py index eaa89083d8..b81b9d6f39 100644 --- a/tests/deduplication/test_deduplications.py +++ b/tests/deduplication/test_deduplications.py @@ -1156,3 +1156,77 @@ def test_custom_rule_applies_to_provider_less_alerts(db_session, client, test_ap assert len(alerts) == 1 # the fingerprint is derived from the rule's field, not from the alert name assert alerts[0]["fingerprint"] == hashlib.sha256(b"billing-api").hexdigest() + + +@pytest.mark.timeout(30) +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + }, + ], + indirect=True, +) +def test_array_payload_to_typed_endpoint_is_ingested(db_session, client, test_app): + # format_alert returns a list, so appending its result nested it and every + # alert in an array payload was dropped after a 202 response + provider = ProvidersFactory.get_provider_class("datadog") + alert_1 = provider.simulate_alert() + alert_2 = provider.simulate_alert() + while alert_2.get("monitor_id") == alert_1.get("monitor_id"): + alert_2 = provider.simulate_alert() + + response = client.post( + "/alerts/event/datadog", + json=[alert_1, alert_2], + headers={"x-api-key": "some-api-key"}, + ) + assert response.status_code == 202 + + wait_for_alerts(client, 2) + + +@pytest.mark.timeout(30) +@pytest.mark.parametrize( + "test_app", + [ + { + "AUTH_TYPE": "NOAUTH", + }, + ], + indirect=True, +) +def test_custom_rule_applies_to_array_of_parsed_alerts(db_session, client, test_app): + # an array posted to the generic endpoint arrives as a list of already-parsed + # AlertDto, which skips _format_alert - the catch-all rule must still apply + db_session.exec(text("DELETE FROM alertdeduplicationrule")) + rule = AlertDeduplicationRule( + name="catch all rule", + description="test", + tenant_id=SINGLE_TENANT_UUID, + provider_type="keep", + provider_id=None, + fingerprint_fields=["service"], + full_deduplication=False, + ignore_fields=[], + last_updated_by="test", + created_by="test", + ) + db_session.add(rule) + db_session.commit() + + client.post( + "/alerts/event", + json=[ + {"name": "first alert", "service": "billing-api", "source": ["nagios"]}, + {"name": "second alert", "service": "billing-api", "source": ["nagios"]}, + ], + headers={"x-api-key": "some-api-key"}, + ) + + wait_for_alerts(client, 1) + + alerts = client.get("/alerts", headers={"x-api-key": "some-api-key"}).json() + assert len(alerts) == 1 + assert alerts[0]["fingerprint"] == hashlib.sha256(b"billing-api").hexdigest() From 118b2dc0c7a45f8a22b6317b983cdba6f5b54b5e Mon Sep 17 00:00:00 2001 From: Shahar Glazner Date: Wed, 9 Sep 2026 10:44:41 +0300 Subject: [PATCH 3/3] chore: Bump version from 0.54.2 to 0.54.3 (#6800) Co-authored-by: Cursor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9dfb05575d..22fbd923cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep" -version = "0.54.2" +version = "0.54.3" description = "Alerting. for developers, by developers." authors = ["Keep Alerting LTD"] packages = [{include = "keep"}]