[pull] main from keephq:main

keep/api/core/db.py

@@ -103,6 +103,9 @@
     "assignee",
 ]
 KEEP_AUDIT_EVENTS_ENABLED = config("KEEP_AUDIT_EVENTS_ENABLED", cast=bool, default=True)
+KEEP_CUSTOM_DEDUPLICATION_ENABLED = config(
+    "KEEP_CUSTOM_DEDUPLICATION_ENABLED", cast=bool, default=True
+)
 
 INTERVAL_WORKFLOWS_RELAUNCH_TIMEOUT = timedelta(minutes=60)
 WORKFLOWS_TIMEOUT = timedelta(minutes=120)
@@ -2584,6 +2587,13 @@ def get_deduplication_rule_by_id(tenant_id, rule_id: str):
 
 
 def get_custom_deduplication_rule(tenant_id, provider_id, provider_type):
+    # check the custom deduplication flag here so every caller behaves the same
+    if not KEEP_CUSTOM_DEDUPLICATION_ENABLED:
+        return None
+    # alerts ingested without a provider are attributed to the "keep" provider.
+    # normalizing here so all callers resolve the same rule row
+    if not provider_type:
+        provider_type = "keep"
     with Session(engine) as session:
         rule = session.exec(
             select(AlertDeduplicationRule)

keep/api/tasks/process_event_task.py

@@ -721,18 +721,31 @@ def process_event(
 
                 if isinstance(event, list):
                     event_list = []
+                    already_formatted = []
                     for event_item in event:
                         if not isinstance(event_item, AlertDto):
-                            event_list.append(
-                                provider_class.format_alert(
-                                    tenant_id=tenant_id,
-                                    event=event_item,
-                                    provider_id=provider_id,
-                                    provider_type=provider_type,
-                                )
+                            # format_alert returns a list, so extend rather than
+                            # append - appending nests it and breaks downstream
+                            formatted_event_item = provider_class.format_alert(
+                                tenant_id=tenant_id,
+                                event=event_item,
+                                provider_id=provider_id,
+                                provider_type=provider_type,
                             )
+                            if formatted_event_item:
+                                event_list.extend(formatted_event_item)
                         else:
+                            # already parsed, so it skipped format_alert and with it
+                            # the custom deduplication rule - collect and apply once
+                            already_formatted.append(event_item)
                             event_list.append(event_item)
+                    if already_formatted:
+                        provider_class.apply_custom_deduplication_rule(
+                            already_formatted,
+                            tenant_id=tenant_id,
+                            provider_id=provider_id,
+                            provider_type=provider_type,
+                        )
                     event = event_list
                 else:
                     event = provider_class.format_alert(
@@ -768,13 +781,30 @@ def process_event(
             if isinstance(event, dict):
                 if not event.get("name"):
                     event["name"] = event.get("id", "unknown alert name")
-                event = [AlertDto(**event)]
+                # format through the "keep" provider instead of building the AlertDto
+                # directly, so that a custom deduplication rule is applied here too
+                event = ProvidersFactory.get_provider_class("keep").format_alert(
+                    tenant_id=tenant_id,
+                    event=event,
+                    provider_id=provider_id,
+                    provider_type=provider_type,
+                )
                 raw_event = [raw_event]
 
             # Prepare the event for the digest
             if isinstance(event, AlertDto):
                 event = [event]
                 raw_event = [raw_event]
+                # an alert that arrives already parsed skips provider formatting, and
+                # with it the custom deduplication rule - apply it explicitly
+                event = ProvidersFactory.get_provider_class(
+                    "keep"
+                ).apply_custom_deduplication_rule(
+                    event,
+                    tenant_id=tenant_id,
+                    provider_id=provider_id,
+                    provider_type=provider_type,
+                )
 
             with tracer.start_as_current_span("process_event_internal_preparation"):
                 __internal_prepartion(event, fingerprint, api_key_name)

keep/providers/base/base_provider.py

@@ -469,13 +469,6 @@ def format_alert(
             )
             return None
         logger.debug("Alert formatted")
-        # after the provider calculated the default fingerprint
-        #   check if there is a custom deduplication rule and apply
-        custom_deduplication_rule = get_custom_deduplication_rule(
-            tenant_id=tenant_id,
-            provider_id=provider_id,
-            provider_type=provider_type,
-        )
 
         if not isinstance(formatted_alert, list):
             formatted_alert.providerId = provider_id
@@ -487,12 +480,50 @@ def format_alert(
                 alert.providerId = provider_id
                 alert.providerType = provider_type
 
-        # if there is no custom deduplication rule, return the formatted alert
+        # after the provider calculated the default fingerprint
+        #   check if there is a custom deduplication rule and apply
+        return cls.apply_custom_deduplication_rule(
+            formatted_alert,
+            tenant_id=tenant_id,
+            provider_id=provider_id,
+            provider_type=provider_type,
+        )
+
+    @classmethod
+    def apply_custom_deduplication_rule(
+        cls,
+        alerts: list[AlertDto],
+        tenant_id: str,
+        provider_id: str | None,
+        provider_type: str | None,
+    ) -> list[AlertDto]:
+        """
+        Override the fingerprint of already-formatted alerts with a custom deduplication rule.
+
+        Alerts that reach Keep already parsed as AlertDto never go through
+        _format_alert, so this has to be callable on its own - otherwise a configured
+        rule is silently ignored for them.
+
+        Args:
+            alerts (list[AlertDto]): The alerts to apply the rule to, in place.
+            tenant_id (str): The tenant id.
+            provider_id (str | None): The provider id, if any.
+            provider_type (str | None): The provider type, if any.
+
+        Returns:
+            list[AlertDto]: The same alerts, with fingerprints overridden if a rule exists.
+        """
+        logger = logging.getLogger(__name__)
+        custom_deduplication_rule = get_custom_deduplication_rule(
+            tenant_id=tenant_id,
+            provider_id=provider_id,
+            provider_type=provider_type,
+        )
+        # if there is no custom deduplication rule, keep the provider's fingerprint
         if not custom_deduplication_rule:
-            return formatted_alert
-        # if there is a custom deduplication rule, apply it
-        # apply the custom deduplication rule to calculate the fingerprint
-        for alert in formatted_alert:
+            return alerts
+
+        for alert in alerts:
             logger.info(
                 "Applying custom deduplication rule",
                 extra={
@@ -504,7 +535,7 @@ def format_alert(
             alert.fingerprint = cls.get_alert_fingerprint(
                 alert, custom_deduplication_rule.fingerprint_fields
             )
-        return formatted_alert
+        return alerts
 
     @staticmethod
     def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str:
@@ -518,10 +549,12 @@ def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str
         Returns:
             str: hexdigest of the fingerprint or the event.name if no fingerprint_fields were given.
         """
+        logger = logging.getLogger(__name__)
         if not fingerprint_fields:
             return alert.name
         fingerprint = hashlib.sha256()
         event_dict = alert.dict()
+        matched_fields = []
         for fingerprint_field in fingerprint_fields:
             keys = fingerprint_field.split(".")
             fingerprint_field_value = event_dict
@@ -535,6 +568,16 @@ def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str
                 fingerprint_field_value = json.dumps(fingerprint_field_value)
             if fingerprint_field_value is not None:
                 fingerprint.update(str(fingerprint_field_value).encode())
+                matched_fields.append(fingerprint_field)
+        if not matched_fields:
+            logger.warning(
+                "None of the fingerprint fields were found on the alert - "
+                "all alerts will share the same fingerprint",
+                extra={
+                    "fingerprint_fields": fingerprint_fields,
+                    "alert_name": alert.name,
+                },
+            )
         return fingerprint.hexdigest()
 
     def get_alerts_configuration(self, alert_id: Optional[str] = None):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "keep"
-version = "0.54.2"
+version = "0.54.3"
 description = "Alerting. for developers, by developers."
 authors = ["Keep Alerting LTD"]
 packages = [{include = "keep"}]