diff --git a/changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md b/changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md new file mode 100644 index 00000000..53a8b296 --- /dev/null +++ b/changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md @@ -0,0 +1,5 @@ +--- +class: fixed +trace: #3592 +--- +`specfuse monitor run`: Azure failure signatures no longer carry per-occurrence tokens — a GUID followed by `_` (the Service Bus `TrackingId:_B4` shape), `TrackingId`/`SystemTracker`/`Reference`/`LockToken` values, HTTP `Date:` headers, and stack traces embedded by Functions host wrapper exceptions are normalized away, so repeat occurrences of one failure land on one issue instead of one each. Existing findings whose signature changes are filed once more under the new signature. diff --git a/specfuse/monitor/providers/_azure_normalize.py b/specfuse/monitor/providers/_azure_normalize.py new file mode 100644 index 00000000..89c4b4a0 --- /dev/null +++ b/specfuse/monitor/providers/_azure_normalize.py @@ -0,0 +1,61 @@ +# +# Copyright 2026 Specfuse contributors +# Licensed under the Apache License, Version 2.0. See LICENSE. +# +"""Message normalization shared by both Azure provider modules. + +A failure's signature is its exception type plus its message with every +per-occurrence token replaced, so two occurrences of one failure fingerprint +the same and land on one issue. Anything left in that varies per occurrence +files a new issue per occurrence: #3592 filed 339 issues in one run for a +handful of failures. +""" + +from __future__ import annotations + +import re + +__all__ = ("normalize_message",) + +# A .NET stack frame (`at Namespace.Type.Method(`). The Functions host wrapper +# exceptions (`Grpc.Core.RpcException`, `Rpc.RpcException`) embed the inner +# exception's whole stack in their message, so each call path would otherwise +# be its own signature. The top in-app frame is read separately (`Where`). +_STACK_RE = re.compile(r"\s+at [A-Za-z_][\w.`<>+\[\],]*\(.*", re.DOTALL) + +# Service Bus per-message tokens. `TrackingId` carries a `_B` partition +# suffix; `SystemTracker` names the entity (`ns:topic:t~|`), +# and MessageLockLost on many subscriptions is one failure, not one per +# subscription — the subscription still shows in the issue body. +_AZURE_TOKEN_RE = re.compile( + r"\b(TrackingId|SystemTracker|Reference|LockToken):\s*[^,\s\"\\]+" +) +# Hex boundaries, not `\b`: `_` is a word character, so `_B4` never +# matched a `\b`-anchored pattern. +_GUID_RE = re.compile( + r"(? str: + """Replace every per-occurrence token in *text* — embedded stack frames, + Service Bus tracking tokens, GUIDs, timestamps, and bare numbers — so two + occurrences of one failure collapse to one signature.""" + text = _STACK_RE.sub(" ", text) + text = _AZURE_TOKEN_RE.sub(lambda m: f"{m.group(1)}:", text) + text = _GUID_RE.sub("", text) + text = _HTTP_DATE_RE.sub("", text) + text = _TIMESTAMP_RE.sub("", text) + text = _NUMBER_RE.sub("", text) + return " ".join(text.split()) diff --git a/specfuse/monitor/providers/azure_app_insights.py b/specfuse/monitor/providers/azure_app_insights.py index 183416b2..1d3aa789 100644 --- a/specfuse/monitor/providers/azure_app_insights.py +++ b/specfuse/monitor/providers/azure_app_insights.py @@ -41,27 +41,9 @@ # Credential construction is shared with the other Azure provider and looked up # reflectively by the core as `build_credential` (#302). from specfuse.monitor.providers._azure_auth import build_credential # noqa: F401 +from specfuse.monitor.providers._azure_normalize import normalize_message from specfuse.monitor.schedule import most_recent_firing -_GUID_RE = re.compile( - r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" -) -_TIMESTAMP_RE = re.compile( - r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b" -) -_NUMBER_RE = re.compile(r"\b\d+\b") - - -def _normalize_message(text: str) -> str: - """Strip GUIDs, timestamps, and bare numbers so two occurrences of the - same underlying failure — differing only in operation/request IDs and - timestamps — collapse to one signature.""" - text = _GUID_RE.sub("", text) - text = _TIMESTAMP_RE.sub("", text) - text = _NUMBER_RE.sub("", text) - return " ".join(text.split()) - - class AppInsightsTransport(Protocol): """The single operation these adapters need from a transport: run a KQL query and hand back the result rows.""" @@ -265,14 +247,14 @@ def fetch_failures(self) -> Iterator[FailureArtifact]: @staticmethod def _signature(row: Mapping[str, object]) -> str: exception_type = str(row.get("type") or "UnknownException") - return f"{exception_type}:{_normalize_message(str(row.get('outerMessage') or ''))}" + return f"{exception_type}:{normalize_message(str(row.get('outerMessage') or ''))}" def _build_artifact( self, row: Mapping[str, object], sample_ids: Sequence[str] = () ) -> FailureArtifact: exception_type = str(row.get("type") or "UnknownException") raw_message = str(row.get("outerMessage") or "") - failure_signature = f"{exception_type}:{_normalize_message(raw_message)}" + failure_signature = f"{exception_type}:{normalize_message(raw_message)}" artifact = FailureArtifact( component=self._component, check_type="error-logs", diff --git a/specfuse/monitor/providers/azure_service_bus.py b/specfuse/monitor/providers/azure_service_bus.py index 10eccdd7..150f969e 100644 --- a/specfuse/monitor/providers/azure_service_bus.py +++ b/specfuse/monitor/providers/azure_service_bus.py @@ -25,7 +25,6 @@ from __future__ import annotations import json -import re from dataclasses import dataclass, replace from datetime import datetime from typing import Iterable, Iterator, List, Mapping, Optional, Protocol, Sequence @@ -37,23 +36,7 @@ # Credential construction is shared with the other Azure provider and looked up # reflectively by the core as `build_credential` (#302). from specfuse.monitor.providers._azure_auth import build_credential # noqa: F401 - -_GUID_RE = re.compile( - r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" -) -_TIMESTAMP_RE = re.compile( - r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b" -) -_NUMBER_RE = re.compile(r"\b\d+\b") - - -def _normalize_message(text: str) -> str: - """Strip message IDs, timestamps, GUIDs, and sequence numbers so two - occurrences of the same poison message collapse to one signature.""" - text = _GUID_RE.sub("", text) - text = _TIMESTAMP_RE.sub("", text) - text = _NUMBER_RE.sub("", text) - return " ".join(text.split()) +from specfuse.monitor.providers._azure_normalize import normalize_message def _exception_type_and_message(message: object) -> tuple[str, str]: @@ -273,7 +256,7 @@ def _build_artifact( ) -> FailureArtifact: failure_class = getattr(message, "dead_letter_reason", None) or "Unknown" exception_type, raw_message = _exception_type_and_message(message) - failure_signature = f"{exception_type}:{_normalize_message(raw_message)}" + failure_signature = f"{exception_type}:{normalize_message(raw_message)}" observed_text = getattr(message, "dead_letter_error_description", None) or "" raw_id = getattr(message, "message_id", None) message_id = str(raw_id) if raw_id else None diff --git a/tests/test_azure_message_normalization.py b/tests/test_azure_message_normalization.py new file mode 100644 index 00000000..aefe33c1 --- /dev/null +++ b/tests/test_azure_message_normalization.py @@ -0,0 +1,172 @@ +# Copyright 2026 Specfuse Contributors +# Licensed under the Apache License, Version 2.0. See LICENSE. +"""#3592: per-message Azure tokens must not survive message normalization. + +One `specfuse monitor run` filed 339 issues for what was a handful of failures: +the Service Bus `TrackingId` GUID carries a `_B` partition suffix, so the +`\\b`-anchored GUID pattern never matched it; the `SystemTracker` subscription, +an HTTP `Date:` header, and stack traces embedded by the Functions host wrapper +exceptions varied per occurrence too. Each case below pairs two occurrences of +one failure that must share a signature, and every pair of distinct failures +must still stay apart. + +Both Azure providers normalize the same way, so each case runs through the +error-logs adapter (App Insights) and the dlq adapter (Service Bus). +""" + +from __future__ import annotations + +import unittest +from dataclasses import dataclass + +from specfuse.monitor.fingerprint import fingerprint_artifact +from specfuse.monitor.providers.azure_app_insights import ErrorLogsAdapter +from specfuse.monitor.providers.azure_service_bus import ServiceBusDlqAdapter + +_LOCK_LOST = ( + "The lock supplied is invalid. Either the lock expired, or the message has " + "already been removed from the queue. For more information please see " + "https://aka.ms/ServiceBusExceptions . Reference:{ref}, TrackingId:{track}, " + "SystemTracker:sbns-acme-prd:topic:sbt-acme~{part}|{sub}, Timestamp:{ts} " + "(MessageLockLost). For troubleshooting information, see " + "https://aka.ms/azsdk/net/servicebus/exceptions/troubleshoot." +) + +LOCK_LOST_A = _LOCK_LOST.format( + ref="1b9d4c1e-7a3f-4e0b-9c55-2f6a8d7e1a10", + track="4311145c-fcf8-4a21-9bdb-95c15255c504_B4", + part="16383", + sub="on-order-created", + ts="2026-11-16T10:00:00", +) +LOCK_LOST_B = _LOCK_LOST.format( + ref="9e2a7b30-55c1-4d8e-8f0a-6b1c3d2e4f50", + track="68128c21-0bb5-44ea-8e81-77822ab6f35b_B11", + part="12287", + sub="on-invoice-paid-sync-ledger", + ts="2026-11-16T11:42:17", +) + +_RPC = ( + 'Status(StatusCode="Unknown", Detail="Azure.Messaging.ServiceBus.' + "ServiceBusException: The lock supplied is invalid. (MessageLockLost)\n" + " at Azure.Messaging.ServiceBus.Amqp.AmqpReceiver.{receiver}(Guid lockToken, " + "TimeSpan timeout)\n" + " at Acme.Functions.{function}.Run(ServiceBusReceivedMessage message) in " + '/src/Acme.Functions/{function}.cs:line {line}")' +) +RPC_A = _RPC.format(receiver="CompleteInternalAsync", function="Orders.OrderCreatedTrigger", line=42) +RPC_B = _RPC.format(receiver="AbandonInternalAsync", function="Billing.InvoicePaidTrigger", line=117) + +_LEASE = ( + "The lease ID specified did not match the lease ID for the blob. " + "RequestId:{req} Time:{time} Status: 412 (The lease ID specified did not match " + "the lease ID for the blob.) ErrorCode: LeaseIdMismatchWithLeaseOperation " + "Headers: Date: {date} Server: Windows-Azure-Blob/1.0" +) +LEASE_A = _LEASE.format( + req="0f8e7d6c-5b4a-4392-8170-6f5e4d3c2b1a", + time="2026-11-14T08:01:02.1234567Z", + date="Fri, 14 Nov 2026 08:01:02 GMT", +) +LEASE_B = _LEASE.format( + req="a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + time="2026-11-15T19:30:45.7654321Z", + date="Sat, 15 Nov 2026 19:30:45 GMT", +) + +# Same failure in each pair; different failures across pairs. +SAME_FAILURE_PAIRS = { + "service-bus-lock-lost": (LOCK_LOST_A, LOCK_LOST_B), + "functions-host-wrapper-with-stack": (RPC_A, RPC_B), + "blob-lease-with-http-date": (LEASE_A, LEASE_B), +} + + +class _Transport: + def __init__(self, messages): + self._messages = messages + + def run_query(self, query): + return [ + {"type": "Acme.FailureException", "outerMessage": m, "operation_Id": f"op{i}"} + for i, m in enumerate(self._messages) + ] + + +@dataclass(frozen=True) +class _DeadLetter: + message_id: str + dead_letter_reason: str + dead_letter_error_description: str + exception_type: str = "Acme.FailureException" + + +class _DlqTransport: + def __init__(self, messages): + self._messages = messages + + def peek_dead_letter_messages(self, *, subscription, max_message_count): + return [ + _DeadLetter(f"m{i}", "MaxDeliveryCountExceeded", m) + for i, m in enumerate(self._messages) + ] + + +def _error_logs_fingerprints(messages): + adapter = ErrorLogsAdapter( + component="acme-worker", + environment={"telemetry": {"provider": "app-insights"}}, + transport=_Transport(messages), + ) + return [fingerprint_artifact(a) for a in adapter.fetch_failures()] + + +def _dlq_fingerprints(messages): + adapter = ServiceBusDlqAdapter( + component="acme-worker", + transport=_DlqTransport(messages), + targets=[{"subscription": "acme-sub", "function": "AcmeFunction"}], + ) + return [fingerprint_artifact(a) for a in adapter.fetch_failures()] + + +_PROVIDERS = {"error-logs": _error_logs_fingerprints, "dlq": _dlq_fingerprints} + + +class SameFailureCollapsesTests(unittest.TestCase): + def test_two_occurrences_of_one_failure_share_a_fingerprint(self): + for provider, fingerprints in _PROVIDERS.items(): + for case, pair in SAME_FAILURE_PAIRS.items(): + with self.subTest(provider=provider, case=case): + first, second = fingerprints(list(pair)) + self.assertEqual(first, second) + + +class DistinctFailuresStayApartTests(unittest.TestCase): + def test_different_failures_keep_different_fingerprints(self): + messages = [pair[0] for pair in SAME_FAILURE_PAIRS.values()] + [ + "Connection to the database failed at the gateway.", + "Connection to the cache failed at the gateway.", + ] + for provider, fingerprints in _PROVIDERS.items(): + with self.subTest(provider=provider): + prints = fingerprints(messages) + self.assertEqual(len(set(prints)), len(messages)) + + +class GuidBoundaryTests(unittest.TestCase): + def test_guid_is_normalized_whatever_follows_it(self): + guid_a = "4311145c-fcf8-4a21-9bdb-95c15255c504" + guid_b = "68128c21-0bb5-44ea-8e81-77822ab6f35b" + for suffix in ("_B4", ")", ".", ""): + for provider, fingerprints in _PROVIDERS.items(): + with self.subTest(provider=provider, suffix=suffix): + first, second = fingerprints( + [f"Entity id={guid_a}{suffix} failed", f"Entity id={guid_b}{suffix} failed"] + ) + self.assertEqual(first, second) + + +if __name__ == "__main__": + unittest.main()