From 6772caaeb7204a29942e83486c5508e0a2de0f25 Mon Sep 17 00:00:00 2001 From: Test Date: Mon, 16 Nov 2026 21:38:00 -0500 Subject: [PATCH] fix(monitor): normalize per-occurrence Azure tokens out of failure signatures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #3592. Both Azure providers built a failure signature from the exception message with GUIDs, timestamps and numbers replaced, but several per-occurrence tokens survived, so every occurrence of one failure got its own fingerprint and its own issue (339 issues in one Backend run): - the GUID pattern was `\b`-anchored, and `_` is a word character, so the Service Bus `TrackingId:_B4` GUID never matched; its `_B` partition suffix varied too; - `SystemTracker` carries the subscription and partition, so MessageLockLost split per subscription; - Azure Storage errors echo an HTTP `Date:` header (`Fri, 14 Nov 2026 ...`); - the Functions host wrapper exceptions (`Grpc.Core.RpcException`, `Rpc.RpcException`) embed the inner exception's whole stack, so each call path was its own signature. The normalization moves to a shared private module, `specfuse/monitor/providers/_azure_normalize.py` (the `_azure_auth.py` pattern), so the two providers cannot drift again. It strips embedded stack frames, replaces TrackingId/SystemTracker/Reference/LockToken values, matches GUIDs on hex boundaries, and replaces HTTP dates, on top of the existing timestamp and number rules. Replaying the 339 real signatures through the new rules leaves 95 distinct; the lock-lost family (224 issues) becomes 3. Existing findings whose signature changes are filed once more under the new signature. Tests: - tests/test_azure_message_normalization.py — for both the error-logs and dlq adapters: two occurrences of a lock-lost, a wrapper-with-stack, and a blob-lease failure each share a fingerprint; distinct failures (including prose containing "at the gateway") stay apart; a GUID followed by `_B4`, `)`, `.` or nothing is normalized. Co-Authored-By: Claude Opus 5.5 --- ...pecfuse-monitor-run-azure-failure-signa.md | 5 + .../monitor/providers/_azure_normalize.py | 61 +++++++ .../monitor/providers/azure_app_insights.py | 24 +-- .../monitor/providers/azure_service_bus.py | 21 +-- tests/test_azure_message_normalization.py | 172 ++++++++++++++++++ 5 files changed, 243 insertions(+), 40 deletions(-) create mode 100644 changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md create mode 100644 specfuse/monitor/providers/_azure_normalize.py create mode 100644 tests/test_azure_message_normalization.py diff --git a/changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md b/changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md new file mode 100644 index 00000000..53a8b296 --- /dev/null +++ b/changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md @@ -0,0 +1,5 @@ +--- +class: fixed +trace: #3592 +--- +`specfuse monitor run`: Azure failure signatures no longer carry per-occurrence tokens — a GUID followed by `_` (the Service Bus `TrackingId:_B4` shape), `TrackingId`/`SystemTracker`/`Reference`/`LockToken` values, HTTP `Date:` headers, and stack traces embedded by Functions host wrapper exceptions are normalized away, so repeat occurrences of one failure land on one issue instead of one each. Existing findings whose signature changes are filed once more under the new signature. diff --git a/specfuse/monitor/providers/_azure_normalize.py b/specfuse/monitor/providers/_azure_normalize.py new file mode 100644 index 00000000..89c4b4a0 --- /dev/null +++ b/specfuse/monitor/providers/_azure_normalize.py @@ -0,0 +1,61 @@ +# +# Copyright 2026 Specfuse contributors +# Licensed under the Apache License, Version 2.0. See LICENSE. +# +"""Message normalization shared by both Azure provider modules. + +A failure's signature is its exception type plus its message with every +per-occurrence token replaced, so two occurrences of one failure fingerprint +the same and land on one issue. Anything left in that varies per occurrence +files a new issue per occurrence: #3592 filed 339 issues in one run for a +handful of failures. +""" + +from __future__ import annotations + +import re + +__all__ = ("normalize_message",) + +# A .NET stack frame (`at Namespace.Type.Method(`). The Functions host wrapper +# exceptions (`Grpc.Core.RpcException`, `Rpc.RpcException`) embed the inner +# exception's whole stack in their message, so each call path would otherwise +# be its own signature. The top in-app frame is read separately (`Where`). +_STACK_RE = re.compile(r"\s+at [A-Za-z_][\w.`<>+\[\],]*\(.*", re.DOTALL) + +# Service Bus per-message tokens. `TrackingId` carries a `_B` partition +# suffix; `SystemTracker` names the entity (`ns:topic:t~|`), +# and MessageLockLost on many subscriptions is one failure, not one per +# subscription — the subscription still shows in the issue body. +_AZURE_TOKEN_RE = re.compile( + r"\b(TrackingId|SystemTracker|Reference|LockToken):\s*[^,\s\"\\]+" +) +# Hex boundaries, not `\b`: `_` is a word character, so `_B4` never +# matched a `\b`-anchored pattern. +_GUID_RE = re.compile( + r"(? str: + """Replace every per-occurrence token in *text* — embedded stack frames, + Service Bus tracking tokens, GUIDs, timestamps, and bare numbers — so two + occurrences of one failure collapse to one signature.""" + text = _STACK_RE.sub(" ", text) + text = _AZURE_TOKEN_RE.sub(lambda m: f"{m.group(1)}:", text) + text = _GUID_RE.sub("", text) + text = _HTTP_DATE_RE.sub("", text) + text = _TIMESTAMP_RE.sub("", text) + text = _NUMBER_RE.sub("", text) + return " ".join(text.split()) diff --git a/specfuse/monitor/providers/azure_app_insights.py b/specfuse/monitor/providers/azure_app_insights.py index 183416b2..1d3aa789 100644 --- a/specfuse/monitor/providers/azure_app_insights.py +++ b/specfuse/monitor/providers/azure_app_insights.py @@ -41,27 +41,9 @@ # Credential construction is shared with the other Azure provider and looked up # reflectively by the core as `build_credential` (#302). from specfuse.monitor.providers._azure_auth import build_credential # noqa: F401 +from specfuse.monitor.providers._azure_normalize import normalize_message from specfuse.monitor.schedule import most_recent_firing -_GUID_RE = re.compile( - r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" -) -_TIMESTAMP_RE = re.compile( - r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b" -) -_NUMBER_RE = re.compile(r"\b\d+\b") - - -def _normalize_message(text: str) -> str: - """Strip GUIDs, timestamps, and bare numbers so two occurrences of the - same underlying failure — differing only in operation/request IDs and - timestamps — collapse to one signature.""" - text = _GUID_RE.sub("", text) - text = _TIMESTAMP_RE.sub("", text) - text = _NUMBER_RE.sub("", text) - return " ".join(text.split()) - - class AppInsightsTransport(Protocol): """The single operation these adapters need from a transport: run a KQL query and hand back the result rows.""" @@ -265,14 +247,14 @@ def fetch_failures(self) -> Iterator[FailureArtifact]: @staticmethod def _signature(row: Mapping[str, object]) -> str: exception_type = str(row.get("type") or "UnknownException") - return f"{exception_type}:{_normalize_message(str(row.get('outerMessage') or ''))}" + return f"{exception_type}:{normalize_message(str(row.get('outerMessage') or ''))}" def _build_artifact( self, row: Mapping[str, object], sample_ids: Sequence[str] = () ) -> FailureArtifact: exception_type = str(row.get("type") or "UnknownException") raw_message = str(row.get("outerMessage") or "") - failure_signature = f"{exception_type}:{_normalize_message(raw_message)}" + failure_signature = f"{exception_type}:{normalize_message(raw_message)}" artifact = FailureArtifact( component=self._component, check_type="error-logs", diff --git a/specfuse/monitor/providers/azure_service_bus.py b/specfuse/monitor/providers/azure_service_bus.py index 10eccdd7..150f969e 100644 --- a/specfuse/monitor/providers/azure_service_bus.py +++ b/specfuse/monitor/providers/azure_service_bus.py @@ -25,7 +25,6 @@ from __future__ import annotations import json -import re from dataclasses import dataclass, replace from datetime import datetime from typing import Iterable, Iterator, List, Mapping, Optional, Protocol, Sequence @@ -37,23 +36,7 @@ # Credential construction is shared with the other Azure provider and looked up # reflectively by the core as `build_credential` (#302). from specfuse.monitor.providers._azure_auth import build_credential # noqa: F401 - -_GUID_RE = re.compile( - r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" -) -_TIMESTAMP_RE = re.compile( - r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b" -) -_NUMBER_RE = re.compile(r"\b\d+\b") - - -def _normalize_message(text: str) -> str: - """Strip message IDs, timestamps, GUIDs, and sequence numbers so two - occurrences of the same poison message collapse to one signature.""" - text = _GUID_RE.sub("", text) - text = _TIMESTAMP_RE.sub("", text) - text = _NUMBER_RE.sub("", text) - return " ".join(text.split()) +from specfuse.monitor.providers._azure_normalize import normalize_message def _exception_type_and_message(message: object) -> tuple[str, str]: @@ -273,7 +256,7 @@ def _build_artifact( ) -> FailureArtifact: failure_class = getattr(message, "dead_letter_reason", None) or "Unknown" exception_type, raw_message = _exception_type_and_message(message) - failure_signature = f"{exception_type}:{_normalize_message(raw_message)}" + failure_signature = f"{exception_type}:{normalize_message(raw_message)}" observed_text = getattr(message, "dead_letter_error_description", None) or "" raw_id = getattr(message, "message_id", None) message_id = str(raw_id) if raw_id else None diff --git a/tests/test_azure_message_normalization.py b/tests/test_azure_message_normalization.py new file mode 100644 index 00000000..aefe33c1 --- /dev/null +++ b/tests/test_azure_message_normalization.py @@ -0,0 +1,172 @@ +# Copyright 2026 Specfuse Contributors +# Licensed under the Apache License, Version 2.0. See LICENSE. +"""#3592: per-message Azure tokens must not survive message normalization. + +One `specfuse monitor run` filed 339 issues for what was a handful of failures: +the Service Bus `TrackingId` GUID carries a `_B` partition suffix, so the +`\\b`-anchored GUID pattern never matched it; the `SystemTracker` subscription, +an HTTP `Date:` header, and stack traces embedded by the Functions host wrapper +exceptions varied per occurrence too. Each case below pairs two occurrences of +one failure that must share a signature, and every pair of distinct failures +must still stay apart. + +Both Azure providers normalize the same way, so each case runs through the +error-logs adapter (App Insights) and the dlq adapter (Service Bus). +""" + +from __future__ import annotations + +import unittest +from dataclasses import dataclass + +from specfuse.monitor.fingerprint import fingerprint_artifact +from specfuse.monitor.providers.azure_app_insights import ErrorLogsAdapter +from specfuse.monitor.providers.azure_service_bus import ServiceBusDlqAdapter + +_LOCK_LOST = ( + "The lock supplied is invalid. Either the lock expired, or the message has " + "already been removed from the queue. For more information please see " + "https://aka.ms/ServiceBusExceptions . Reference:{ref}, TrackingId:{track}, " + "SystemTracker:sbns-acme-prd:topic:sbt-acme~{part}|{sub}, Timestamp:{ts} " + "(MessageLockLost). For troubleshooting information, see " + "https://aka.ms/azsdk/net/servicebus/exceptions/troubleshoot." +) + +LOCK_LOST_A = _LOCK_LOST.format( + ref="1b9d4c1e-7a3f-4e0b-9c55-2f6a8d7e1a10", + track="4311145c-fcf8-4a21-9bdb-95c15255c504_B4", + part="16383", + sub="on-order-created", + ts="2026-11-16T10:00:00", +) +LOCK_LOST_B = _LOCK_LOST.format( + ref="9e2a7b30-55c1-4d8e-8f0a-6b1c3d2e4f50", + track="68128c21-0bb5-44ea-8e81-77822ab6f35b_B11", + part="12287", + sub="on-invoice-paid-sync-ledger", + ts="2026-11-16T11:42:17", +) + +_RPC = ( + 'Status(StatusCode="Unknown", Detail="Azure.Messaging.ServiceBus.' + "ServiceBusException: The lock supplied is invalid. (MessageLockLost)\n" + " at Azure.Messaging.ServiceBus.Amqp.AmqpReceiver.{receiver}(Guid lockToken, " + "TimeSpan timeout)\n" + " at Acme.Functions.{function}.Run(ServiceBusReceivedMessage message) in " + '/src/Acme.Functions/{function}.cs:line {line}")' +) +RPC_A = _RPC.format(receiver="CompleteInternalAsync", function="Orders.OrderCreatedTrigger", line=42) +RPC_B = _RPC.format(receiver="AbandonInternalAsync", function="Billing.InvoicePaidTrigger", line=117) + +_LEASE = ( + "The lease ID specified did not match the lease ID for the blob. " + "RequestId:{req} Time:{time} Status: 412 (The lease ID specified did not match " + "the lease ID for the blob.) ErrorCode: LeaseIdMismatchWithLeaseOperation " + "Headers: Date: {date} Server: Windows-Azure-Blob/1.0" +) +LEASE_A = _LEASE.format( + req="0f8e7d6c-5b4a-4392-8170-6f5e4d3c2b1a", + time="2026-11-14T08:01:02.1234567Z", + date="Fri, 14 Nov 2026 08:01:02 GMT", +) +LEASE_B = _LEASE.format( + req="a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + time="2026-11-15T19:30:45.7654321Z", + date="Sat, 15 Nov 2026 19:30:45 GMT", +) + +# Same failure in each pair; different failures across pairs. +SAME_FAILURE_PAIRS = { + "service-bus-lock-lost": (LOCK_LOST_A, LOCK_LOST_B), + "functions-host-wrapper-with-stack": (RPC_A, RPC_B), + "blob-lease-with-http-date": (LEASE_A, LEASE_B), +} + + +class _Transport: + def __init__(self, messages): + self._messages = messages + + def run_query(self, query): + return [ + {"type": "Acme.FailureException", "outerMessage": m, "operation_Id": f"op{i}"} + for i, m in enumerate(self._messages) + ] + + +@dataclass(frozen=True) +class _DeadLetter: + message_id: str + dead_letter_reason: str + dead_letter_error_description: str + exception_type: str = "Acme.FailureException" + + +class _DlqTransport: + def __init__(self, messages): + self._messages = messages + + def peek_dead_letter_messages(self, *, subscription, max_message_count): + return [ + _DeadLetter(f"m{i}", "MaxDeliveryCountExceeded", m) + for i, m in enumerate(self._messages) + ] + + +def _error_logs_fingerprints(messages): + adapter = ErrorLogsAdapter( + component="acme-worker", + environment={"telemetry": {"provider": "app-insights"}}, + transport=_Transport(messages), + ) + return [fingerprint_artifact(a) for a in adapter.fetch_failures()] + + +def _dlq_fingerprints(messages): + adapter = ServiceBusDlqAdapter( + component="acme-worker", + transport=_DlqTransport(messages), + targets=[{"subscription": "acme-sub", "function": "AcmeFunction"}], + ) + return [fingerprint_artifact(a) for a in adapter.fetch_failures()] + + +_PROVIDERS = {"error-logs": _error_logs_fingerprints, "dlq": _dlq_fingerprints} + + +class SameFailureCollapsesTests(unittest.TestCase): + def test_two_occurrences_of_one_failure_share_a_fingerprint(self): + for provider, fingerprints in _PROVIDERS.items(): + for case, pair in SAME_FAILURE_PAIRS.items(): + with self.subTest(provider=provider, case=case): + first, second = fingerprints(list(pair)) + self.assertEqual(first, second) + + +class DistinctFailuresStayApartTests(unittest.TestCase): + def test_different_failures_keep_different_fingerprints(self): + messages = [pair[0] for pair in SAME_FAILURE_PAIRS.values()] + [ + "Connection to the database failed at the gateway.", + "Connection to the cache failed at the gateway.", + ] + for provider, fingerprints in _PROVIDERS.items(): + with self.subTest(provider=provider): + prints = fingerprints(messages) + self.assertEqual(len(set(prints)), len(messages)) + + +class GuidBoundaryTests(unittest.TestCase): + def test_guid_is_normalized_whatever_follows_it(self): + guid_a = "4311145c-fcf8-4a21-9bdb-95c15255c504" + guid_b = "68128c21-0bb5-44ea-8e81-77822ab6f35b" + for suffix in ("_B4", ")", ".", ""): + for provider, fingerprints in _PROVIDERS.items(): + with self.subTest(provider=provider, suffix=suffix): + first, second = fingerprints( + [f"Entity id={guid_a}{suffix} failed", f"Entity id={guid_b}{suffix} failed"] + ) + self.assertEqual(first, second) + + +if __name__ == "__main__": + unittest.main()