Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changelog.d/3592-specfuse-monitor-run-azure-failure-signa.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
class: fixed
trace: #3592
---
`specfuse monitor run`: Azure failure signatures no longer carry per-occurrence tokens — a GUID followed by `_` (the Service Bus `TrackingId:<guid>_B4` shape), `TrackingId`/`SystemTracker`/`Reference`/`LockToken` values, HTTP `Date:` headers, and stack traces embedded by Functions host wrapper exceptions are normalized away, so repeat occurrences of one failure land on one issue instead of one each. Existing findings whose signature changes are filed once more under the new signature.
61 changes: 61 additions & 0 deletions specfuse/monitor/providers/_azure_normalize.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#
# Copyright 2026 Specfuse contributors
# Licensed under the Apache License, Version 2.0. See LICENSE.
#
"""Message normalization shared by both Azure provider modules.

A failure's signature is its exception type plus its message with every
per-occurrence token replaced, so two occurrences of one failure fingerprint
the same and land on one issue. Anything left in that varies per occurrence
files a new issue per occurrence: #3592 filed 339 issues in one run for a
handful of failures.
"""

from __future__ import annotations

import re

__all__ = ("normalize_message",)

# A .NET stack frame (`at Namespace.Type.Method(`). The Functions host wrapper
# exceptions (`Grpc.Core.RpcException`, `Rpc.RpcException`) embed the inner
# exception's whole stack in their message, so each call path would otherwise
# be its own signature. The top in-app frame is read separately (`Where`).
_STACK_RE = re.compile(r"\s+at [A-Za-z_][\w.`<>+\[\],]*\(.*", re.DOTALL)

# Service Bus per-message tokens. `TrackingId` carries a `_B<n>` partition
# suffix; `SystemTracker` names the entity (`ns:topic:t~<partition>|<sub>`),
# and MessageLockLost on many subscriptions is one failure, not one per
# subscription — the subscription still shows in the issue body.
_AZURE_TOKEN_RE = re.compile(
r"\b(TrackingId|SystemTracker|Reference|LockToken):\s*[^,\s\"\\]+"
)
# Hex boundaries, not `\b`: `_` is a word character, so `<guid>_B4` never
# matched a `\b`-anchored pattern.
_GUID_RE = re.compile(
r"(?<![0-9a-fA-F])[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(?![0-9a-fA-F])"
)
_TIMESTAMP_RE = re.compile(
r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b"
)
# An HTTP `Date:` header value (RFC 1123), as Azure Storage errors echo it.
_HTTP_DATE_RE = re.compile(
r"\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{1,2} "
r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} "
r"\d{2}:\d{2}:\d{2} GMT\b"
)
_NUMBER_RE = re.compile(r"\b\d+\b")


def normalize_message(text: str) -> str:
"""Replace every per-occurrence token in *text* — embedded stack frames,
Service Bus tracking tokens, GUIDs, timestamps, and bare numbers — so two
occurrences of one failure collapse to one signature."""
text = _STACK_RE.sub(" <stack>", text)
text = _AZURE_TOKEN_RE.sub(lambda m: f"{m.group(1)}:<id>", text)
text = _GUID_RE.sub("<guid>", text)
text = _HTTP_DATE_RE.sub("<timestamp>", text)
text = _TIMESTAMP_RE.sub("<timestamp>", text)
text = _NUMBER_RE.sub("<num>", text)
return " ".join(text.split())
24 changes: 3 additions & 21 deletions specfuse/monitor/providers/azure_app_insights.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,27 +41,9 @@
# Credential construction is shared with the other Azure provider and looked up
# reflectively by the core as `build_credential` (#302).
from specfuse.monitor.providers._azure_auth import build_credential # noqa: F401
from specfuse.monitor.providers._azure_normalize import normalize_message
from specfuse.monitor.schedule import most_recent_firing

_GUID_RE = re.compile(
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
)
_TIMESTAMP_RE = re.compile(
r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b"
)
_NUMBER_RE = re.compile(r"\b\d+\b")


def _normalize_message(text: str) -> str:
"""Strip GUIDs, timestamps, and bare numbers so two occurrences of the
same underlying failure — differing only in operation/request IDs and
timestamps — collapse to one signature."""
text = _GUID_RE.sub("<guid>", text)
text = _TIMESTAMP_RE.sub("<timestamp>", text)
text = _NUMBER_RE.sub("<num>", text)
return " ".join(text.split())


class AppInsightsTransport(Protocol):
"""The single operation these adapters need from a transport: run a KQL
query and hand back the result rows."""
Expand Down Expand Up @@ -265,14 +247,14 @@ def fetch_failures(self) -> Iterator[FailureArtifact]:
@staticmethod
def _signature(row: Mapping[str, object]) -> str:
exception_type = str(row.get("type") or "UnknownException")
return f"{exception_type}:{_normalize_message(str(row.get('outerMessage') or ''))}"
return f"{exception_type}:{normalize_message(str(row.get('outerMessage') or ''))}"

def _build_artifact(
self, row: Mapping[str, object], sample_ids: Sequence[str] = ()
) -> FailureArtifact:
exception_type = str(row.get("type") or "UnknownException")
raw_message = str(row.get("outerMessage") or "")
failure_signature = f"{exception_type}:{_normalize_message(raw_message)}"
failure_signature = f"{exception_type}:{normalize_message(raw_message)}"
artifact = FailureArtifact(
component=self._component,
check_type="error-logs",
Expand Down
21 changes: 2 additions & 19 deletions specfuse/monitor/providers/azure_service_bus.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
from __future__ import annotations

import json
import re
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Iterable, Iterator, List, Mapping, Optional, Protocol, Sequence
Expand All @@ -37,23 +36,7 @@
# Credential construction is shared with the other Azure provider and looked up
# reflectively by the core as `build_credential` (#302).
from specfuse.monitor.providers._azure_auth import build_credential # noqa: F401

_GUID_RE = re.compile(
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
)
_TIMESTAMP_RE = re.compile(
r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\b"
)
_NUMBER_RE = re.compile(r"\b\d+\b")


def _normalize_message(text: str) -> str:
"""Strip message IDs, timestamps, GUIDs, and sequence numbers so two
occurrences of the same poison message collapse to one signature."""
text = _GUID_RE.sub("<guid>", text)
text = _TIMESTAMP_RE.sub("<timestamp>", text)
text = _NUMBER_RE.sub("<num>", text)
return " ".join(text.split())
from specfuse.monitor.providers._azure_normalize import normalize_message


def _exception_type_and_message(message: object) -> tuple[str, str]:
Expand Down Expand Up @@ -273,7 +256,7 @@ def _build_artifact(
) -> FailureArtifact:
failure_class = getattr(message, "dead_letter_reason", None) or "Unknown"
exception_type, raw_message = _exception_type_and_message(message)
failure_signature = f"{exception_type}:{_normalize_message(raw_message)}"
failure_signature = f"{exception_type}:{normalize_message(raw_message)}"
observed_text = getattr(message, "dead_letter_error_description", None) or ""
raw_id = getattr(message, "message_id", None)
message_id = str(raw_id) if raw_id else None
Expand Down
172 changes: 172 additions & 0 deletions tests/test_azure_message_normalization.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
# Copyright 2026 Specfuse Contributors
# Licensed under the Apache License, Version 2.0. See LICENSE.
"""#3592: per-message Azure tokens must not survive message normalization.

One `specfuse monitor run` filed 339 issues for what was a handful of failures:
the Service Bus `TrackingId` GUID carries a `_B<n>` partition suffix, so the
`\\b`-anchored GUID pattern never matched it; the `SystemTracker` subscription,
an HTTP `Date:` header, and stack traces embedded by the Functions host wrapper
exceptions varied per occurrence too. Each case below pairs two occurrences of
one failure that must share a signature, and every pair of distinct failures
must still stay apart.

Both Azure providers normalize the same way, so each case runs through the
error-logs adapter (App Insights) and the dlq adapter (Service Bus).
"""

from __future__ import annotations

import unittest
from dataclasses import dataclass

from specfuse.monitor.fingerprint import fingerprint_artifact
from specfuse.monitor.providers.azure_app_insights import ErrorLogsAdapter
from specfuse.monitor.providers.azure_service_bus import ServiceBusDlqAdapter

_LOCK_LOST = (
"The lock supplied is invalid. Either the lock expired, or the message has "
"already been removed from the queue. For more information please see "
"https://aka.ms/ServiceBusExceptions . Reference:{ref}, TrackingId:{track}, "
"SystemTracker:sbns-acme-prd:topic:sbt-acme~{part}|{sub}, Timestamp:{ts} "
"(MessageLockLost). For troubleshooting information, see "
"https://aka.ms/azsdk/net/servicebus/exceptions/troubleshoot."
)

LOCK_LOST_A = _LOCK_LOST.format(
ref="1b9d4c1e-7a3f-4e0b-9c55-2f6a8d7e1a10",
track="4311145c-fcf8-4a21-9bdb-95c15255c504_B4",
part="16383",
sub="on-order-created",
ts="2026-11-16T10:00:00",
)
LOCK_LOST_B = _LOCK_LOST.format(
ref="9e2a7b30-55c1-4d8e-8f0a-6b1c3d2e4f50",
track="68128c21-0bb5-44ea-8e81-77822ab6f35b_B11",
part="12287",
sub="on-invoice-paid-sync-ledger",
ts="2026-11-16T11:42:17",
)

_RPC = (
'Status(StatusCode="Unknown", Detail="Azure.Messaging.ServiceBus.'
"ServiceBusException: The lock supplied is invalid. (MessageLockLost)\n"
" at Azure.Messaging.ServiceBus.Amqp.AmqpReceiver.{receiver}(Guid lockToken, "
"TimeSpan timeout)\n"
" at Acme.Functions.{function}.Run(ServiceBusReceivedMessage message) in "
'/src/Acme.Functions/{function}.cs:line {line}")'
)
RPC_A = _RPC.format(receiver="CompleteInternalAsync", function="Orders.OrderCreatedTrigger", line=42)
RPC_B = _RPC.format(receiver="AbandonInternalAsync", function="Billing.InvoicePaidTrigger", line=117)

_LEASE = (
"The lease ID specified did not match the lease ID for the blob. "
"RequestId:{req} Time:{time} Status: 412 (The lease ID specified did not match "
"the lease ID for the blob.) ErrorCode: LeaseIdMismatchWithLeaseOperation "
"Headers: Date: {date} Server: Windows-Azure-Blob/1.0"
)
LEASE_A = _LEASE.format(
req="0f8e7d6c-5b4a-4392-8170-6f5e4d3c2b1a",
time="2026-11-14T08:01:02.1234567Z",
date="Fri, 14 Nov 2026 08:01:02 GMT",
)
LEASE_B = _LEASE.format(
req="a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d",
time="2026-11-15T19:30:45.7654321Z",
date="Sat, 15 Nov 2026 19:30:45 GMT",
)

# Same failure in each pair; different failures across pairs.
SAME_FAILURE_PAIRS = {
"service-bus-lock-lost": (LOCK_LOST_A, LOCK_LOST_B),
"functions-host-wrapper-with-stack": (RPC_A, RPC_B),
"blob-lease-with-http-date": (LEASE_A, LEASE_B),
}


class _Transport:
def __init__(self, messages):
self._messages = messages

def run_query(self, query):
return [
{"type": "Acme.FailureException", "outerMessage": m, "operation_Id": f"op{i}"}
for i, m in enumerate(self._messages)
]


@dataclass(frozen=True)
class _DeadLetter:
message_id: str
dead_letter_reason: str
dead_letter_error_description: str
exception_type: str = "Acme.FailureException"


class _DlqTransport:
def __init__(self, messages):
self._messages = messages

def peek_dead_letter_messages(self, *, subscription, max_message_count):
return [
_DeadLetter(f"m{i}", "MaxDeliveryCountExceeded", m)
for i, m in enumerate(self._messages)
]


def _error_logs_fingerprints(messages):
adapter = ErrorLogsAdapter(
component="acme-worker",
environment={"telemetry": {"provider": "app-insights"}},
transport=_Transport(messages),
)
return [fingerprint_artifact(a) for a in adapter.fetch_failures()]


def _dlq_fingerprints(messages):
adapter = ServiceBusDlqAdapter(
component="acme-worker",
transport=_DlqTransport(messages),
targets=[{"subscription": "acme-sub", "function": "AcmeFunction"}],
)
return [fingerprint_artifact(a) for a in adapter.fetch_failures()]


_PROVIDERS = {"error-logs": _error_logs_fingerprints, "dlq": _dlq_fingerprints}


class SameFailureCollapsesTests(unittest.TestCase):
def test_two_occurrences_of_one_failure_share_a_fingerprint(self):
for provider, fingerprints in _PROVIDERS.items():
for case, pair in SAME_FAILURE_PAIRS.items():
with self.subTest(provider=provider, case=case):
first, second = fingerprints(list(pair))
self.assertEqual(first, second)


class DistinctFailuresStayApartTests(unittest.TestCase):
def test_different_failures_keep_different_fingerprints(self):
messages = [pair[0] for pair in SAME_FAILURE_PAIRS.values()] + [
"Connection to the database failed at the gateway.",
"Connection to the cache failed at the gateway.",
]
for provider, fingerprints in _PROVIDERS.items():
with self.subTest(provider=provider):
prints = fingerprints(messages)
self.assertEqual(len(set(prints)), len(messages))


class GuidBoundaryTests(unittest.TestCase):
def test_guid_is_normalized_whatever_follows_it(self):
guid_a = "4311145c-fcf8-4a21-9bdb-95c15255c504"
guid_b = "68128c21-0bb5-44ea-8e81-77822ab6f35b"
for suffix in ("_B4", ")", ".", ""):
for provider, fingerprints in _PROVIDERS.items():
with self.subTest(provider=provider, suffix=suffix):
first, second = fingerprints(
[f"Entity id={guid_a}{suffix} failed", f"Entity id={guid_b}{suffix} failed"]
)
self.assertEqual(first, second)


if __name__ == "__main__":
unittest.main()
Loading