diff --git a/changelog.d/3565-specfuse-monitor-run-a-dlq-or-queue-stal.md b/changelog.d/3565-specfuse-monitor-run-a-dlq-or-queue-stal.md new file mode 100644 index 00000000..6be8215f --- /dev/null +++ b/changelog.d/3565-specfuse-monitor-run-a-dlq-or-queue-stal.md @@ -0,0 +1,5 @@ +--- +class: fixed +trace: #3565 +--- +`specfuse monitor run`: a `dlq` or `queue-stalled` target whose Service Bus subscription does not exist is now skipped and reported in the run summary; the check keeps reading the other targets instead of failing as a whole. diff --git a/specfuse/monitor/providers/azure_service_bus.py b/specfuse/monitor/providers/azure_service_bus.py index b7b1a326..10eccdd7 100644 --- a/specfuse/monitor/providers/azure_service_bus.py +++ b/specfuse/monitor/providers/azure_service_bus.py @@ -194,6 +194,23 @@ class DeadLetterMessage(Protocol): dead_letter_error_description: str +# What the Azure SDK raises for a subscription that does not exist (#3565): +# `azure.servicebus.exceptions.MessagingEntityNotFoundError` from a peek, +# `azure.core.exceptions.ResourceNotFoundError` from a management read. +# Matched by class name so this module needs no Azure SDK import. +_MISSING_ENTITY_ERRORS = frozenset( + {"MessagingEntityNotFoundError", "ResourceNotFoundError"} +) + + +def _is_missing_entity(exc: BaseException) -> bool: + return any(cls.__name__ in _MISSING_ENTITY_ERRORS for cls in type(exc).__mro__) + + +def _missing_reason(exc: BaseException) -> str: + return f"subscription not found ({type(exc).__name__})" + + class ServiceBusDlqTransport(Protocol): """The peek-only operation this adapter needs from a transport.""" @@ -226,15 +243,28 @@ def __init__( self._transport = transport self._targets = targets self._max_message_count = max_message_count + self.skipped_targets: List[SkippedTarget] = [] def fetch_failures(self) -> Iterator[FailureArtifact]: for target in self._targets: subscription = target["subscription"] function = target["function"] - messages = self._transport.peek_dead_letter_messages( - subscription=subscription, - max_message_count=self._max_message_count, - ) + try: + messages = list( + self._transport.peek_dead_letter_messages( + subscription=subscription, + max_message_count=self._max_message_count, + ) + ) + except Exception as exc: + if not _is_missing_entity(exc): + raise + # A retired or not-yet-deployed subscription is routine: report + # it and keep reading the other targets (#3565). + self.skipped_targets.append( + SkippedTarget(subscription, function, _missing_reason(exc)) + ) + continue for message in messages: yield self._build_artifact(subscription, function, message) @@ -291,7 +321,7 @@ def _age_text(seconds: float) -> str: @dataclass(frozen=True) class SkippedTarget: - """A `queue-stalled` target the adapter did not evaluate, and why.""" + """A `dlq` or `queue-stalled` target the adapter did not evaluate, and why.""" subscription: str function: str @@ -352,12 +382,20 @@ def fetch_failures(self) -> Iterator[FailureArtifact]: continue stall_after_seconds = parse_stall_after_seconds(stall_after_raw) - active_count = self._transport.get_active_message_count( - subscription=subscription - ) - oldest_enqueued = self._transport.get_oldest_message_enqueued_time( - subscription=subscription - ) + try: + active_count = self._transport.get_active_message_count( + subscription=subscription + ) + oldest_enqueued = self._transport.get_oldest_message_enqueued_time( + subscription=subscription + ) + except Exception as exc: + if not _is_missing_entity(exc): + raise + self.skipped_targets.append( + SkippedTarget(subscription, function, _missing_reason(exc)) + ) + continue if oldest_enqueued is None: continue age_seconds = (self._reference - oldest_enqueued).total_seconds() diff --git a/tests/test_service_bus_missing_subscription.py b/tests/test_service_bus_missing_subscription.py new file mode 100644 index 00000000..d229f95b --- /dev/null +++ b/tests/test_service_bus_missing_subscription.py @@ -0,0 +1,123 @@ +# Copyright 2026 Specfuse Contributors +# Licensed under the Apache License, Version 2.0. See LICENSE. +"""#3565: one missing subscription must not fail the whole `dlq` or +`queue-stalled` check. The missing target is skipped and reported; the other +targets are still read. Any other transport error still fails the check.""" + +from __future__ import annotations + +import unittest +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from specfuse.monitor.providers.azure_service_bus import ( + QueueStalledAdapter, + ServiceBusDlqAdapter, +) + +_NOW = datetime(2026, 11, 3, 12, 0, 0, tzinfo=timezone.utc) + + +# Stand-ins named like the Azure SDK classes the real transports raise, so the +# test needs no Azure SDK installed. +class MessagingEntityNotFoundError(Exception): + pass + + +class ResourceNotFoundError(Exception): + pass + + +class ServiceRequestError(Exception): + pass + + +@dataclass(frozen=True) +class _Message: + message_id: str + dead_letter_reason: str = "MaxDeliveryCountExceeded" + dead_letter_error_description: str = "System.TimeoutException: timed out" + + +class _DlqTransport: + def __init__(self, missing, error=MessagingEntityNotFoundError): + self._missing = missing + self._error = error + + def peek_dead_letter_messages(self, *, subscription, max_message_count): + if subscription == self._missing: + raise self._error(f"The messaging entity '{subscription}' could not be found.") + return [_Message(message_id=f"{subscription}-1")] + + +class _StalledTransport: + def __init__(self, missing, error=ResourceNotFoundError): + self._missing = missing + self._error = error + + def _check(self, subscription): + if subscription == self._missing: + raise self._error("Operation returned an invalid status 'Not Found'") + + def get_active_message_count(self, *, subscription): + self._check(subscription) + return 3 + + def get_oldest_message_enqueued_time(self, *, subscription): + self._check(subscription) + return _NOW - timedelta(hours=2) + + +_TARGETS = [ + {"subscription": "first-sub", "function": "First", "stall_after": "10m"}, + {"subscription": "retired-sub", "function": "Retired", "stall_after": "10m"}, + {"subscription": "last-sub", "function": "Last", "stall_after": "10m"}, +] + + +class DlqMissingSubscriptionTests(unittest.TestCase): + def _adapter(self, transport): + return ServiceBusDlqAdapter(component="svc", transport=transport, targets=_TARGETS) + + def test_missing_subscription_is_skipped_and_others_still_read(self): + adapter = self._adapter(_DlqTransport("retired-sub")) + artifacts = list(adapter.fetch_failures()) + self.assertEqual( + [a.message_id for a in artifacts], ["first-sub-1", "last-sub-1"] + ) + self.assertEqual(len(adapter.skipped_targets), 1) + skipped = adapter.skipped_targets[0] + self.assertEqual( + (skipped.subscription, skipped.function), ("retired-sub", "Retired") + ) + self.assertIn("not found", skipped.reason) + + def test_other_transport_errors_still_fail_the_check(self): + adapter = self._adapter(_DlqTransport("retired-sub", error=ServiceRequestError)) + with self.assertRaises(ServiceRequestError): + list(adapter.fetch_failures()) + + +class QueueStalledMissingSubscriptionTests(unittest.TestCase): + def _adapter(self, transport): + return QueueStalledAdapter( + component="svc", transport=transport, targets=_TARGETS, reference=_NOW + ) + + def test_missing_subscription_is_skipped_and_others_still_read(self): + adapter = self._adapter(_StalledTransport("retired-sub")) + artifacts = list(adapter.fetch_failures()) + self.assertEqual(len(artifacts), 2) + self.assertEqual( + [s.subscription for s in adapter.skipped_targets], ["retired-sub"] + ) + self.assertIn("not found", adapter.skipped_targets[0].reason) + + def test_other_transport_errors_still_fail_the_check(self): + adapter = self._adapter(_StalledTransport("retired-sub", error=ServiceRequestError)) + with self.assertRaises(ServiceRequestError): + list(adapter.fetch_failures()) + + +if __name__ == "__main__": + unittest.main()