Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changelog.d/3565-specfuse-monitor-run-a-dlq-or-queue-stal.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
class: fixed
trace: #3565
---
`specfuse monitor run`: a `dlq` or `queue-stalled` target whose Service Bus subscription does not exist is now skipped and reported in the run summary; the check keeps reading the other targets instead of failing as a whole.
60 changes: 49 additions & 11 deletions specfuse/monitor/providers/azure_service_bus.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,23 @@ class DeadLetterMessage(Protocol):
dead_letter_error_description: str


# What the Azure SDK raises for a subscription that does not exist (#3565):
# `azure.servicebus.exceptions.MessagingEntityNotFoundError` from a peek,
# `azure.core.exceptions.ResourceNotFoundError` from a management read.
# Matched by class name so this module needs no Azure SDK import.
_MISSING_ENTITY_ERRORS = frozenset(
{"MessagingEntityNotFoundError", "ResourceNotFoundError"}
)


def _is_missing_entity(exc: BaseException) -> bool:
return any(cls.__name__ in _MISSING_ENTITY_ERRORS for cls in type(exc).__mro__)


def _missing_reason(exc: BaseException) -> str:
return f"subscription not found ({type(exc).__name__})"


class ServiceBusDlqTransport(Protocol):
"""The peek-only operation this adapter needs from a transport."""

Expand Down Expand Up @@ -226,15 +243,28 @@ def __init__(
self._transport = transport
self._targets = targets
self._max_message_count = max_message_count
self.skipped_targets: List[SkippedTarget] = []

def fetch_failures(self) -> Iterator[FailureArtifact]:
for target in self._targets:
subscription = target["subscription"]
function = target["function"]
messages = self._transport.peek_dead_letter_messages(
subscription=subscription,
max_message_count=self._max_message_count,
)
try:
messages = list(
self._transport.peek_dead_letter_messages(
subscription=subscription,
max_message_count=self._max_message_count,
)
)
except Exception as exc:
if not _is_missing_entity(exc):
raise
# A retired or not-yet-deployed subscription is routine: report
# it and keep reading the other targets (#3565).
self.skipped_targets.append(
SkippedTarget(subscription, function, _missing_reason(exc))
)
continue
for message in messages:
yield self._build_artifact(subscription, function, message)

Expand Down Expand Up @@ -291,7 +321,7 @@ def _age_text(seconds: float) -> str:

@dataclass(frozen=True)
class SkippedTarget:
"""A `queue-stalled` target the adapter did not evaluate, and why."""
"""A `dlq` or `queue-stalled` target the adapter did not evaluate, and why."""

subscription: str
function: str
Expand Down Expand Up @@ -352,12 +382,20 @@ def fetch_failures(self) -> Iterator[FailureArtifact]:
continue

stall_after_seconds = parse_stall_after_seconds(stall_after_raw)
active_count = self._transport.get_active_message_count(
subscription=subscription
)
oldest_enqueued = self._transport.get_oldest_message_enqueued_time(
subscription=subscription
)
try:
active_count = self._transport.get_active_message_count(
subscription=subscription
)
oldest_enqueued = self._transport.get_oldest_message_enqueued_time(
subscription=subscription
)
except Exception as exc:
if not _is_missing_entity(exc):
raise
self.skipped_targets.append(
SkippedTarget(subscription, function, _missing_reason(exc))
)
continue
if oldest_enqueued is None:
continue
age_seconds = (self._reference - oldest_enqueued).total_seconds()
Expand Down
123 changes: 123 additions & 0 deletions tests/test_service_bus_missing_subscription.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
# Copyright 2026 Specfuse Contributors
# Licensed under the Apache License, Version 2.0. See LICENSE.
"""#3565: one missing subscription must not fail the whole `dlq` or
`queue-stalled` check. The missing target is skipped and reported; the other
targets are still read. Any other transport error still fails the check."""

from __future__ import annotations

import unittest
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone

from specfuse.monitor.providers.azure_service_bus import (
QueueStalledAdapter,
ServiceBusDlqAdapter,
)

_NOW = datetime(2026, 11, 3, 12, 0, 0, tzinfo=timezone.utc)


# Stand-ins named like the Azure SDK classes the real transports raise, so the
# test needs no Azure SDK installed.
class MessagingEntityNotFoundError(Exception):
pass


class ResourceNotFoundError(Exception):
pass


class ServiceRequestError(Exception):
pass


@dataclass(frozen=True)
class _Message:
message_id: str
dead_letter_reason: str = "MaxDeliveryCountExceeded"
dead_letter_error_description: str = "System.TimeoutException: timed out"


class _DlqTransport:
def __init__(self, missing, error=MessagingEntityNotFoundError):
self._missing = missing
self._error = error

def peek_dead_letter_messages(self, *, subscription, max_message_count):
if subscription == self._missing:
raise self._error(f"The messaging entity '{subscription}' could not be found.")
return [_Message(message_id=f"{subscription}-1")]


class _StalledTransport:
def __init__(self, missing, error=ResourceNotFoundError):
self._missing = missing
self._error = error

def _check(self, subscription):
if subscription == self._missing:
raise self._error("Operation returned an invalid status 'Not Found'")

def get_active_message_count(self, *, subscription):
self._check(subscription)
return 3

def get_oldest_message_enqueued_time(self, *, subscription):
self._check(subscription)
return _NOW - timedelta(hours=2)


_TARGETS = [
{"subscription": "first-sub", "function": "First", "stall_after": "10m"},
{"subscription": "retired-sub", "function": "Retired", "stall_after": "10m"},
{"subscription": "last-sub", "function": "Last", "stall_after": "10m"},
]


class DlqMissingSubscriptionTests(unittest.TestCase):
def _adapter(self, transport):
return ServiceBusDlqAdapter(component="svc", transport=transport, targets=_TARGETS)

def test_missing_subscription_is_skipped_and_others_still_read(self):
adapter = self._adapter(_DlqTransport("retired-sub"))
artifacts = list(adapter.fetch_failures())
self.assertEqual(
[a.message_id for a in artifacts], ["first-sub-1", "last-sub-1"]
)
self.assertEqual(len(adapter.skipped_targets), 1)
skipped = adapter.skipped_targets[0]
self.assertEqual(
(skipped.subscription, skipped.function), ("retired-sub", "Retired")
)
self.assertIn("not found", skipped.reason)

def test_other_transport_errors_still_fail_the_check(self):
adapter = self._adapter(_DlqTransport("retired-sub", error=ServiceRequestError))
with self.assertRaises(ServiceRequestError):
list(adapter.fetch_failures())


class QueueStalledMissingSubscriptionTests(unittest.TestCase):
def _adapter(self, transport):
return QueueStalledAdapter(
component="svc", transport=transport, targets=_TARGETS, reference=_NOW
)

def test_missing_subscription_is_skipped_and_others_still_read(self):
adapter = self._adapter(_StalledTransport("retired-sub"))
artifacts = list(adapter.fetch_failures())
self.assertEqual(len(artifacts), 2)
self.assertEqual(
[s.subscription for s in adapter.skipped_targets], ["retired-sub"]
)
self.assertIn("not found", adapter.skipped_targets[0].reason)

def test_other_transport_errors_still_fail_the_check(self):
adapter = self._adapter(_StalledTransport("retired-sub", error=ServiceRequestError))
with self.assertRaises(ServiceRequestError):
list(adapter.fetch_failures())


if __name__ == "__main__":
unittest.main()
Loading