Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changelog.d/3564-specfuse-monitor-run-the-heartbeat-check.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
class: fixed
trace: #3564
---
`specfuse monitor run`: the `heartbeat` check no longer files a `silent-schedule` finding when a timer fires a fraction of a second before its scheduled instant; a heartbeat up to 5 seconds early counts as that firing.
16 changes: 14 additions & 2 deletions specfuse/monitor/providers/azure_app_insights.py
Original file line number Diff line number Diff line change
Expand Up @@ -1012,6 +1012,14 @@ def _parse_heartbeat_timestamp(value: object) -> Optional[datetime]:
return datetime.fromisoformat(text)


# A timer host can fire a hair before its scheduled instant (#3564: an Azure
# Functions `0 */5 * * * *` timer logged 03:59:59.999 for a 04:00:00 firing).
# A heartbeat this close before the expected firing counts as that firing. Kept
# to seconds, not minutes, so a missed firing on a one-minute or sub-minute
# schedule is still caught.
HEARTBEAT_EARLY_FIRING_TOLERANCE = timedelta(seconds=5)


class HeartbeatAdapter:
"""``TelemetryAdapter`` for the ``heartbeat`` check type
(FEAT-2026-0040/T07).
Expand All @@ -1022,7 +1030,8 @@ class HeartbeatAdapter:
under its declared dialect (``specfuse.monitor.schedule``, reading T04's
dialect rather than inferring it) was expected to fire, and compares
that against the target's last observed heartbeat row. A target that
reported in at or after its most recent expected firing yields nothing;
reported in at or after its most recent expected firing (less
`HEARTBEAT_EARLY_FIRING_TOLERANCE`) yields nothing;
only a target that has gone silent yields a `FailureArtifact`, built
through `FailureArtifact.from_target` so its `name` round-trips into
`target_coordinates` and keeps distinct silent schedules on one
Expand Down Expand Up @@ -1091,7 +1100,10 @@ def fetch_failures(self) -> Iterator[FailureArtifact]:
# Pending first firing: emission began after this firing.
continue
observed = last_seen.get(name)
if observed is not None and observed >= expected:
if (
observed is not None
and observed >= expected - HEARTBEAT_EARLY_FIRING_TOLERANCE
):
continue
yield self._build_artifact(
target, expected, observed, details.get(name, {}), host_last_seen, earliest
Expand Down
70 changes: 70 additions & 0 deletions tests/test_heartbeat_early_firing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# Copyright 2026 Specfuse Contributors
# Licensed under the Apache License, Version 2.0. See LICENSE.
"""#3564: a timer host can fire a hair before its scheduled instant. A
heartbeat that lands within the early-firing tolerance of the expected firing
counts as that firing; one further before it is still silent."""

from __future__ import annotations

import unittest
from datetime import datetime, timezone

from specfuse.monitor.providers.azure_app_insights import HeartbeatAdapter

_ENVIRONMENT = {"telemetry": {"provider": "app-insights"}}
# Most recent expected firing of the 5-minute schedule below is 04:00:00.
_REFERENCE = datetime(2026, 11, 3, 4, 2, 0, tzinfo=timezone.utc)
_TARGETS = [
{
"name": "five-minute-sweep",
"cron": "0 */5 * * * *",
"dialect": "seconds-first-6",
"timezone": "Etc/UTC",
}
]


class _Transport:
def __init__(self, last_heartbeat):
self._rows = [
{
"name": "five-minute-sweep",
"last_heartbeat": last_heartbeat,
"first_heartbeat": "2026-11-01T00:00:00Z",
}
]

def run_query(self, query):
return list(self._rows)


def _artifacts(last_heartbeat):
adapter = HeartbeatAdapter(
component="acme-functions-host",
environment=_ENVIRONMENT,
transport=_Transport(last_heartbeat),
targets=_TARGETS,
reference_time=_REFERENCE,
)
return list(adapter.fetch_failures())


class HeartbeatEarlyFiringTests(unittest.TestCase):
def test_heartbeat_a_fraction_of_a_second_early_is_not_silent(self):
# The reported case: under a millisecond before 04:00:00.
self.assertEqual(_artifacts("2026-11-03T03:59:59.999238Z"), [])

def test_heartbeat_just_inside_the_tolerance_is_not_silent(self):
self.assertEqual(_artifacts("2026-11-03T03:59:55Z"), [])

def test_heartbeat_from_the_previous_firing_is_still_silent(self):
artifacts = _artifacts("2026-11-03T03:55:00Z")
self.assertEqual(len(artifacts), 1)
self.assertEqual(artifacts[0].failure_class, "silent-schedule")

def test_heartbeat_just_outside_the_tolerance_is_still_silent(self):
self.assertEqual(len(_artifacts("2026-11-03T03:59:54Z")), 1)


if __name__ == "__main__":
unittest.main()
Loading