From 53539dbb4c389d1d7574f40406f139e641ace7fc Mon Sep 17 00:00:00 2001 From: Test Date: Sun, 15 Nov 2026 06:58:33 -0500 Subject: [PATCH] fix(monitor): count a heartbeat a few seconds early as the expected firing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #3564. `HeartbeatAdapter` treated a schedule as silent whenever its last heartbeat was before the most recent expected firing, with no tolerance. Timer hosts can fire slightly early (an Azure Functions `0 */5 * * * *` timer logged 03:59:59.999238Z for a 04:00:00Z firing, about 1.4% of heartbeats over a week), so healthy schedules got `silent-schedule` findings that persist for a whole period on daily or weekly schedules. A heartbeat now counts as the expected firing when it lands within `HEARTBEAT_EARLY_FIRING_TOLERANCE` (5 seconds) before it. The issue suggested one minute; 5 seconds covers the observed early firing while a missed firing on a one-minute or seconds-dialect schedule is still caught. Tests: - tests/test_heartbeat_early_firing.py — sub-millisecond and 5-second-early heartbeats are not silent; 6 seconds early and the previous firing's heartbeat still are. Co-Authored-By: Claude Opus 5.5 --- ...pecfuse-monitor-run-the-heartbeat-check.md | 5 ++ .../monitor/providers/azure_app_insights.py | 16 ++++- tests/test_heartbeat_early_firing.py | 70 +++++++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 changelog.d/3564-specfuse-monitor-run-the-heartbeat-check.md create mode 100644 tests/test_heartbeat_early_firing.py diff --git a/changelog.d/3564-specfuse-monitor-run-the-heartbeat-check.md b/changelog.d/3564-specfuse-monitor-run-the-heartbeat-check.md new file mode 100644 index 00000000..c3a85f64 --- /dev/null +++ b/changelog.d/3564-specfuse-monitor-run-the-heartbeat-check.md @@ -0,0 +1,5 @@ +--- +class: fixed +trace: #3564 +--- +`specfuse monitor run`: the `heartbeat` check no longer files a `silent-schedule` finding when a timer fires a fraction of a second before its scheduled instant; a heartbeat up to 5 seconds early counts as that firing. diff --git a/specfuse/monitor/providers/azure_app_insights.py b/specfuse/monitor/providers/azure_app_insights.py index ba14b825..183416b2 100644 --- a/specfuse/monitor/providers/azure_app_insights.py +++ b/specfuse/monitor/providers/azure_app_insights.py @@ -1012,6 +1012,14 @@ def _parse_heartbeat_timestamp(value: object) -> Optional[datetime]: return datetime.fromisoformat(text) +# A timer host can fire a hair before its scheduled instant (#3564: an Azure +# Functions `0 */5 * * * *` timer logged 03:59:59.999 for a 04:00:00 firing). +# A heartbeat this close before the expected firing counts as that firing. Kept +# to seconds, not minutes, so a missed firing on a one-minute or sub-minute +# schedule is still caught. +HEARTBEAT_EARLY_FIRING_TOLERANCE = timedelta(seconds=5) + + class HeartbeatAdapter: """``TelemetryAdapter`` for the ``heartbeat`` check type (FEAT-2026-0040/T07). @@ -1022,7 +1030,8 @@ class HeartbeatAdapter: under its declared dialect (``specfuse.monitor.schedule``, reading T04's dialect rather than inferring it) was expected to fire, and compares that against the target's last observed heartbeat row. A target that - reported in at or after its most recent expected firing yields nothing; + reported in at or after its most recent expected firing (less + `HEARTBEAT_EARLY_FIRING_TOLERANCE`) yields nothing; only a target that has gone silent yields a `FailureArtifact`, built through `FailureArtifact.from_target` so its `name` round-trips into `target_coordinates` and keeps distinct silent schedules on one @@ -1091,7 +1100,10 @@ def fetch_failures(self) -> Iterator[FailureArtifact]: # Pending first firing: emission began after this firing. continue observed = last_seen.get(name) - if observed is not None and observed >= expected: + if ( + observed is not None + and observed >= expected - HEARTBEAT_EARLY_FIRING_TOLERANCE + ): continue yield self._build_artifact( target, expected, observed, details.get(name, {}), host_last_seen, earliest diff --git a/tests/test_heartbeat_early_firing.py b/tests/test_heartbeat_early_firing.py new file mode 100644 index 00000000..32b8231d --- /dev/null +++ b/tests/test_heartbeat_early_firing.py @@ -0,0 +1,70 @@ +# Copyright 2026 Specfuse Contributors +# Licensed under the Apache License, Version 2.0. See LICENSE. +"""#3564: a timer host can fire a hair before its scheduled instant. A +heartbeat that lands within the early-firing tolerance of the expected firing +counts as that firing; one further before it is still silent.""" + +from __future__ import annotations + +import unittest +from datetime import datetime, timezone + +from specfuse.monitor.providers.azure_app_insights import HeartbeatAdapter + +_ENVIRONMENT = {"telemetry": {"provider": "app-insights"}} +# Most recent expected firing of the 5-minute schedule below is 04:00:00. +_REFERENCE = datetime(2026, 11, 3, 4, 2, 0, tzinfo=timezone.utc) +_TARGETS = [ + { + "name": "five-minute-sweep", + "cron": "0 */5 * * * *", + "dialect": "seconds-first-6", + "timezone": "Etc/UTC", + } +] + + +class _Transport: + def __init__(self, last_heartbeat): + self._rows = [ + { + "name": "five-minute-sweep", + "last_heartbeat": last_heartbeat, + "first_heartbeat": "2026-11-01T00:00:00Z", + } + ] + + def run_query(self, query): + return list(self._rows) + + +def _artifacts(last_heartbeat): + adapter = HeartbeatAdapter( + component="acme-functions-host", + environment=_ENVIRONMENT, + transport=_Transport(last_heartbeat), + targets=_TARGETS, + reference_time=_REFERENCE, + ) + return list(adapter.fetch_failures()) + + +class HeartbeatEarlyFiringTests(unittest.TestCase): + def test_heartbeat_a_fraction_of_a_second_early_is_not_silent(self): + # The reported case: under a millisecond before 04:00:00. + self.assertEqual(_artifacts("2026-11-03T03:59:59.999238Z"), []) + + def test_heartbeat_just_inside_the_tolerance_is_not_silent(self): + self.assertEqual(_artifacts("2026-11-03T03:59:55Z"), []) + + def test_heartbeat_from_the_previous_firing_is_still_silent(self): + artifacts = _artifacts("2026-11-03T03:55:00Z") + self.assertEqual(len(artifacts), 1) + self.assertEqual(artifacts[0].failure_class, "silent-schedule") + + def test_heartbeat_just_outside_the_tolerance_is_still_silent(self): + self.assertEqual(len(_artifacts("2026-11-03T03:59:54Z")), 1) + + +if __name__ == "__main__": + unittest.main()