From d2e9683d0df4c961f85ee11aff4f3033ba63b5f3 Mon Sep 17 00:00:00 2001 From: Gavin Acosta <155584291+Defiantearth@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:17:41 -0500 Subject: [PATCH 1/4] Fixed Arista EOS reboots not being detected when waiting for the device to reload. --- changes/407.fixed | 1 + pyntc/devices/eos_device.py | 47 +++++++++++++++++----- tests/unit/test_devices/test_eos_device.py | 22 +++++++++- 3 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 changes/407.fixed diff --git a/changes/407.fixed b/changes/407.fixed new file mode 100644 index 00000000..72c5f908 --- /dev/null +++ b/changes/407.fixed @@ -0,0 +1 @@ +Fixed Arista EOS reboots not being detected when waiting for the device to reload. diff --git a/pyntc/devices/eos_device.py b/pyntc/devices/eos_device.py index 2c112c85..17c193e1 100644 --- a/pyntc/devices/eos_device.py +++ b/pyntc/devices/eos_device.py @@ -27,6 +27,7 @@ EOS_SUPPORTED_HASHING_ALGORITHMS = {"md5", "sha1", "sha256", "sha512"} # Subset of HASHING_ALGORITHMS for EOS verify EOS_SUPPORTED_SCHEMES = {"http", "https", "scp", "ftp", "sftp", "tftp"} +DEFAULT_REBOOT_TIMEOUT = 3600 # seconds (1 hour) to wait for a device to come back after a reboot BASIC_FACTS_KM = {"model": "modelName", "os_version": "internalVersion", "serial_number": "serialNumber"} INTERFACES_KM = { "speed": "bandwidth", @@ -177,15 +178,38 @@ def _uptime_to_string(self, uptime): return f"{days:02d}:{hours:02d}:{mins:02d}:{seconds:02d}" - def _wait_for_device_reboot(self, timeout=3600): + def _wait_for_device_reboot(self, original_uptime, timeout=DEFAULT_REBOOT_TIMEOUT): + """Block until the device reboots, detected by uptime dropping below original_uptime. + + Args: + original_uptime (int): Device uptime in seconds captured before the reboot. + timeout (int): Max seconds to poll for the device to return. Defaults to 3600. + + Raises: + RebootTimeoutError: When the device does not report a reset uptime within timeout. + """ start = time.time() while time.time() - start < timeout: try: - self.show("show hostname") - log.debug("Host %s: Device rebooted.", self.host) - return - except: # noqa E722 # nosec # pylint: disable=bare-except - time.sleep(10) + self._uptime = None # bust the cached value so we re-read from the device + current_uptime = self.uptime + if current_uptime < original_uptime: + log.info( + "Host %s: Device rebooted (uptime %ss < pre-reboot %ss).", + self.host, + current_uptime, + original_uptime, + ) + return + log.debug( + "Host %s: Reachable but uptime %ss >= pre-reboot %ss; still waiting.", + self.host, + current_uptime, + original_uptime, + ) + except Exception as exc: # pylint: disable=broad-exception-caught + log.debug("Host %s: Reboot probe failed (%s); will retry.", self.host, exc) + time.sleep(10) log.error("Host %s: Device timed out while rebooting.", self.host) raise RebootTimeoutError(hostname=self.hostname, wait_time=timeout) @@ -729,14 +753,13 @@ def install_os(self, image_name, reboot=True, **vendor_specifics): Returns: (bool): True if device OS is succesfully installed. """ - timeout = vendor_specifics.get("timeout", 3600) + timeout = vendor_specifics.get("timeout", DEFAULT_REBOOT_TIMEOUT) if not self._image_booted(image_name): self.set_boot_options(image_name, **vendor_specifics) if not reboot: log.info("Host %s: OS image %s boot options set. Reboot the device to apply", self.host, image_name) return True - self.reboot() - self._wait_for_device_reboot(timeout=timeout) + self.reboot(wait_for_reload=True, timeout=timeout) if not self._image_booted(image_name): log.error("Host %s: OS install error for image %s", self.host, image_name) raise OSInstallError(hostname=self.hostname, desired_boot=image_name) @@ -770,12 +793,13 @@ def open(self): log.debug("Host %s: Connection to controller was opened successfully.", self.host) - def reboot(self, wait_for_reload=False, **kwargs): + def reboot(self, wait_for_reload=False, timeout=DEFAULT_REBOOT_TIMEOUT, **kwargs): """ Reload the controller or controller pair. Args: wait_for_reload (bool): Whether or not reboot method should also run _wait_for_device_reboot(). Defaults to False. + timeout (int): Max seconds to poll for the device to return when wait_for_reload is True. Defaults to 3600. kwargs (dict): Additional keyword arguments, such as confirm. Raises: @@ -790,10 +814,11 @@ def reboot(self, wait_for_reload=False, **kwargs): if kwargs.get("confirm"): log.warning("Passing 'confirm' to reboot method is deprecated.") + original_uptime = self.uptime if wait_for_reload else None self.show("reload now") log.info("Host %s: Device rebooted.", self.host) if wait_for_reload: - self._wait_for_device_reboot() + self._wait_for_device_reboot(original_uptime, timeout=timeout) def rollback(self, rollback_to): """Rollback device configuration. diff --git a/tests/unit/test_devices/test_eos_device.py b/tests/unit/test_devices/test_eos_device.py index 287025ba..517a5ed1 100644 --- a/tests/unit/test_devices/test_eos_device.py +++ b/tests/unit/test_devices/test_eos_device.py @@ -9,7 +9,7 @@ from pyntc.devices.base_device import RollbackError from pyntc.devices.eos_device import FileTransferError from pyntc.devices.system_features.vlans.eos_vlans import EOSVlans -from pyntc.errors import CommandError, CommandListError, NotEnoughFreeSpaceError # noqa: F401 +from pyntc.errors import CommandError, CommandListError, NotEnoughFreeSpaceError, RebootTimeoutError # noqa: F401 from pyntc.utils.models import FileCopyModel from .device_mocks.eos import config, enable, send_command, send_command_expect @@ -286,6 +286,26 @@ def test_reboot(self): self.device.reboot() self.device.native.enable.assert_called_with(["reload now"], encoding="json") + @mock.patch("pyntc.devices.eos_device.time") + @mock.patch.object(EOSDevice, "uptime", new_callable=mock.PropertyMock) + def test_wait_for_device_reboot(self, mock_device_uptime, mock_time): + mock_time.time.side_effect = [0, 1, 2, 3] + mock_device_uptime.side_effect = [10005, Exception("unreachable"), 12] + + self.device._wait_for_device_reboot(original_uptime=10000) + + self.assertEqual(mock_device_uptime.call_count, 3) + self.assertEqual(mock_time.sleep.call_count, 2) + + @mock.patch("pyntc.devices.eos_device.time") + @mock.patch.object(EOSDevice, "uptime", new_callable=mock.PropertyMock) + def test_wait_for_device_reboot_timeout(self, mock_device_uptime, mock_time): + mock_time.time.side_effect = [0, 5, 15] + mock_device_uptime.return_value = 10005 + + with self.assertRaises(RebootTimeoutError): + self.device._wait_for_device_reboot(original_uptime=10000, timeout=10) + def test_boot_options(self): boot_options = self.device.boot_options self.assertEqual(boot_options, {"sys": "EOS.swi"}) From 5a7590124962ee2dcc9c031b36e4f5bc528861c3 Mon Sep 17 00:00:00 2001 From: Gavin Acosta <155584291+Defiantearth@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:09:02 -0500 Subject: [PATCH 2/4] Using boot time instead of uptime --- pyntc/devices/eos_device.py | 44 ++++++++++++++-------- tests/unit/test_devices/test_eos_device.py | 32 +++++++++++----- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/pyntc/devices/eos_device.py b/pyntc/devices/eos_device.py index 17c193e1..31d445ae 100644 --- a/pyntc/devices/eos_device.py +++ b/pyntc/devices/eos_device.py @@ -178,34 +178,36 @@ def _uptime_to_string(self, uptime): return f"{days:02d}:{hours:02d}:{mins:02d}:{seconds:02d}" - def _wait_for_device_reboot(self, original_uptime, timeout=DEFAULT_REBOOT_TIMEOUT): - """Block until the device reboots, detected by uptime dropping below original_uptime. + def _wait_for_device_reboot(self, original_boot_time, timeout=DEFAULT_REBOOT_TIMEOUT): + """Block until the device reboots, detected by its boot time advancing past original_boot_time. Args: - original_uptime (int): Device uptime in seconds captured before the reboot. + original_boot_time (float): Device boot time (epoch seconds) captured before the reboot. timeout (int): Max seconds to poll for the device to return. Defaults to 3600. Raises: - RebootTimeoutError: When the device does not report a reset uptime within timeout. + ValueError: When original_boot_time is None (no pre-reboot boot time was captured). + RebootTimeoutError: When the device does not report a later boot time within timeout. """ + if original_boot_time is None: + raise ValueError("original_boot_time is required to detect a reboot; capture it before issuing the reload.") + start = time.time() while time.time() - start < timeout: try: - self._uptime = None # bust the cached value so we re-read from the device - current_uptime = self.uptime - if current_uptime < original_uptime: + current_boot_time = self.boot_time + if current_boot_time > original_boot_time: log.info( - "Host %s: Device rebooted (uptime %ss < pre-reboot %ss).", + "Host %s: Device rebooted (boot time %s > pre-reboot %s).", self.host, - current_uptime, - original_uptime, + current_boot_time, + original_boot_time, ) return log.debug( - "Host %s: Reachable but uptime %ss >= pre-reboot %ss; still waiting.", + "Host %s: Reachable but boot time unchanged (%s); still waiting.", self.host, - current_uptime, - original_uptime, + current_boot_time, ) except Exception as exc: # pylint: disable=broad-exception-caught log.debug("Host %s: Reboot probe failed (%s); will retry.", self.host, exc) @@ -287,6 +289,15 @@ def enable(self): log.debug("Host %s: Device enabled", self.host) + @property + def boot_time(self): + """Get the epoch timestamp (seconds) of the device's last boot, read live from ``show version``. + + Returns: + (float): Epoch seconds when the device last booted, per ``bootupTimestamp``. + """ + return self.show("show version")["bootupTimestamp"] + @property def uptime(self): """ @@ -798,7 +809,8 @@ def reboot(self, wait_for_reload=False, timeout=DEFAULT_REBOOT_TIMEOUT, **kwargs Reload the controller or controller pair. Args: - wait_for_reload (bool): Whether or not reboot method should also run _wait_for_device_reboot(). Defaults to False. + wait_for_reload (bool): When True, block until the device reboots (detected via + the pre-reboot boot time capture) before returning. Defaults to False. timeout (int): Max seconds to poll for the device to return when wait_for_reload is True. Defaults to 3600. kwargs (dict): Additional keyword arguments, such as confirm. @@ -814,11 +826,11 @@ def reboot(self, wait_for_reload=False, timeout=DEFAULT_REBOOT_TIMEOUT, **kwargs if kwargs.get("confirm"): log.warning("Passing 'confirm' to reboot method is deprecated.") - original_uptime = self.uptime if wait_for_reload else None + original_boot_time = self.boot_time if wait_for_reload else None self.show("reload now") log.info("Host %s: Device rebooted.", self.host) if wait_for_reload: - self._wait_for_device_reboot(original_uptime, timeout=timeout) + self._wait_for_device_reboot(original_boot_time, timeout=timeout) def rollback(self, rollback_to): """Rollback device configuration. diff --git a/tests/unit/test_devices/test_eos_device.py b/tests/unit/test_devices/test_eos_device.py index 517a5ed1..0a3bd790 100644 --- a/tests/unit/test_devices/test_eos_device.py +++ b/tests/unit/test_devices/test_eos_device.py @@ -287,24 +287,38 @@ def test_reboot(self): self.device.native.enable.assert_called_with(["reload now"], encoding="json") @mock.patch("pyntc.devices.eos_device.time") - @mock.patch.object(EOSDevice, "uptime", new_callable=mock.PropertyMock) - def test_wait_for_device_reboot(self, mock_device_uptime, mock_time): + @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock) + def test_wait_for_device_reboot(self, mock_boot_time, mock_time): mock_time.time.side_effect = [0, 1, 2, 3] - mock_device_uptime.side_effect = [10005, Exception("unreachable"), 12] + mock_boot_time.side_effect = [1000, Exception("unreachable"), 2000] - self.device._wait_for_device_reboot(original_uptime=10000) + self.device._wait_for_device_reboot(original_boot_time=1000) - self.assertEqual(mock_device_uptime.call_count, 3) + self.assertEqual(mock_boot_time.call_count, 3) self.assertEqual(mock_time.sleep.call_count, 2) @mock.patch("pyntc.devices.eos_device.time") - @mock.patch.object(EOSDevice, "uptime", new_callable=mock.PropertyMock) - def test_wait_for_device_reboot_timeout(self, mock_device_uptime, mock_time): + @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock) + def test_wait_for_device_reboot_low_pre_reboot_uptime(self, mock_boot_time, mock_time): + mock_time.time.side_effect = [0, 1, 2] + mock_boot_time.side_effect = [Exception("unreachable"), 2000] + + self.device._wait_for_device_reboot(original_boot_time=1000) + + self.assertEqual(mock_boot_time.call_count, 2) + + @mock.patch("pyntc.devices.eos_device.time") + @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock) + def test_wait_for_device_reboot_timeout(self, mock_boot_time, mock_time): mock_time.time.side_effect = [0, 5, 15] - mock_device_uptime.return_value = 10005 + mock_boot_time.return_value = 1000 with self.assertRaises(RebootTimeoutError): - self.device._wait_for_device_reboot(original_uptime=10000, timeout=10) + self.device._wait_for_device_reboot(original_boot_time=1000, timeout=10) + + def test_wait_for_device_reboot_requires_boot_time(self): + with self.assertRaises(ValueError): + self.device._wait_for_device_reboot(original_boot_time=None) def test_boot_options(self): boot_options = self.device.boot_options From 444f2bcc93dc5f1259e7fe1aea83aae361cf419b Mon Sep 17 00:00:00 2001 From: Gavin Acosta <155584291+Defiantearth@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:12:51 -0500 Subject: [PATCH 3/4] remove duplicate test --- tests/unit/test_devices/test_eos_device.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/unit/test_devices/test_eos_device.py b/tests/unit/test_devices/test_eos_device.py index 0a3bd790..b76a10d4 100644 --- a/tests/unit/test_devices/test_eos_device.py +++ b/tests/unit/test_devices/test_eos_device.py @@ -297,16 +297,6 @@ def test_wait_for_device_reboot(self, mock_boot_time, mock_time): self.assertEqual(mock_boot_time.call_count, 3) self.assertEqual(mock_time.sleep.call_count, 2) - @mock.patch("pyntc.devices.eos_device.time") - @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock) - def test_wait_for_device_reboot_low_pre_reboot_uptime(self, mock_boot_time, mock_time): - mock_time.time.side_effect = [0, 1, 2] - mock_boot_time.side_effect = [Exception("unreachable"), 2000] - - self.device._wait_for_device_reboot(original_boot_time=1000) - - self.assertEqual(mock_boot_time.call_count, 2) - @mock.patch("pyntc.devices.eos_device.time") @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock) def test_wait_for_device_reboot_timeout(self, mock_boot_time, mock_time): From bc63b4f7fde5ab2781f7e25f17d81b90f15ee75a Mon Sep 17 00:00:00 2001 From: Gavin Acosta <155584291+Defiantearth@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:19:06 -0500 Subject: [PATCH 4/4] add boot time test --- tests/unit/test_devices/test_eos_device.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_devices/test_eos_device.py b/tests/unit/test_devices/test_eos_device.py index b76a10d4..05f3ea5c 100644 --- a/tests/unit/test_devices/test_eos_device.py +++ b/tests/unit/test_devices/test_eos_device.py @@ -360,6 +360,12 @@ def test_uptime(self): self.assertIsInstance(uptime, int) self.assertEqual(uptime, expected) + def test_boot_time(self): + expected = self.device.show("show version")["bootupTimestamp"] + boot_time = self.device.boot_time + self.assertIsInstance(boot_time, float) + self.assertEqual(boot_time, expected) + @mock.patch.object(EOSDevice, "_uptime_to_string", autospec=True) def test_uptime_string(self, mock_upt_str): mock_upt_str.return_value = "02:00:03:38"