Fixed Arista EOS reboots not being detected when waiting for the device to reload

changes/407.fixed

@@ -0,0 +1 @@
+Fixed Arista EOS reboots not being detected when waiting for the device to reload.

pyntc/devices/eos_device.py

@@ -27,6 +27,7 @@
 
 EOS_SUPPORTED_HASHING_ALGORITHMS = {"md5", "sha1", "sha256", "sha512"}  # Subset of HASHING_ALGORITHMS for EOS verify
 EOS_SUPPORTED_SCHEMES = {"http", "https", "scp", "ftp", "sftp", "tftp"}
+DEFAULT_REBOOT_TIMEOUT = 3600  # seconds (1 hour) to wait for a device to come back after a reboot
 BASIC_FACTS_KM = {"model": "modelName", "os_version": "internalVersion", "serial_number": "serialNumber"}
 INTERFACES_KM = {
     "speed": "bandwidth",
@@ -177,15 +178,40 @@ def _uptime_to_string(self, uptime):
 
         return f"{days:02d}:{hours:02d}:{mins:02d}:{seconds:02d}"
 
-    def _wait_for_device_reboot(self, timeout=3600):
+    def _wait_for_device_reboot(self, original_boot_time, timeout=DEFAULT_REBOOT_TIMEOUT):
+        """Block until the device reboots, detected by its boot time advancing past original_boot_time.
+
+        Args:
+            original_boot_time (float): Device boot time (epoch seconds) captured before the reboot.
+            timeout (int): Max seconds to poll for the device to return. Defaults to 3600.
+
+        Raises:
+            ValueError: When original_boot_time is None (no pre-reboot boot time was captured).
+            RebootTimeoutError: When the device does not report a later boot time within timeout.
+        """
+        if original_boot_time is None:
+            raise ValueError("original_boot_time is required to detect a reboot; capture it before issuing the reload.")
+
         start = time.time()
         while time.time() - start < timeout:
             try:
-                self.show("show hostname")
-                log.debug("Host %s: Device rebooted.", self.host)
-                return
-            except:  # noqa E722 # nosec  # pylint: disable=bare-except
-                time.sleep(10)
+                current_boot_time = self.boot_time
+                if current_boot_time > original_boot_time:
+                    log.info(
+                        "Host %s: Device rebooted (boot time %s > pre-reboot %s).",
+                        self.host,
+                        current_boot_time,
+                        original_boot_time,
+                    )
+                    return
+                log.debug(
+                    "Host %s: Reachable but boot time unchanged (%s); still waiting.",
+                    self.host,
+                    current_boot_time,
+                )
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                log.debug("Host %s: Reboot probe failed (%s); will retry.", self.host, exc)
+            time.sleep(10)
 
         log.error("Host %s: Device timed out while rebooting.", self.host)
         raise RebootTimeoutError(hostname=self.hostname, wait_time=timeout)
@@ -263,6 +289,15 @@ def enable(self):
 
         log.debug("Host %s: Device enabled", self.host)
 
+    @property
+    def boot_time(self):
+        """Get the epoch timestamp (seconds) of the device's last boot, read live from ``show version``.
+
+        Returns:
+            (float): Epoch seconds when the device last booted, per ``bootupTimestamp``.
+        """
+        return self.show("show version")["bootupTimestamp"]
+
     @property
     def uptime(self):
         """
@@ -729,14 +764,13 @@ def install_os(self, image_name, reboot=True, **vendor_specifics):
         Returns:
             (bool): True if device OS is succesfully installed.
         """
-        timeout = vendor_specifics.get("timeout", 3600)
+        timeout = vendor_specifics.get("timeout", DEFAULT_REBOOT_TIMEOUT)
         if not self._image_booted(image_name):
             self.set_boot_options(image_name, **vendor_specifics)
             if not reboot:
                 log.info("Host %s: OS image %s boot options set. Reboot the device to apply", self.host, image_name)
                 return True
-            self.reboot()
-            self._wait_for_device_reboot(timeout=timeout)
+            self.reboot(wait_for_reload=True, timeout=timeout)
             if not self._image_booted(image_name):
                 log.error("Host %s: OS install error for image %s", self.host, image_name)
                 raise OSInstallError(hostname=self.hostname, desired_boot=image_name)
@@ -770,12 +804,14 @@ def open(self):
 
         log.debug("Host %s: Connection to controller was opened successfully.", self.host)
 
-    def reboot(self, wait_for_reload=False, **kwargs):
+    def reboot(self, wait_for_reload=False, timeout=DEFAULT_REBOOT_TIMEOUT, **kwargs):
         """
         Reload the controller or controller pair.
 
         Args:
-            wait_for_reload (bool): Whether or not reboot method should also run _wait_for_device_reboot(). Defaults to False.
+            wait_for_reload (bool): When True, block until the device reboots (detected via
+                the pre-reboot boot time capture) before returning. Defaults to False.
+            timeout (int): Max seconds to poll for the device to return when wait_for_reload is True. Defaults to 3600.
             kwargs (dict): Additional keyword arguments, such as confirm.
 
         Raises:
@@ -790,10 +826,11 @@ def reboot(self, wait_for_reload=False, **kwargs):
         if kwargs.get("confirm"):
             log.warning("Passing 'confirm' to reboot method is deprecated.")
 
+        original_boot_time = self.boot_time if wait_for_reload else None
         self.show("reload now")
         log.info("Host %s: Device rebooted.", self.host)
         if wait_for_reload:
-            self._wait_for_device_reboot()
+            self._wait_for_device_reboot(original_boot_time, timeout=timeout)
 
     def rollback(self, rollback_to):
         """Rollback device configuration.

tests/unit/test_devices/test_eos_device.py

@@ -9,7 +9,7 @@
 from pyntc.devices.base_device import RollbackError
 from pyntc.devices.eos_device import FileTransferError
 from pyntc.devices.system_features.vlans.eos_vlans import EOSVlans
-from pyntc.errors import CommandError, CommandListError, NotEnoughFreeSpaceError  # noqa: F401
+from pyntc.errors import CommandError, CommandListError, NotEnoughFreeSpaceError, RebootTimeoutError  # noqa: F401
 from pyntc.utils.models import FileCopyModel
 
 from .device_mocks.eos import config, enable, send_command, send_command_expect
@@ -286,6 +286,30 @@ def test_reboot(self):
         self.device.reboot()
         self.device.native.enable.assert_called_with(["reload now"], encoding="json")
 
+    @mock.patch("pyntc.devices.eos_device.time")
+    @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock)
+    def test_wait_for_device_reboot(self, mock_boot_time, mock_time):
+        mock_time.time.side_effect = [0, 1, 2, 3]
+        mock_boot_time.side_effect = [1000, Exception("unreachable"), 2000]
+
+        self.device._wait_for_device_reboot(original_boot_time=1000)
+
+        self.assertEqual(mock_boot_time.call_count, 3)
+        self.assertEqual(mock_time.sleep.call_count, 2)
+
+    @mock.patch("pyntc.devices.eos_device.time")
+    @mock.patch.object(EOSDevice, "boot_time", new_callable=mock.PropertyMock)
+    def test_wait_for_device_reboot_timeout(self, mock_boot_time, mock_time):
+        mock_time.time.side_effect = [0, 5, 15]
+        mock_boot_time.return_value = 1000
+
+        with self.assertRaises(RebootTimeoutError):
+            self.device._wait_for_device_reboot(original_boot_time=1000, timeout=10)
+
+    def test_wait_for_device_reboot_requires_boot_time(self):
+        with self.assertRaises(ValueError):
+            self.device._wait_for_device_reboot(original_boot_time=None)
+
     def test_boot_options(self):
         boot_options = self.device.boot_options
         self.assertEqual(boot_options, {"sys": "EOS.swi"})
@@ -336,6 +360,12 @@ def test_uptime(self):
         self.assertIsInstance(uptime, int)
         self.assertEqual(uptime, expected)
 
+    def test_boot_time(self):
+        expected = self.device.show("show version")["bootupTimestamp"]
+        boot_time = self.device.boot_time
+        self.assertIsInstance(boot_time, float)
+        self.assertEqual(boot_time, expected)
+
     @mock.patch.object(EOSDevice, "_uptime_to_string", autospec=True)
     def test_uptime_string(self, mock_upt_str):
         mock_upt_str.return_value = "02:00:03:38"