CSTACKEX-380: Update the automation tests to perform assertions on kvm host and libvirt

test/integration/plugins/ontap/README.md

@@ -44,6 +44,9 @@ test/integration/plugins/ontap/
 ├── ontap.cfg                     # Environment config (IPs, credentials, zone info)
 ├── ontap_test_base.py            # Shared base class and ONTAP REST client
 ├── guest_ssh.py                  # Guest SSH (paramiko) for the VM workflow suites
+├── helpers/
+│   ├── kvm_host.py               # KVM host checks: services, libvirt domain XML, by-id devices, iSCSI sessions
+│   └── template_cache_*.py       # Template-cache setup, assertions and negative workflow
 ├── setup_env.sh                  # venv: Marvin plus requirements.txt
 ├── requirements.txt              # setuptools, pyvmomi, paramiko pins
 ├── TEST_CASES.md                 # Full test case reference table (151 tests)
@@ -356,6 +359,10 @@ another ONTAP pool is present.
 | `_create_volume(pool_id)` | Creates a CloudStack data volume on the given pool |
 | `_delete_pool(pool_id, forced)` | Enters Maintenance then calls `deleteStoragePool` |
 | `_parse_pool_details(pool)` | Extracts key→value pairs from the pool's `details` list |
+| `_assert_kvm_hosts_ready()` | Called from `_setup_cloudstack_resources()`: libvirtd, cloudstack-agent (and iscsid for iSCSI) active, `virsh` responsive, CloudStack host `Up`/`Enabled`; fails the class early otherwise |
+| `_vm_placement(vm_id)` | Live host IP, libvirt instance name and VM object (deploy-time values go stale after stop/start) |
+| `_assert_vm_disks_on_host(vm_id, pool, label)` | iSCSI: libvirt domain `running`; XML identity/sizing; one `block`/`raw`/`virtio` disk per CloudStack volume sourced from `/dev/disk/by-id/wwn-0x<WWID>`; volume path is `/<target IQN>/<WWID>` and the WWID matches the ONTAP LUN serial; LUN mapped to the host's igroup; block device present; iSCSI session to an SVM data LIF |
+| `_assert_disk_detached_on_host(...)`, `_assert_devices_gone(...)`, `_assert_domain_absent(...)` | After detach / stop / destroy: the disk is out of the domain XML, the by-id device and the libvirt domain are gone |
 | `OntapRestClient` | Thin HTTPS client for ONTAP REST API calls |
 
 ### `OntapRestClient` methods at a glance

test/integration/plugins/ontap/helpers/kvm_host.py

@@ -23,12 +23,22 @@
 
 import json
 import logging
+import re
 import shlex
+import time
+import xml.etree.ElementTree as ET
 
 from marvin.sshClient import SshClient
 
 logger = logging.getLogger("kvm_host")
 
+# '6' (NAA IEEE Registered Extended) + NetApp OUI '00a098' + pad nibble '0'.
+# Mirrors OntapStorageConstants.NETAPP_NAA_PREFIX in the ONTAP plugin. It is
+# deliberately re-implemented here (not read from the plugin) so a mistake in
+# the plugin's serial -> WWID mapping is caught by the tests.
+NETAPP_NAA_PREFIX = "600a0980"
+WWID_RE = re.compile(r"^%s[0-9a-f]{24}$" % NETAPP_NAA_PREFIX)
+
 
 def cleanup_kvm_storage_pool_mounts(kvm_hosts_ssh_creds, pool_uuid):
     """Unmount and undefine the libvirt pool for ``pool_uuid`` on each host.
@@ -135,3 +145,165 @@ def get_vm_data_disk_capacity(kvm_hosts_ssh_creds, instance_name):
         "Could not read VM data-disk capacity from any KVM host: %s"
         % "; ".join(errors)
     )
+
+
+# ---------------------------------------------------------------------------
+# Host-side verification helpers (services, libvirt domain, iSCSI devices)
+#
+# Marvin's SshClient.execute() ignores exit codes and returns stderr only when
+# stdout is empty, so every command below merges stderr (2>&1) or prints an
+# explicit sentinel and the callers look for that text.
+# ---------------------------------------------------------------------------
+
+def naa_from_serial(serial):
+    """Return the SCSI WWID the plugin derives from an ONTAP LUN serial number.
+
+    The WWID is NETAPP_NAA_PREFIX followed by the hex of the 12 ASCII serial
+    bytes, e.g. serial ``x0M-8?/kEsJT`` ->
+    ``600a098078304d2d383f2f6b45734a54``.
+    """
+    if not serial:
+        raise ValueError("ONTAP reported no LUN serial number")
+    return NETAPP_NAA_PREFIX + "".join(
+        "%02x" % b for b in serial.encode("ascii"))
+
+
+def by_id_paths(wwid):
+    """The two /dev/disk/by-id aliases udev creates for a NetApp LUN WWID."""
+    return ["/dev/disk/by-id/wwn-0x%s" % wwid,
+            "/dev/disk/by-id/scsi-3%s" % wwid]
+
+
+def _creds_for(kvm_hosts_ssh_creds, host_ip):
+    for creds in kvm_hosts_ssh_creds:
+        if creds.get("host") == host_ip:
+            return creds
+    raise RuntimeError("No SSH credentials for KVM host %s" % host_ip)
+
+
+def run_on_host(kvm_hosts_ssh_creds, host_ip, command, timeout=20.0):
+    """Run one command on a KVM host and return its output lines."""
+    creds = _creds_for(kvm_hosts_ssh_creds, host_ip)
+    ssh = SshClient(
+        host_ip, 22, creds["user"], creds["password"],
+        retries=3, delay=3, timeout=timeout,
+    )
+    return ssh.execute(command) or []
+
+
+def service_state(kvm_hosts_ssh_creds, host_ip, unit):
+    """``systemctl is-active`` for a unit ('active', 'inactive', 'failed')."""
+    out = run_on_host(
+        kvm_hosts_ssh_creds, host_ip,
+        "systemctl is-active %s 2>&1; true" % shlex.quote(unit))
+    return out[0].strip() if out else "unknown"
+
+
+def virsh_responsive(kvm_hosts_ssh_creds, host_ip):
+    """True when libvirt answers a plain ``virsh list`` within 15 seconds."""
+    out = run_on_host(
+        kvm_hosts_ssh_creds, host_ip,
+        "timeout 15 virsh list >/dev/null 2>&1 "
+        "&& echo VIRSH_OK || echo VIRSH_FAIL")
+    return "VIRSH_OK" in "".join(out)
+
+
+def domain_state(kvm_hosts_ssh_creds, host_ip, instance_name):
+    """libvirt state of a domain: 'running', 'shut off', ... or 'absent'.
+
+    CloudStack defines its guests as transient domains, so a stopped or
+    destroyed VM is 'absent' rather than 'shut off'.
+    """
+    out = run_on_host(
+        kvm_hosts_ssh_creds, host_ip,
+        "timeout 15 virsh domstate %s 2>&1; true"
+        % shlex.quote(str(instance_name)))
+    text = " ".join(line.strip() for line in out).strip()
+    lowered = text.lower()
+    if "failed to get domain" in lowered or "domain not found" in lowered:
+        return "absent"
+    return text
+
+
+def domain_xml(kvm_hosts_ssh_creds, host_ip, instance_name):
+    """Return the parsed ``virsh dumpxml`` root element, or None if absent."""
+    out = run_on_host(
+        kvm_hosts_ssh_creds, host_ip,
+        "timeout 15 virsh dumpxml %s 2>&1; true"
+        % shlex.quote(str(instance_name)))
+    text = "\n".join(out)
+    start = text.find("<domain")
+    if start < 0:
+        return None
+    return ET.fromstring(text[start:])
+
+
+def parse_domain_disks(domain_root):
+    """One dict per ``<disk>`` of a libvirt domain XML (any device type)."""
+    disks = []
+    for disk in domain_root.findall("./devices/disk"):
+        driver = disk.find("driver")
+        source = disk.find("source")
+        target = disk.find("target")
+        disks.append({
+            "device": disk.get("device"),
+            "type": disk.get("type"),
+            "driver_name": driver.get("name") if driver is not None else None,
+            "driver_type": driver.get("type") if driver is not None else None,
+            "cache": driver.get("cache") if driver is not None else None,
+            "source_dev": source.get("dev") if source is not None else None,
+            "source_file": source.get("file") if source is not None else None,
+            "target_dev": target.get("dev") if target is not None else None,
+            "target_bus": target.get("bus") if target is not None else None,
+            "serial": disk.findtext("serial"),
+        })
+    return disks
+
+
+def device_present(kvm_hosts_ssh_creds, host_ip, wwid):
+    """True when a block device for the WWID exists under /dev/disk/by-id."""
+    wwn, scsi = by_id_paths(wwid)
+    out = run_on_host(
+        kvm_hosts_ssh_creds, host_ip,
+        "test -b %s -o -b %s && echo DEV_YES || echo DEV_NO"
+        % (shlex.quote(wwn), shlex.quote(scsi)))
+    return "DEV_YES" in "".join(out)
+
+
+def wait_device(kvm_hosts_ssh_creds, host_ip, wwid, present,
+                timeout=90, interval=3):
+    """Poll until the WWID's device appears/disappears; return the final state.
+
+    udev and the plugin's SCSI rescan/removal are asynchronous, so a single
+    immediate check would be flaky.
+    """
+    deadline = time.time() + timeout
+    state = device_present(kvm_hosts_ssh_creds, host_ip, wwid)
+    while state != present and time.time() < deadline:
+        time.sleep(interval)
+        state = device_present(kvm_hosts_ssh_creds, host_ip, wwid)
+    return state
+
+
+_SESSION_RE = re.compile(
+    r"^\w+:\s*\[(\d+)\]\s+([^,\s]+):(\d+),(\d+)\s+(\S+)")
+
+
+def iscsi_sessions(kvm_hosts_ssh_creds, host_ip):
+    """Parse ``iscsiadm -m session`` into dicts (sid, portal_ip, port, target).
+
+    Returns [] when there are no sessions.
+    """
+    out = run_on_host(
+        kvm_hosts_ssh_creds, host_ip, "iscsiadm -m session 2>&1; true")
+    sessions = []
+    for line in out:
+        match = _SESSION_RE.match(line.strip())
+        if match:
+            sessions.append({
+                "sid": match.group(1),
+                "portal_ip": match.group(2),
+                "port": match.group(3),
+                "target": match.group(5),
+            })
+    return sessions

test/integration/plugins/ontap/helpers/template_cache_negative_workflow.py

@@ -617,8 +617,38 @@ def step_03_deleted_cache_blocks_reuse(self):
             )
             self._assert_cache_on_ontap(pool, spool)
 
+            # iSCSI only (NFS has no LUNs, igroups or block devices).
+            seed_host = None
+            if self.PROTOCOL.upper() == "ISCSI":
+                # KVM host / libvirt: the seed VM runs from a by-id block disk
+                # (the ROOT LUN, a clone of the cache LUN) with the right XML,
+                # igroup and session.
+                seed_host = self._assert_vm_disks_on_host(
+                    vm1.id, pool, "seed VM")
+                # ONTAP: the cache LUN is only mapped while the template is
+                # copied into it; once the VM runs it must not stay mapped to
+                # any host.
+                cache_lun = "/vol/%s/cs_tmpl_%s" % (
+                    pool.name, self.__class__.template_db_id)
+                stray_maps = [
+                    m for m in self.ontap.list_lun_maps_for_volume(
+                        self.svm_name, pool.name)
+                    if m.get("lun", {}).get("name") == cache_lun]
+                self.assertEqual(
+                    stray_maps, [],
+                    "Template cache LUN %s is still mapped after the seed VM "
+                    "started: %s" % (cache_lun, stray_maps))
+
             self._destroy_vm_static(vm1.id)
             vm1 = None
+            # KVM host: the seed VM's domain and ROOT device are released
+            if seed_host:
+                self._assert_domain_absent(
+                    seed_host["host_ip"], seed_host["instance"],
+                    "seed VM destroyed")
+                self._assert_devices_gone(
+                    seed_host["host_ip"], list(seed_host["wwids"].values()),
+                    "seed VM destroyed")
             # Allow clone/ROOT LUN cleanup so the cache LUN is no longer a
             # FlexClone parent before out-of-band delete.
             time.sleep(30)