diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index 6e0d0e7d6be5..60c5e5805a2b 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -32,7 +32,7 @@ CI wiring: test/integration/plugins/ontap/ ├── ontap.cfg # Environment config (IPs, credentials, zone info) ├── ontap_test_base.py # Shared base class and ONTAP REST client -├── TEST_CASES.md # Full test case reference table (62 tests) +├── TEST_CASES.md # Full test case reference table (87 tests) ├── README.md # This file │ ├── nfs3/ @@ -69,7 +69,7 @@ The ONTAP plugin (`plugins/storage/volume/ontap/`) integrates CloudStack's prima | Aspect | NFS3 | iSCSI | |--------|------|-------| -| ONTAP object per pool | FlexVol + export policy | FlexVol + igroup per KVM host | +| ONTAP object per pool | FlexVol + export policy | FlexVol; igroups are shared per KVM host and SVM | | ONTAP object per CS volume | None (FlexVol is shared) | One LUN inside the FlexVol | | Host connectivity | NFS mount | iSCSI login (IQN-based) | | Volume detach from running VM | Works via virtio hot-unplug | Requires KVM guest to support SCSI hot-unplug | @@ -244,14 +244,16 @@ pool = self.__class__.pool self.pool = pool ``` -**Guard assertion at the start of every test (except test_01)** +**Guard assertion at the start of every test that depends on a previous one** -Every test after the first starts with an assertion that the previous step's resource exists. This produces a clear, readable failure message instead of a confusing `AttributeError`: +Every test in a sequential workflow starts with an assertion that the previous step's resource exists. This produces a clear, readable failure message instead of a confusing `AttributeError`: ```python -def test_03_enable_storage_pool(self): - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") +def test_05_enable_storage_pool(self): + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") ``` +Isolated negative tests own everything they create, so they carry no such guard. In the pool lifecycle and zone-scoped suites the two create-rejection negatives are numbered `test_01` and `test_02` so that a misconfigured SVM fails within a minute rather than after the full workflow. + **Creating a storage pool — always use indexed `details[N].key` syntax** The CloudStack API for `createStoragePool` requires plugin details to be passed as indexed parameters. **Never call `StoragePool.create()` directly** — it does not support this syntax: @@ -270,6 +272,15 @@ result = self._poll_pool_state(pool.id, "Maintenance", timeout=120) self.assertEqual(result.state, "Maintenance") ``` +**Shared iSCSI igroups** + +iSCSI igroups are named from the host UUID and SVM, not from the storage pool. +Each iSCSI suite snapshots existing host igroups during `setUpClass()` and +checks that pool-only operations preserve that baseline. Tests may therefore +run while another ONTAP iSCSI pool uses the same SVM. The negative tests that +deliberately delete igroups still require exclusive SVM use and skip when +another ONTAP pool is present. + --- ## Shared base — `ontap_test_base.py` @@ -295,7 +306,7 @@ self.assertEqual(result.state, "Maintenance") | `get_data_lifs(svm_name)` | NFS data LIF count | NFS3 pool lifecycle | | `get_igroup(svm_name, name)` | iSCSI igroup existence and initiator list | iSCSI suites | | `list_luns_in_volume(svm_name, vol_name)` | LUNs present in a FlexVol | iSCSI volume/instance suites | -| `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI instance suite | +| `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI pool-with-volumes/instance suites | | `list_files_in_volume(svm_name, vol_name)` | Files inside a FlexVol | NFS3 instance suite | --- @@ -304,14 +315,14 @@ self.assertEqual(result.state, "Maintenance") | Suite | File | Tests | What it covers | |-------|------|-------|---------------| -| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete | -| NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 7 | Same + live volume present; negative delete guard | -| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone scope — all hosts connected via `attachZone` | +| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 12 | Existing lifecycle plus duplicate-name and aggregate-space create rejects, and empty-pool deletion with pre-deleted FlexVol/export policy | +| NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 10 | Existing lifecycle plus deletion with pre-deleted FlexVol/export policy and cancel-maintenance after CS volume deletion | +| NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 8 | Zone lifecycle plus duplicate-name/aggregate-space create rejects and pre-deleted FlexVol/export policy deletes | | NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | | NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle with hot-plug/detach | -| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete + igroups | -| iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 7 | Same + live LUN present; negative delete guard | -| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone scope | +| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 12 | Existing lifecycle plus duplicate-name and aggregate-space create rejects, and empty-pool deletion with pre-deleted FlexVol/igroups | +| iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 11 | Existing lifecycle plus deletion with pre-deleted FlexVol/igroups, maintenance with pre-deleted LUN maps, and cancel-maintenance after CS volume deletion | +| iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 8 | Zone lifecycle plus duplicate-name/aggregate-space create rejects and pre-deleted FlexVol/igroup deletes | | iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | | iSCSI VM + Volume Attach | `iscsi/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle; LUN-maps on VM start/stop/detach | diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md index 73dc1990a5b6..5a26d839b0d9 100644 --- a/test/integration/plugins/ontap/TEST_CASES.md +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -19,7 +19,7 @@ # ONTAP Integration Test Cases -Complete reference for all 62 test cases across 10 test suites. +Complete reference for all 87 test cases across 10 test suites. Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. --- @@ -42,18 +42,22 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_pool_lifecycle.py` **Class:** `TestOntapNFS3PrimaryStorageWorkflow` **Tag:** `nfs3_workflow` -**Total:** 8 tests | **Scope:** cluster-scoped NFS3 pool, no volumes for tests 01–06 +**Total:** 12 tests | **Scope:** cluster-scoped NFS3 pool, no-volume workflow through test 08 and isolated negative tests 01–02 and 11–12 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | -| 02 | `test_02_disable_storage_pool` | Disable the pool (admin operation) | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | -| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | -| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_04 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 06 | `test_06_delete_pool_from_maintenance` | Enter maintenance then permanently delete the pool | test_05 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol deleted (not found by `GET /api/storage/volumes?name=`); export policy deleted | positive | -| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume on it | test_06 (pool deleted; creates new pool) | New `pool.state == "Up"`; `createVolume` returns non-None volume object | FlexVol `online` after volume allocation; export policy present | positive | -| 08 | `test_08_delete_volume_and_pool` | Delete the volume then force-delete the pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 01 | `test_01_reject_create_when_flexvol_name_exists` | Reject pool creation when ONTAP already has a FlexVol with the requested name | isolated | `CloudstackAPIException`; no CS pool created | Pre-existing FlexVol remains until test cleanup | negative | +| 02 | `test_02_reject_create_when_no_aggregate_space` | Reject pool creation when requested capacity exceeds every assigned online aggregate's available space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no CS pool created | No FlexVol created | negative | +| 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | +| 04 | `test_04_disable_storage_pool` | Disable the pool (admin operation) | test_06 | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | +| 05 | `test_05_enable_storage_pool` | Re-enable the pool | test_08 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 06 | `test_06_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_05 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | +| 07 | `test_07_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_11 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 08 | `test_08_delete_pool_from_maintenance` | Enter maintenance then permanently delete the original pool | test_07 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol and export policy deleted | positive | +| 09 | `test_09_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack data volume | test_08 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; export policy present | positive | +| 10 | `test_10_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_17 | VM destroyed; volume and pool no longer listed | FlexVol and export policy deleted | cleanup | +| 11 | `test_11_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; export policy cleaned up | negative | +| 12 | `test_12_delete_pool_with_export_policy_predeleted` | Delete an empty pool after its export policy was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; export policy remains absent | negative | --- @@ -62,7 +66,7 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_pool_with_volumes.py` **Class:** `TestOntapNFS3PoolWithVolumes` **Tag:** `nfs3_with_volumes` -**Total:** 7 tests | **Scope:** cluster-scoped NFS3 pool with a live CloudStack volume throughout +**Total:** 10 tests | **Scope:** cluster-scoped NFS3 pool with a live CloudStack volume, plus isolated negative workflows | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| @@ -73,6 +77,9 @@ Each suite is sequential — tests must run in numbered order; each step builds | 05 | `test_05_cancel_maintenance_with_volume` | Cancel maintenance with volume — verifies the NFS3 cancel-maintenance fix | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online` | positive | | 06 | `test_06_forced_false_delete_rejected` | Attempt to delete pool (forced=False) with volume present — must be rejected | test_05 | `deleteStoragePool(forced=False)` raises `CloudstackAPIException`; pool still listed in `Maintenance` state | FlexVol still `online`; no ONTAP objects removed | negative | | 07 | `test_07_force_delete_pool_and_cleanup` | Cancel maintenance, delete volume, then force-delete pool | test_06 | Pool no longer listed; volume no longer listed | FlexVol deleted; export policy deleted | cleanup | +| 08 | `test_08_delete_pool_with_volume_flexvol_missing` | Force-delete a pool with a CS volume after its FlexVol was removed directly from ONTAP | isolated | Pool removed; leftover volume record cleaned | FlexVol remains absent | negative | +| 09 | `test_09_delete_pool_with_volume_export_policy_missing` | Force-delete a pool with a CS volume after its export policy was removed directly from ONTAP | isolated | Pool removed; leftover volume record cleaned | FlexVol deleted; export policy remains absent | negative | +| 10 | `test_10_cancel_maintenance_after_volume_deleted` | Cancel maintenance after the pool's CS volume has been deleted | isolated | Pool returns to `Up`; volume absent | FlexVol online; export policy present | negative | --- @@ -81,14 +88,18 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_zone_scoped_pool.py` **Class:** `TestOntapZoneScopedPool` **Tag:** `zone_pool` -**Total:** 4 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) +**Total:** 8 tests | **Scope:** zone-scoped NFS3 pool (scope=ZONE, all hosts in zone connected) | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped NFS3 pool; CloudStack calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; export policy exists and contains **every** cluster host IP; at least one NFS data LIF present | positive | -| 02 | `test_02_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | -| 03 | `test_03_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | -| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_03 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 01 | `test_01_create_zone_pool_rejected_when_flexvol_exists` | Reject zone-scoped pool creation when a same-name FlexVol exists | isolated | `CloudstackAPIException`; no CS pool created | Pre-existing FlexVol preserved until cleanup | negative | +| 02 | `test_02_create_zone_pool_rejected_when_no_aggregate_space` | Reject creation when requested capacity exceeds every online aggregate's free space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no pool created | No FlexVol created | negative | +| 03 | `test_03_create_zone_scoped_pool` | Create a zone-scoped NFS3 pool; CloudStack calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; export policy exists and contains **every** cluster host IP; at least one NFS data LIF present | positive | +| 04 | `test_04_disable_zone_scoped_pool` | Disable the zone-scoped pool | test_05 | `pool.state == "Disabled"` | FlexVol unchanged; export policy unchanged | positive | +| 05 | `test_05_enable_zone_scoped_pool` | Re-enable the zone-scoped pool | test_04 | `pool.state == "Up"` | FlexVol unchanged; export policy unchanged | positive | +| 06 | `test_06_delete_zone_scoped_pool` | Enter maintenance and force-delete the zone-scoped pool | test_05 | Pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 07 | `test_07_delete_zone_pool_with_flexvol_predeleted` | Delete an empty zone pool after its FlexVol was removed directly | isolated | Pool removed | FlexVol remains absent; export policy cleaned up | negative | +| 08 | `test_08_delete_zone_pool_with_export_policy_predeleted` | Delete an empty zone pool after its export policy was removed directly | isolated | Pool removed | FlexVol deleted; export policy remains absent | negative | --- @@ -124,7 +135,7 @@ Each suite is sequential — tests must run in numbered order; each step builds | 04 | `test_04_attach_volume_to_vm` | Attach the ONTAP data volume to the running VM (hot-plug) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id`; `attachVolume` job succeeds | FlexVol `online`; after attach, a data file matching volume UUID present in FlexVol (`list_files_in_volume`) | positive | | 05 | `test_05_stop_vm_export_retained` | Stop the running VM with volume attached | test_04 | `vm.state == "Stopped"` | FlexVol still `online`; NFS export policy still present | positive | | 06 | `test_06_start_vm_volume_accessible` | Start the stopped VM | test_05 | `vm.state == "Running"` | FlexVol still `online` | positive | -| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM (TDS Detach NFS3) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | | 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM no longer listed; volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | cleanup | --- @@ -134,18 +145,22 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_pool_lifecycle.py` **Class:** `TestOntapISCSIPoolLifecycle` **Tag:** `iscsi_workflow` -**Total:** 8 tests | **Scope:** cluster-scoped iSCSI pool, no volumes for tests 01–06 +**Total:** 12 tests | **Scope:** cluster-scoped iSCSI pool, no-volume workflow through test 08 and isolated negative tests 01–02 and 11–12 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | -| 02 | `test_02_disable_storage_pool` | Disable the pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online` | positive | -| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | -| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance | test_04 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 06 | `test_06_enter_maintenance_and_delete_pool` | Enter maintenance then force-delete the pool | test_05 | Pool no longer listed | FlexVol deleted; all igroups for cluster hosts deleted | positive | -| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume (creates a LUN) | test_06 (new pool) | New `pool.state == "Up"`; volume object non-None | FlexVol `online`; ≥1 LUN present inside FlexVol (`list_luns_in_volume`) | positive | -| 08 | `test_08_delete_volume_and_pool` | Delete the volume (removes LUN), enter maintenance, force-delete pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | LUN no longer in FlexVol; FlexVol deleted; igroups deleted | positive | +| 01 | `test_01_reject_create_when_flexvol_name_exists` | Reject pool creation when ONTAP already has a FlexVol with the requested name | isolated | `CloudstackAPIException`; no CS pool created | Pre-existing FlexVol remains until cleanup | negative | +| 02 | `test_02_reject_create_when_no_aggregate_space` | Reject pool creation when requested capacity exceeds every assigned online aggregate's available space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no CS pool created | No FlexVol created | negative | +| 03 | `test_03_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; shared `cs_{hostUuid}_{svmName}` igroups unchanged from suite-start baseline | positive | +| 04 | `test_04_disable_storage_pool` | Disable the pool | test_06 | `pool.state == "Disabled"` | FlexVol still `online` | positive | +| 05 | `test_05_enable_storage_pool` | Re-enable the pool | test_08 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 06 | `test_06_enter_maintenance_mode` | Put pool into maintenance | test_05 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | +| 07 | `test_07_cancel_maintenance_mode` | Cancel maintenance | test_11 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 08 | `test_08_enter_maintenance_and_delete_pool` | Enter maintenance then delete the original pool | test_07 | Pool no longer listed | FlexVol and test-pool LUN maps deleted; shared igroup baseline restored | positive | +| 09 | `test_09_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack volume | test_08 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; at least one LUN present | positive | +| 10 | `test_10_delete_volume_and_pool` | Detach and destroy the VM, delete the volume, then force-delete the pool | test_17 | VM destroyed; volume and pool no longer listed | LUN, FlexVol, and test-pool maps deleted; shared igroup baseline restored | cleanup | +| 11 | `test_11_delete_pool_with_flexvol_predeleted` | Delete an empty pool after its FlexVol was removed directly from ONTAP | isolated | Pool removed successfully | FlexVol remains absent; igroups cleaned up | negative | +| 12 | `test_12_delete_pool_with_igroups_predeleted` | Delete an empty pool after host igroups were removed directly from ONTAP | isolated | Pool removed successfully | FlexVol deleted; igroups remain absent | negative | --- @@ -154,7 +169,7 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_pool_with_volumes.py` **Class:** `TestOntapISCSIPoolWithVolumes` **Tag:** `iscsi_workflow` -**Total:** 7 tests | **Scope:** cluster-scoped iSCSI pool with a live CloudStack volume (LUN) throughout +**Total:** 11 tests | **Scope:** cluster-scoped iSCSI pool with a live CloudStack volume (LUN), plus isolated negative workflows | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| @@ -162,9 +177,13 @@ Each suite is sequential — tests must run in numbered order; each step builds | 02 | `test_02_disable_pool_volume_survives` | Disable pool with volume present | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance with volume | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online`; LUN still present | positive | -| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume (TDS iSCSI cancel maintenance) | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 06 | `test_06_forced_false_delete_rejected` | Attempt `deleteStoragePool(forced=False)` with LUN-backed volume present — must be rejected | test_05 | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | -| 07 | `test_07_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_06 (`pool`, `volume`) | Volume gone; pool gone | LUN removed; FlexVol deleted; igroups deleted | cleanup | +| 07 | `test_07_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_06 (`pool`, `volume`) | Volume gone; pool gone | LUN and FlexVol deleted; shared igroup baseline restored | cleanup | +| 08 | `test_08_delete_pool_with_volume_flexvol_missing` | Force-delete a pool with a CS volume after its FlexVol and LUN were removed directly | isolated | Pool removed; leftover volume record cleaned | FlexVol and LUN remain absent | negative | +| 09 | `test_09_delete_pool_with_volume_igroups_missing` | Force-delete a pool with a CS volume after host igroups were removed directly | isolated | Pool removed; leftover volume record cleaned | FlexVol deleted; igroups remain absent | negative | +| 10 | `test_10_enter_maintenance_lun_maps_predeleted` | Enter maintenance after LUN maps were removed directly on ONTAP | isolated | Pool reaches Maintenance | LUN maps remain absent | negative | +| 11 | `test_11_cancel_maintenance_after_volume_deleted` | Cancel maintenance after the pool's CS volume has been deleted | isolated | Pool returns to `Up`; volume absent | LUN absent; FlexVol online | negative | --- @@ -173,14 +192,18 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_zone_scoped_pool.py` **Class:** `TestOntapISCSIZoneScopedPool` **Tag:** `iscsi_zone_pool` -**Total:** 4 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) +**Total:** 8 tests | **Scope:** zone-scoped iSCSI pool (scope=ZONE) | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_zone_scoped_pool` | Create a zone-scoped iSCSI pool; CS calls `attachZone()` to connect all eligible KVM hosts | setUpClass | `pool.state == "Up"` | FlexVol `online`; igroup per cluster host, each with host IQN as initiator | positive | -| 02 | `test_02_disable_zone_scoped_pool` | Disable pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | -| 03 | `test_03_enable_zone_scoped_pool` | Re-enable pool | test_02 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | -| 04 | `test_04_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_03 | Pool no longer listed | FlexVol deleted; all igroups deleted | positive | +| 01 | `test_01_create_zone_pool_rejected_when_flexvol_exists` | Reject zone-scoped pool creation when a same-name FlexVol exists | isolated | `CloudstackAPIException`; no CS pool created | Pre-existing FlexVol preserved until cleanup | negative | +| 02 | `test_02_create_zone_pool_rejected_when_no_aggregate_space` | Reject creation when requested capacity exceeds every online aggregate's free space | isolated | `CloudstackAPIException` containing `No suitable aggregates`; no pool created | No FlexVol created | negative | +| 03 | `test_03_create_zone_scoped_pool` | Create a zone-scoped iSCSI pool | setUpClass | `pool.state == "Up"` | FlexVol `online`; shared host igroups unchanged from suite-start baseline | positive | +| 04 | `test_04_disable_zone_scoped_pool` | Disable pool | test_05 | `pool.state == "Disabled"` | FlexVol unchanged; igroups unchanged | positive | +| 05 | `test_05_enable_zone_scoped_pool` | Re-enable pool | test_04 | `pool.state == "Up"` | FlexVol unchanged; igroups unchanged | positive | +| 06 | `test_06_delete_zone_scoped_pool` | Enter maintenance then delete pool | test_05 | Pool no longer listed | FlexVol and test-pool maps deleted; shared igroup baseline restored | positive | +| 07 | `test_07_delete_zone_pool_with_flexvol_predeleted` | Delete an empty zone pool after its FlexVol was removed directly | isolated | Pool removed | FlexVol remains absent; igroups cleaned up | negative | +| 08 | `test_08_delete_zone_pool_with_igroups_predeleted` | Delete an empty zone pool after host igroups were removed directly | isolated | Pool removed | FlexVol deleted; igroups remain absent | negative | --- @@ -197,7 +220,7 @@ Each suite is sequential — tests must run in numbered order; each step builds | 02 | `test_02_delete_volume` | Delete the volume — the LUN is removed from the FlexVol | test_01 (`pool`, `volume`) | Volume no longer listed | LUN no longer in FlexVol; FlexVol itself still `online` | positive | | 03 | `test_03_recreate_volume_for_delete_tests` | Re-create a volume (LUN re-created) — setup for negative tests | test_02 | New volume non-None | LUN present in FlexVol again | positive | | 04 | `test_04_forced_false_delete_with_volume_fails` | Enter maintenance then attempt `deleteStoragePool(forced=False)` with LUN present — must be rejected | test_03 (`pool`, `volume`) | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | -| 05 | `test_05_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_04 | Volume gone; pool gone | LUN removed; FlexVol deleted; igroups deleted | positive | +| 05 | `test_05_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_04 | Volume gone; pool gone | LUN and FlexVol deleted; shared igroup baseline restored | positive | --- @@ -210,16 +233,16 @@ Each suite is sequential — tests must run in numbered order; each step builds | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; igroup per cluster host with host IQN | positive | +| 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; shared host igroups unchanged from suite-start baseline | positive | | 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume (creates a LUN in the FlexVol) | test_01 (`pool`) | Volume non-None | ≥1 LUN in FlexVol | positive | | 03 | `test_03_deploy_vm` | Deploy VM using first ready KVM template; verify 0 LUN-maps exist before attach | test_02 (`volume`) | `vm.state == "Running"`; 0 LUN-maps on ONTAP | 0 LUN-maps (`list_lun_maps_for_volume` returns empty) | positive | -| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created (TDS SN 27) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | -| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed (TDS VM Stop iSCSI) | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | -| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created (TDS VM Start iSCSI) | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | -| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM (TDS Detach iSCSI) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | -| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; volume gone; pool gone | FlexVol deleted; all LUNs and igroups deleted | cleanup | +| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | +| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | +| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | +| 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; volume gone; pool gone | FlexVol and test LUNs/maps deleted; shared igroup baseline restored | cleanup | -> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. All other 61 tests pass. +> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. --- @@ -227,14 +250,14 @@ Each suite is sequential — tests must run in numbered order; each step builds | Suite | Protocol | Scope | Tests | Status | |-------|---------|-------|-------|--------| -| NFS3 Pool Lifecycle | NFS3 | Cluster | 8 | ✅ | -| NFS3 Pool with Volumes | NFS3 | Cluster | 7 | ✅ | -| NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | +| NFS3 Pool Lifecycle | NFS3 | Cluster | 12 | ⚠️ new negatives need hardware run | +| NFS3 Pool with Volumes | NFS3 | Cluster | 10 | ⚠️ new negatives need hardware run | +| NFS3 Zone-Scoped Pool | NFS3 | Zone | 8 | ⚠️ new negatives need hardware run | | NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | | NFS3 VM + Volume Attach | NFS3 | Cluster | 8 | ✅ | -| iSCSI Pool Lifecycle | iSCSI | Cluster | 8 | ✅ | -| iSCSI Pool with Volumes | iSCSI | Cluster | 7 | ✅ | -| iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | +| iSCSI Pool Lifecycle | iSCSI | Cluster | 12 | ⚠️ new negatives need hardware run | +| iSCSI Pool with Volumes | iSCSI | Cluster | 11 | ⚠️ new negatives need hardware run | +| iSCSI Zone-Scoped Pool | iSCSI | Zone | 8 | ⚠️ new negatives need hardware run | | iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | | iSCSI VM + Volume Attach | iSCSI | Cluster | 8 | ⚠️ 7/8 | -| **Total** | | | **62** | **61 passing** | +| **Total** | | | **87** | **New negative cases require a hardware run; 1 known VM hot-detach environment failure** | diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py index cc87bacf0e76..b6d9b5791ed3 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -19,18 +19,22 @@ Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage pool lifecycle (no volumes). -Tests are numbered test_01 ... test_08 and must run in that order. Each step +Tests are numbered test_01 ... test_12 and must run in that order. Each step builds on the shared state established by the previous step. Workflow: - 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Enter maintenance mode and delete the storage pool - 07 Create a new pool and allocate a CloudStack data volume (LUN created) - 08 Delete the volume (LUN removed), enter maintenance, force-delete pool + 01 Reject create when a FlexVol of that name already exists on ONTAP + 02 Reject create when no online assigned aggregate has enough free space + 03 Create primary storage pool + 04 Disable storage pool + 05 Enable storage pool + 06 Enter maintenance mode + 07 Cancel maintenance mode + 08 Enter maintenance mode and delete the storage pool + 09 Create a new pool and allocate a CloudStack data volume (LUN created) + 10 Delete the volume (LUN removed), enter maintenance, force-delete pool + 11 Delete an empty pool whose FlexVol was deleted directly on ONTAP + 12 Delete an empty pool whose igroups were deleted on ONTAP Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -61,6 +65,7 @@ enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) +from marvin.cloudstackException import CloudstackAPIException from marvin.lib.base import StoragePool from marvin.lib.common import list_storage_pools @@ -88,6 +93,7 @@ class TestData: DETAIL_STORAGE_IP = "storageIP" ONTAP_MIN_VOLUME_SIZE = 1677721600 + ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 def __init__(self, storage_ip, svm_name, username, password, scope="CLUSTER", provider="NetApp ONTAP", @@ -191,10 +197,14 @@ def setUpClass(cls): # Helpers # ------------------------------------------------------------------ - def _create_pool(self): + def _create_pool(self, pool_name=None, capacitybytes=None): + """Create a pool; name and capacity default to the suite's values.""" ps = self.testdata[TestData.primaryStorage] storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] - pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + if pool_name is None: + pool_name = "OntapISCSI_%d" % random.randint(0, 99999) + if capacitybytes is None: + capacitybytes = ps["capacitybytes"] cmd = createStoragePoolAPI.createStoragePoolCmd() cmd.name = pool_name @@ -205,7 +215,7 @@ def _create_pool(self): cmd.scope = ps[TestData.scope] cmd.provider = ps[TestData.provider] cmd.tags = ps[TestData.tags] - cmd.capacitybytes = ps["capacitybytes"] + cmd.capacitybytes = capacitybytes cmd.hypervisor = "KVM" cmd.managed = True @@ -273,7 +283,120 @@ def _assert_pool_capacity(self, pool, label): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_01_create_primary_storage_pool(self): + def test_01_reject_create_when_flexvol_name_exists(self): + """ + Pre-create a FlexVol on ONTAP, then ask CloudStack for a pool of the + same name. ONTAP refuses the duplicate, so the create must fail. + + Verifies: + - createStoragePool raises CloudstackAPIException + - no pool of that name is left in CloudStack + - the pre-existing FlexVol is untouched (the plugin must not + adopt or delete a volume it did not create) + """ + self._sweep_tracked_pool2() + pool_name = self._throwaway_pool_name("Dup") + try: + self.ontap.create_flexvol( + self.svm_name, pool_name, TestData.ONTAP_MIN_VOLUME_SIZE, + nas_path=False, + ) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-created ONTAP FlexVol '%s' not found; cannot test the " + "duplicate-name rejection" % pool_name, + ) + log_progress( + logger, "info", + "Pre-created FlexVol '%s'; requesting a pool of the same name " + "(expect reject)", pool_name, + ) + with self.assertRaises(CloudstackAPIException) as caught: + self.__class__.pool2 = self._create_pool(pool_name=pool_name) + log_progress( + logger, "info", + "Rejected duplicate-name create for '%s': %s", + pool_name, caught.exception, + ) + self._assert_no_pool_named(pool_name) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-existing ONTAP FlexVol '%s' was removed by the failed " + "pool create" % pool_name, + ) + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool_name + ) + + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_02_reject_create_when_no_aggregate_space(self): + """ + Ask for 1 GiB more than the largest online aggregate assigned to the + SVM can provide, so no aggregate qualifies and the plugin refuses + before creating anything. + + Verifies: + - createStoragePool raises CloudstackAPIException + - the error names the aggregate shortage rather than some other + failure ('No suitable aggregates') + - no pool is left in CloudStack and no FlexVol on ONTAP + """ + self._sweep_tracked_pool2() + max_free = self.ontap.max_online_aggregate_available_bytes( + self.svm_name + ) + if not max_free: + self.skipTest( + "No online aggregate with reported free space is assigned to " + "SVM '%s'; cannot build an unsatisfiable request" + % self.svm_name + ) + requested = int(max_free) + 1024 ** 3 + if requested > TestData.ONTAP_MAX_VOLUME_SIZE: + self.skipTest( + "Largest aggregate free space (%d B) + 1 GiB exceeds the " + "ONTAP FlexVol maximum (%d B); the request would be refused " + "for the size limit rather than the aggregate shortage" + % (max_free, TestData.ONTAP_MAX_VOLUME_SIZE) + ) + + pool_name = self._throwaway_pool_name("NoSpace") + log_progress( + logger, "info", + "Requesting pool '%s' of %d B; largest online aggregate on SVM " + "'%s' has %d B free (expect reject)", + pool_name, requested, self.svm_name, max_free, + ) + try: + with self.assertRaises(CloudstackAPIException) as caught: + self.__class__.pool2 = self._create_pool( + pool_name=pool_name, capacitybytes=requested + ) + error_text = str(caught.exception) + log_progress( + logger, "info", + "Rejected no-space create for '%s': %s", pool_name, error_text, + ) + self.assertIn( + "No suitable aggregates", error_text, + "Expected the rejection to report 'No suitable aggregates', " + "got: %s" % error_text, + ) + self._assert_no_pool_named(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' was created despite the rejected pool " + "create" % pool_name, + ) + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool_name + ) + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_create_primary_storage_pool(self): """ Create an iSCSI primary storage pool and verify: - CloudStack state is Up, type is OntapiSCSI @@ -331,13 +454,13 @@ def test_01_create_primary_storage_pool(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): + def test_04_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled - ONTAP: FlexVol is still online (disable is a CS-only state change) """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -361,13 +484,13 @@ def test_02_disable_storage_pool(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): + def test_05_enable_storage_pool(self): """ Re-enable the pool and verify: - CloudStack reports Up - ONTAP: FlexVol is still online (enable is a CS-only state change) """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -391,13 +514,13 @@ def test_03_enable_storage_pool(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): + def test_06_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance - ONTAP: FlexVol is still online (maintenance is a CS-only state change) """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() cmd.id = self.__class__.pool.id @@ -420,13 +543,13 @@ def test_04_enter_maintenance_mode(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): + def test_07_cancel_maintenance_mode(self): """ Cancel maintenance and verify: - CloudStack reports Up - ONTAP: FlexVol is still online """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() cmd.id = self.__class__.pool.id @@ -448,13 +571,13 @@ def test_05_cancel_maintenance_mode(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_06_enter_maintenance_and_delete_pool(self): + def test_08_enter_maintenance_and_delete_pool(self): """ Enter maintenance mode then delete the pool. Verifies the pool is removed from CloudStack and the backing ONTAP FlexVol is deleted. """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") pool = self.__class__.pool pool_name = pool.name @@ -497,7 +620,7 @@ def test_06_enter_maintenance_and_delete_pool(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_07_create_volume_on_pool(self): + def test_09_create_volume_on_pool(self): """ Create a new iSCSI pool and allocate a CloudStack data volume. For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. @@ -563,7 +686,7 @@ def test_07_create_volume_on_pool(self): # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_08_delete_volume_and_pool(self): + def test_10_delete_volume_and_pool(self): """ Delete the volume from test_07, enter maintenance, then force-delete the pool. @@ -574,8 +697,8 @@ def test_08_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: igroups for all cluster hosts deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_09 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_09 must pass first") pool = self.__class__.pool pool_name = pool.name @@ -643,3 +766,250 @@ def test_08_delete_volume_and_pool(self): igroup, "ONTAP igroup '%s' still exists after pool deletion" % igroup_name ) + + def _throwaway_pool_name(self, suffix): + return "OntapISCSI%s_%d" % (suffix, random.randint(0, 99999)) + + + def _cleanup_throwaway_pool(self, pool, flexvol_name=None): + """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + + Never raises, so a failed assertion in the test body is the error + that surfaces. + """ + if pool is not None: + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + if listed: + try: + if listed[0].state != "Maintenance": + maint_cmd = ( + enableStorageMaintenance + .enableStorageMaintenanceCmd() + ) + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state( + pool.id, "Maintenance", timeout=120 + ) + except Exception as exc: + logger.warning( + "cleanup: could not put pool '%s' into Maintenance: %s", + pool.name, exc, + ) + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "cleanup: could not delete pool '%s': %s", + pool.name, exc, + ) + if flexvol_name is None: + flexvol_name = pool.name + if flexvol_name: + try: + if self.ontap.get_volume(flexvol_name) is not None: + self.ontap.offline_and_delete_volume(flexvol_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete ONTAP FlexVol '%s': %s", + flexvol_name, exc, + ) + if pool is None: + return + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + if not remaining: + self.__class__.pool2 = None + + + def _assert_no_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except CloudstackAPIException: + listed = None + self.assertFalse( + listed, + "CloudStack should hold no pool named '%s' after a rejected " + "create, found: %s" % (pool_name, listed), + ) + + + def _assert_pool_gone_from_cs(self, pool_id, pool_name): + try: + remaining = list_storage_pools(self.apiClient, id=pool_id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "Pool '%s' still listed in CloudStack after deletion" % pool_name, + ) + + + def _sweep_tracked_pool2(self): + """Clean a prior leftover before reusing the shared pool2 slot.""" + if self.__class__.pool2 is None: + return + self._cleanup_throwaway_pool(self.__class__.pool2) + if self.__class__.pool2 is not None: + self.skipTest("A previously tracked pool could not be cleaned up") + + + def _create_throwaway_pool(self, suffix): + """Create an isolated pool, park it in pool2, and return it.""" + self._sweep_tracked_pool2() + pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) + self.__class__.pool2 = pool + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (pool.name, pool.state), + ) + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ) + return pool + + + def _enter_maintenance(self, pool): + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_11_delete_pool_with_flexvol_predeleted(self): + """ + Delete the backing FlexVol directly on ONTAP, then delete the empty + pool through CloudStack. Deletion must tolerate the missing volume + rather than leaving an undeletable pool behind. + + Verifies: + - deleteStoragePool succeeds with the FlexVol already gone + - the pool is removed from CloudStack + """ + pool = self._create_throwaway_pool("PreDelVol") + try: + self._enter_maintenance(pool) + log_progress( + logger, "info", + "Deleting ONTAP FlexVol '%s' behind CloudStack's back", + pool.name, + ) + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still present after direct deletion" + % pool.name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' reappeared after pool deletion" + % pool.name, + ) + self.__class__.pool2 = None + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool.name + ) + + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_12_delete_pool_with_igroups_predeleted(self): + """ + Delete the per-host igroups directly on ONTAP, then delete the empty + pool through CloudStack. Deletion must tolerate the missing igroups + and still remove the FlexVol. + + The igroup name is keyed off the host UUID and the SVM, not the pool, + so the igroups are shared by every ONTAP pool on that SVM. This test + therefore assumes no other pool is in use on the SVM, which holds + here because test_10 removed the workflow pools. + + The plugin only creates an igroup when a host is first granted access + to a LUN, so a freshly created empty pool has none. The test seeds + them on ONTAP under the exact names the plugin would use, which both + proves the naming scheme still matches and makes the pre-deletion a + real precondition rather than a no-op. + + Verifies: + - deleteStoragePool succeeds with the igroups already gone + - the pool is removed from CloudStack + - ONTAP: the FlexVol is deleted + """ + other_pools = self._other_ontap_pools_on_svm(None) + if other_pools: + self.skipTest( + "Pre-deleting SVM-wide igroups requires exclusive SVM use; " + "found other ONTAP pool(s): %s" + % ", ".join(str(getattr(p, "name", p)) for p in other_pools) + ) + specs = self._iscsi_host_specs() + if not specs: + self.skipTest( + "No cluster host advertises an iSCSI IQN, so no igroup name " + "can be derived" + ) + pool = self._create_throwaway_pool("PreDelIgroup") + seeded = [] + try: + for igroup_name, iqn in specs: + if self.ontap.get_igroup(self.svm_name, igroup_name) is None: + log_progress( + logger, "info", + "Seeding ONTAP igroup '%s' with initiator '%s'", + igroup_name, iqn, + ) + self.ontap.create_igroup(self.svm_name, igroup_name, iqn) + seeded.append(igroup_name) + present = [name for name, _ in specs] + for igroup_name in present: + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' should exist before the pre-deletion" + % igroup_name, + ) + + self._enter_maintenance(pool) + log_progress( + logger, "info", + "Deleting ONTAP igroups %s behind CloudStack's back", present, + ) + for igroup_name in present: + self.ontap.delete_igroup(self.svm_name, igroup_name) + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' still present after direct deletion" + % igroup_name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still exists after pool deletion" + % pool.name, + ) + self.__class__.pool2 = None + finally: + for igroup_name in seeded: + try: + self.ontap.delete_igroup(self.svm_name, igroup_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete seeded igroup '%s': %s", + igroup_name, exc, + ) + self._cleanup_throwaway_pool( + self.__class__.pool2, flexvol_name=pool.name + ) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py index 9dd49761c1dc..e372b89f800a 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_with_volumes.py @@ -47,6 +47,15 @@ 06 Re-enter maintenance; forced=False delete rejected (Neg SN 5) 07 Delete volume from Maintenance, then force-delete pool (SN 7) +Isolated tests (test_08 onwards) run after that workflow and share no state +with it. Each one builds its own pool plus CloudStack volume, breaks a single +ONTAP object behind CloudStack's back, and force-deletes the pool: + + 08 FlexVol pre-deleted on ONTAP, then force-delete pool with CS volume + 09 Host igroups pre-deleted on ONTAP, then force-delete pool with CS volume + 10 Enter maintenance with CS volume after LUN maps are pre-deleted + 11 Cancel maintenance once the CS volume has been deleted + Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed - KVM cluster where every host has iSCSI initiator configured @@ -62,7 +71,6 @@ import base64 import logging import random -import re import unittest from nose.plugins.attrib import attr @@ -71,6 +79,7 @@ cancelStorageMaintenance, createStoragePool as createStoragePoolAPI, deleteVolume as deleteVolumeAPI, + destroyVolume as destroyVolumeAPI, enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) @@ -82,7 +91,6 @@ logger = logging.getLogger("TestOntapISCSIPoolWithVolumes") - # --------------------------------------------------------------------------- # Test data # --------------------------------------------------------------------------- @@ -141,17 +149,6 @@ def __init__(self, storage_ip, svm_name, username, password, } -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _igroup_name(svm_name, host_name): - """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" - short = host_name.split(".")[0] - sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) - return "cs_%s_%s" % (svm_name, sanitized) - - # --------------------------------------------------------------------------- # Test class # --------------------------------------------------------------------------- @@ -159,7 +156,7 @@ def _igroup_name(svm_name, host_name): class TestOntapISCSIPoolWithVolumes(OntapTestBase): """ iSCSI pool lifecycle tests with a CloudStack data volume present throughout. - All 7 tests are sequential and share class-level state. + Tests 01-07 are sequential; tests 08-10 use isolated throwaway resources. """ _vol_name_prefix = "OntapISCSIWV" @@ -201,6 +198,7 @@ def setUpClass(cls): cls.svm_name = svm_name cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + cls._capture_igroup_baseline() # ------------------------------------------------------------------ # Helpers @@ -339,18 +337,12 @@ def test_01_create_pool_and_volume(self): "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") ) - # ONTAP: igroup must exist for each cluster host that has an IQN - for host in self.cluster_hosts: - iqn = getattr(host, "storageurl", None) - if not iqn or not iqn.startswith("iqn."): - continue - igroup_name = _igroup_name(self.svm_name, host.name) - igroup = self.ontap.get_igroup(self.svm_name, igroup_name) - self.assertIsNotNone( - igroup, - "ONTAP igroup '%s' not found for host '%s'" - % (igroup_name, host.name) - ) + # ONTAP: the plugin creates an igroup only when a host is first + # granted access to a LUN, so the empty pool must not change the + # suite-start igroup baseline. + self._assert_igroup_baseline_unchanged( + "after creating an empty pool" + ) # Allocate a CloudStack data volume on this pool vol = self._create_volume(pool.id) @@ -359,6 +351,9 @@ def test_01_create_pool_and_volume(self): # ONTAP: a LUN must exist in the FlexVol after volume creation self._assert_lun_exists(pool.name, "after volume creation") + self._assert_igroup_baseline_unchanged( + "after creating an unattached volume" + ) # Capacity reporting: LUN allocated but FlexVol size unchanged self._assert_pool_capacity(pool, "volume-allocated") @@ -693,15 +688,546 @@ def test_07_delete_volume_and_force_delete_pool(self): "ONTAP FlexVol '%s' still exists after pool force deletion" % pool_name ) - # ONTAP: igroups for all cluster hosts must be deleted - for host in self.cluster_hosts: - iqn = getattr(host, "storageurl", None) - if not iqn or not iqn.startswith("iqn."): - continue - igroup_name = _igroup_name(self.svm_name, host.name) - igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + self._assert_no_lun_maps_for_volume( + pool_name, "after pool force deletion" + ) + self._assert_igroup_baseline_unchanged("after pool force deletion") + + # ================================================================== + # Isolated tests — appended after the sequential workflow above. + # + # Each one creates its own pool and CloudStack volume in the pool2 / + # volume2 slots (which OntapTestBase.tearDownClass also sweeps), runs a + # single scenario, and cleans up in a finally block. They never reuse a + # pool destroyed by another test. + # ================================================================== + + def _host_igroup_names(self): + """igroup names the plugin creates, one per cluster host with an IQN. + + Built from the host UUID so the names match the plugin. + """ + return [name for name, _ in self._host_igroup_specs()] + + def _host_igroup_specs(self): + """(igroup name, initiator IQN) per cluster host that reports an IQN.""" + return self._iscsi_host_specs() + + def _enter_maintenance(self, pool): + """Put the pool into Maintenance and wait for the state to settle.""" + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = pool.id + self.apiClient.enableStorageMaintenance(cmd) + return self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + def _create_isolated_pool_with_volume(self, label): + """Build a fresh pool plus CS volume for one isolated scenario. + + Returns ``(pool, volume)``. Any pool a previous isolated test could + not clean up is swept first so its ONTAP objects are never orphaned by + the overwrite of the pool2 slot. + """ + if self.__class__.pool2 is not None: + self._cleanup_isolated_pool( + self.__class__.pool2, "leftover-from-previous-isolated-test" + ) + + pool = self._create_pool() + self.__class__.pool2 = pool + logger.info("[%s] created isolated pool '%s'", label, pool.name) + + self.assertEqual( + pool.state, "Up", + "[%s] new pool state should be 'Up', got '%s'" % (label, pool.state) + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol not found for new pool '%s'" % (label, pool.name) + ) + + vol = self._create_volume(pool.id) + self.__class__.volume2 = vol + self.assertIsNotNone(vol, "[%s] createVolume returned None" % label) + self._assert_lun_exists(pool.name, "%s: after volume creation" % label) + return pool, vol + + def _assert_pool_absent(self, pool, label, delete_error=None): + """Assert CloudStack no longer lists the pool.""" + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + self.assertFalse( + remaining, + "[%s] pool '%s' is still listed after deleteStoragePool(forced=True)%s" + % (label, pool.name, + "; the API raised: %s" % delete_error if delete_error else "") + ) + + def _purge_cs_volume_record(self, vol, label): + """Remove a CS volume record the forced pool delete may have left. + + The backing LUN is already gone at this point, so a failure here only + affects tidiness — the volume stays in the volume2 slot for + tearDownClass to retry and no exception is raised. + """ + if vol is None: + return + if self._volume_exists_in_cs(vol.id): + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + logger.info("[%s] deleted leftover CS volume record %s", + label, vol.id) + except Exception as exc: + logger.warning("[%s] could not delete leftover CS volume %s: %s", + label, vol.id, exc) + else: + logger.info("[%s] CS volume %s was removed along with the pool", + label, vol.id) + if not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + + def _exit_maintenance(self, pool, label): + """Bring a pool out of Maintenance so its volumes can be deleted.""" + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + return False + if not listed: + return False + if listed[0].state != "Maintenance": + return True + try: + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = pool.id + self.apiClient.cancelStorageMaintenance(cmd) + self._poll_pool_state(pool.id, "Up", timeout=120) + return True + except Exception as exc: + logger.warning("[%s] could not cancel maintenance on '%s': %s", + label, pool.name, exc) + return False + + def _enter_maintenance_quietly(self, pool, label): + """Enter Maintenance, tolerating a pool whose backend is already gone.""" + try: + self._enter_maintenance(pool) + except Exception as exc: + logger.warning("[%s] could not enter maintenance on '%s': %s", + label, pool.name, exc) + + DESTROYED_VOLUME_STATES = ("destroy", "destroyed", "expunging", "expunged") + + def _cs_volume_state(self, vol_id): + """Return the CloudStack volume state, or None when it is not listed.""" + vol = self._get_cs_volume(vol_id) + return getattr(vol, "state", None) if vol is not None else None + + def _volume_cleared_for_pool_delete(self, vol_id): + """True once the volume no longer blocks deleteStoragePool(forced).""" + state = self._cs_volume_state(vol_id) + return state is None or state.lower() in self.DESTROYED_VOLUME_STATES + + def _remove_cs_volume(self, pool, vol, label): + """Clear the CloudStack volume so the pool can be force-deleted. + + deleteStoragePool(forced=True) refuses while any volume on the pool is + in a state other than Destroy. deleteVolume is tried first because it + also reclaims the backing storage, but it expunges through libvirt and + fails when the FlexVol is already gone. destroyVolume(expunge=False) + is the fallback: it only moves the record to Destroy, which is all the + forced pool delete requires - it expunges the leftovers itself. + """ + if vol is None or not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + return True + self._exit_maintenance(pool, label) + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + logger.warning("[%s] deleteVolume failed for %s (%s); falling back " + "to destroyVolume without expunge", + label, vol.id, exc) + try: + cmd = destroyVolumeAPI.destroyVolumeCmd() + cmd.id = vol.id + cmd.expunge = False + self.apiClient.destroyVolume(cmd) + except Exception as destroy_exc: + logger.warning("[%s] destroyVolume also failed for %s: %s", + label, vol.id, destroy_exc) + if not self._volume_cleared_for_pool_delete(vol.id): + return False + if not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + return True + + def _cleanup_isolated_pool(self, pool, label): + """Best-effort teardown of one isolated pool and its ONTAP FlexVol. + + Igroups are intentionally left alone: their names carry no pool + identity, so the pool delete owns their removal. + """ + if pool is None: + return + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + if listed: + self._remove_cs_volume(pool, self.__class__.volume2, label) + try: + listed = list_storage_pools(self.apiClient, id=pool.id) or listed + if listed[0].state != "Maintenance": + self._enter_maintenance(pool) + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning("[%s] could not force-delete pool '%s': %s", + label, pool.name, exc) + try: + self.ontap.offline_and_delete_volume(pool.name) + except Exception as exc: + logger.warning("[%s] ONTAP FlexVol cleanup for '%s' failed: %s", + label, pool.name, exc) + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + if not listed: + self.__class__.pool2 = None + + # ------------------------------------------------------------------ + # Step 08 — FlexVol deleted on ONTAP before the pool delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_08_delete_pool_with_volume_flexvol_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after its + ONTAP FlexVol — and with it the volume's LUN — has been removed behind + CloudStack's back. + + Uses its own pool and volume, so the pool is known to be healthy up to + the point the FlexVol is destroyed. Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing FlexVol + - the CloudStack pool record is removed + - the leftover CS volume record can still be cleaned up + """ + label = "flexvol-missing" + pool, vol = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool) + + self.ontap.offline_and_delete_volume(pool.name) self.assertIsNone( - igroup, - "ONTAP igroup '%s' still exists after pool force deletion" - % igroup_name + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should be gone before the pool delete" + % (label, pool.name) + ) + self.assertEqual( + len(self.ontap.list_luns_in_volume(self.svm_name, pool.name)), 0, + "[%s] LUNs should have gone with the FlexVol '%s'" + % (label, pool.name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + self._enter_maintenance_quietly(pool, label) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate a missing " + "FlexVol, but raised: %s" % (label, delete_error) + ) + + self._assert_igroup_baseline_unchanged( + "[%s] after pool delete with missing FlexVol" % label + ) + + self._purge_cs_volume_record(vol, label) + finally: + self._cleanup_isolated_pool(pool, label) + + # ------------------------------------------------------------------ + # Step 09 — Host igroups deleted on ONTAP before the delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_09_delete_pool_with_volume_igroups_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after the host + igroups have been removed behind CloudStack's back. + + Uses its own pool and volume. The volume is not attached to any VM, so + no LUN maps reference the igroups and they delete cleanly. Unlike + test_08 the FlexVol is still present, so the plugin is expected to + remove it as part of the delete. Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing igroups + - the CloudStack pool record is removed + - the ONTAP FlexVol is deleted and no igroup is left behind + """ + other_pools = self._other_ontap_pools_on_svm(None) + if other_pools: + self.skipTest( + "Pre-deleting SVM-wide igroups requires exclusive SVM use; " + "found other ONTAP pool(s): %s" + % ", ".join(str(getattr(p, "name", p)) for p in other_pools) + ) + label = "igroups-missing" + pool, vol = self._create_isolated_pool_with_volume(label) + seeded = [] + try: + igroup_specs = self._host_igroup_specs() + self.assertTrue( + igroup_specs, + "[%s] no cluster host reports an IQN, so there is no igroup " + "to remove" % label + ) + igroup_names = [name for name, _ in igroup_specs] + + # The volume is not attached to a VM, so the plugin has never + # granted a host access and created no igroups. Seed them under + # the plugin's own names so the pre-deletion is a real one. + for name, iqn in igroup_specs: + if self.ontap.get_igroup(self.svm_name, name) is None: + logger.info("[%s] seeding igroup '%s' with initiator '%s'", + label, name, iqn) + self.ontap.create_igroup(self.svm_name, name, iqn) + seeded.append(name) + + self._enter_maintenance(pool) + + deleted = [] + for name in igroup_names: + if self.ontap.get_igroup(self.svm_name, name) is None: + continue + self.ontap.delete_igroup(self.svm_name, name) + deleted.append(name) + logger.info("[%s] deleted %d of %d host igroup(s): %s", + label, len(deleted), len(igroup_names), deleted) + + for name in igroup_names: + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, name), + "[%s] igroup '%s' should be gone before the pool delete" + % (label, name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + self._enter_maintenance_quietly(pool, label) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate missing " + "igroups, but raised: %s" % (label, delete_error) + ) + + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should have been deleted with the pool" + % (label, pool.name) + ) + for name in igroup_names: + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, name), + "[%s] igroup '%s' reappeared during the pool delete" + % (label, name) + ) + + self._purge_cs_volume_record(vol, label) + finally: + for name in seeded: + try: + self.ontap.delete_igroup(self.svm_name, name) + except Exception as exc: + logger.warning( + "[%s] cleanup: could not delete seeded igroup '%s': %s", + label, name, exc) + self._cleanup_isolated_pool(pool, label) + + # ------------------------------------------------------------------ + # Step 10 — Enter maintenance after LUN maps are pre-deleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_10_enter_maintenance_lun_maps_predeleted(self): + """ + Enter maintenance with a CloudStack volume after its ONTAP LUN maps + have been deleted behind CloudStack's back. Verifies: + - enableStorageMaintenance tolerates already-absent LUN maps + - the pool reaches Maintenance and the CS volume remains present + - the LUN remains online and its maps remain absent + """ + label = "maintenance-lun-maps-missing" + pool, vol = self._create_isolated_pool_with_volume(label) + seeded_igroup = None + try: + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + luns, + "[%s] no LUN found in FlexVol '%s'" % (label, pool.name) + ) + lun_path = luns[0].get("name") + + igroup_specs = self._host_igroup_specs() + self.assertTrue( + igroup_specs, + "[%s] no cluster host reports an IQN" % label + ) + igroup_name, initiator_iqn = igroup_specs[0] + if self.ontap.get_igroup(self.svm_name, igroup_name) is None: + self.ontap.create_igroup( + self.svm_name, igroup_name, initiator_iqn + ) + seeded_igroup = igroup_name + + self.ontap.create_lun_map( + self.svm_name, lun_path, igroup_name + ) + maps = self.ontap.list_lun_maps_for_volume( + self.svm_name, pool.name + ) + self.assertTrue( + maps, + "[%s] failed to seed a LUN map for '%s'" % (label, lun_path) + ) + + for lun_map in maps: + self.ontap.delete_lun_map(lun_map) + self.assertEqual( + self.ontap.list_lun_maps_for_volume( + self.svm_name, pool.name + ), + [], + "[%s] LUN maps should be absent before maintenance" % label + ) + + self._enter_maintenance(pool) + self.assertTrue( + self._volume_exists_in_cs(vol.id), + "[%s] CS volume disappeared after entering maintenance" % label + ) + self._assert_lun_exists( + pool.name, "after entering Maintenance with maps pre-deleted" + ) + self.assertEqual( + self.ontap.list_lun_maps_for_volume( + self.svm_name, pool.name + ), + [], + "[%s] LUN maps unexpectedly reappeared during maintenance" + % label + ) + finally: + self._cleanup_isolated_pool(pool, label) + if seeded_igroup: + try: + self.ontap.delete_igroup( + self.svm_name, seeded_igroup + ) + except Exception as exc: + logger.warning( + "[%s] cleanup: could not delete seeded igroup '%s': %s", + label, seeded_igroup, exc + ) + + # ------------------------------------------------------------------ + # Step 11 — Cancel maintenance once the CS volume has been deleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_with_volumes"], required_hardware=True) + def test_11_cancel_maintenance_after_volume_deleted(self): + """ + Cancel maintenance on a pool whose CloudStack volume has been deleted. + + Complements test_05, which cancels maintenance with the volume still + present. On iSCSI the volume can be deleted while the pool sits in + Maintenance, so that is the order used here. Verifies: + - the LUN is removed when the volume is deleted + - the pool returns to Up + - the ONTAP FlexVol is still online + """ + label = "cancel-maintenance-no-volume" + pool, vol = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool) + + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.assertFalse( + self._volume_exists_in_cs(vol.id), + "[%s] CS volume %s should be gone before cancel maintenance" + % (label, vol.id) + ) + self.__class__.volume2 = None + vol = None + + luns_after = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertEqual( + len(luns_after), 0, + "[%s] expected 0 LUNs in FlexVol '%s' after volume deletion, " + "found %d: %s" % (label, pool.name, len(luns_after), luns_after) + ) + + cancel_cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cancel_cmd.id = pool.id + self.apiClient.cancelStorageMaintenance(cancel_cmd) + + result = self._poll_pool_state(pool.id, "Up", timeout=120) + self.assertEqual( + result.state, "Up", + "[%s] pool should be 'Up' after cancel maintenance, got '%s'" + % (label, result.state) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol '%s' disappeared after cancel maintenance" + % (label, pool.name) + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "[%s] ONTAP FlexVol should be 'online' after cancel " + "maintenance, got '%s'" % (label, ontap_vol.get("state")) ) + finally: + self._purge_cs_volume_record(vol, label) + self._cleanup_isolated_pool(pool, label) diff --git a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py index 847a026a3bd5..0712c66bf96e 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py @@ -18,16 +18,24 @@ """ Zone-scoped primary storage lifecycle tests for NetApp ONTAP (iSCSI). -Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). CloudStack calls -OntapPrimaryDatastoreLifecycle.attachZone(), which connects all eligible KVM -hosts in the zone and creates igroups for each host's IQN. - -Workflow: - 01 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; - igroup present for each cluster host IQN - 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged - 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged - 04 Delete zone-scoped pool — pool gone; FlexVol deleted; igroups deleted +Creates a zone-scoped pool (scope=ZONE, no clusterid/podid). Host igroups are +shared by host and SVM and are created only when a LUN is granted to a host, +not when an empty pool is created. + +Test order — 03-06 are a sequential workflow that must run in order; 01, 02, +07 and 08 are isolated negative/recovery cases, each owning the pool it +creates, so they can be run on their own: + 01 Create rejected when a FlexVol of the same name already exists + 02 Create rejected when no assigned online aggregate has enough free space + 03 Create zone-scoped iSCSI pool — pool.state Up; ONTAP FlexVol online; + pre-existing shared igroups unchanged + 04 Grow zone-scoped pool — capacity increased; FlexVol resized; state Up + 05 Shrink zone-scoped pool — capacity back to its pre-grow value; state Up + 06 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 07 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 08 Delete zone-scoped pool — pool gone; FlexVol deleted; baseline restored + 09 Delete an empty pool whose FlexVol was removed behind CloudStack's back + 10 Delete an empty pool whose igroups were removed beforehand Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -40,14 +48,14 @@ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ test/integration/plugins/ontap/iscsi/pool/test_zone_scoped_pool.py -v -Note: Tests 01-04 share class-level state (sequential). Always run the full +Note: Tests 03-06 share class-level state (sequential). Always run the full suite. """ import base64 import logging import random -import re +import time import unittest from nose.plugins.attrib import attr @@ -57,10 +65,16 @@ enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) +from marvin.cloudstackException import CloudstackAPIException from marvin.lib.base import StoragePool from marvin.lib.common import list_storage_pools -from ontap_test_base import OntapRestClient, OntapTestBase, get_datacenter_config +from ontap_test_base import ( + OntapRestClient, + OntapTestBase, + get_datacenter_config, + log_progress, +) logger = logging.getLogger("TestOntapISCSIZoneScopedPool") @@ -122,17 +136,6 @@ def __init__(self, storage_ip, svm_name, username, password, } -# --------------------------------------------------------------------------- -# iSCSI path helpers -# --------------------------------------------------------------------------- - -def _igroup_name(svm_name, host_name): - """Mirror OntapStorageUtils.getIgroupName: cs_{svmName}_{sanitizedHostName}""" - short = host_name.split(".")[0] - sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", short) - return "cs_%s_%s" % (svm_name, sanitized) - - # --------------------------------------------------------------------------- # Sequential workflow test class # --------------------------------------------------------------------------- @@ -141,6 +144,11 @@ class TestOntapISCSIZoneScopedPool(OntapTestBase): _vol_name_prefix = "OntapISCSIZoneVol" + ONE_GIB = 1024 ** 3 + # Above this much free aggregate space, asking for "max free + 1 GiB" + # stops being a meaningful request, so the no-space test skips instead. + MAX_AGGREGATE_FREE_FOR_NO_SPACE_TEST = 300 * 1024 ** 4 + @classmethod def setUpClass(cls): super(TestOntapISCSIZoneScopedPool, cls).setUpClass() @@ -176,6 +184,7 @@ def setUpClass(cls): cls.svm_name = svm_name cls._setup_cloudstack_resources(config, cls.testdata[TestData.account]) + cls._capture_igroup_baseline() # No per-test tearDown — state intentionally persists between steps. @@ -183,11 +192,17 @@ def setUpClass(cls): # Helpers # ------------------------------------------------------------------ - def _create_zone_pool(self): - """Create a zone-scoped iSCSI pool (no clusterid / podid).""" + def _create_zone_pool(self, name=None, capacitybytes=None): + """Create a zone-scoped iSCSI pool (no clusterid / podid). + + ``name`` and ``capacitybytes`` let the isolated negative tests drive + the pool name (to collide with a pre-created FlexVol) and the + requested size (to exceed every aggregate) without touching the + shared test data. + """ ps = self.testdata[TestData.primaryStorage] storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] - pool_name = "OntapZoneISCSI_%d" % random.randint(0, 99999) + pool_name = name or "OntapZoneISCSI_%d" % random.randint(0, 99999) cmd = createStoragePoolAPI.createStoragePoolCmd() cmd.name = pool_name @@ -197,7 +212,7 @@ def _create_zone_pool(self): cmd.scope = "ZONE" cmd.provider = ps[TestData.provider] cmd.tags = ps[TestData.tags] - cmd.capacitybytes = ps["capacitybytes"] + cmd.capacitybytes = capacitybytes or ps["capacitybytes"] cmd.hypervisor = "KVM" cmd.managed = True @@ -209,14 +224,21 @@ def _create_zone_pool(self): response = self.apiClient.createStoragePool(cmd) return StoragePool(response.__dict__) - def _assert_igroups_for_hosts(self, expect_present): - """Assert igroups are present (or absent) for each cluster host IQN.""" + def _hosts_with_iqn(self): + """Cluster hosts that report an iSCSI IQN, as (host, iqn) pairs.""" + pairs = [] for host in self.cluster_hosts: iqn = (getattr(host, "storageurl", None) - or getattr(host, "StorageUrl", None)) - if not iqn or not iqn.startswith("iqn."): - continue - igroup_name = _igroup_name(self.svm_name, host.name) + or getattr(host, "StorageUrl", None) + or self.host_iqn(host)) + if iqn and iqn.startswith("iqn."): + pairs.append((host, iqn)) + return pairs + + def _assert_igroups_for_hosts(self, expect_present): + """Assert igroups are present (or absent) for each cluster host IQN.""" + for host, iqn in self._hosts_with_iqn(): + igroup_name = self._igroup_name(host.id) igroup = self.ontap.get_igroup(self.svm_name, igroup_name) if expect_present: self.assertIsNotNone( @@ -238,20 +260,243 @@ def _assert_igroups_for_hosts(self, expect_present): "ONTAP igroup '%s' still exists after pool deletion" % igroup_name ) + # ---- helpers for the isolated tests (01, 02, 09, 10) --------------- + + def _require_ontap_client(self, *method_names): + """Skip when the shared OntapRestClient lacks a backend helper.""" + missing = [n for n in method_names if not hasattr(self.ontap, n)] + if missing: + raise unittest.SkipTest( + "OntapRestClient does not provide %s; update " + "ontap_test_base.py before running this test" + % ", ".join(missing) + ) + + def _create_isolated_zone_pool(self, name): + """Create a throwaway zone pool and register it for class teardown.""" + pool = self._create_zone_pool(name=name) + self.__class__.pool2 = pool + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (name, pool.state) + ) + return pool + + def _wait_for_pool_state_quietly(self, pool_id, target_state, + timeout=120, interval=5): + """Poll for a pool state, returning False instead of failing the test. + + Used on the recovery paths where the backend has deliberately been + broken, so entering Maintenance is allowed to fail. + """ + deadline = time.time() + timeout + while time.time() < deadline: + pools = list_storage_pools(self.apiClient, id=pool_id) + if not pools or pools[0].state == target_state: + return True + time.sleep(interval) + return False + + def _enter_maintenance_quietly(self, pool_id): + """Request Maintenance without failing when the backend is broken.""" + try: + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool_id + self.apiClient.enableStorageMaintenance(maint_cmd) + except Exception as exc: + logger.warning( + "enableStorageMaintenance failed for pool %s: %s" + % (pool_id, exc) + ) + return False + return self._wait_for_pool_state_quietly(pool_id, "Maintenance") + + def _cs_pool_exists(self, pool_id): + try: + return bool(list_storage_pools(self.apiClient, id=pool_id)) + except Exception: + return False + + def _assert_no_cs_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with the given name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except Exception: + listed = None + self.assertFalse( + listed, + "CloudStack should not have created pool '%s' after the " + "rejected request" % pool_name + ) + + def _force_cleanup_zone_pool(self, pool): + """Best-effort removal of a throwaway pool left behind by a failure.""" + if pool is None: + return + if not self._cs_pool_exists(pool.id): + self.__class__.pool2 = None + return + try: + self._enter_maintenance_quietly(pool.id) + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "could not force-delete throwaway pool %s: %s" + % (pool.id, exc) + ) + return + if not self._cs_pool_exists(pool.id): + self.__class__.pool2 = None + + def _force_delete_flexvol(self, vol_name): + """Best-effort ONTAP FlexVol removal for a throwaway volume.""" + try: + self.ontap.offline_and_delete_volume(vol_name) + except Exception as exc: + logger.warning( + "could not delete ONTAP FlexVol '%s': %s" % (vol_name, exc) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create rejected when a FlexVol of the same name exists + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_01_create_zone_pool_rejected_when_flexvol_exists(self): + """ + Pre-create a FlexVol on the SVM, then ask CloudStack for a + zone-scoped iSCSI pool with that exact name. + Verifies: + - createStoragePool raises CloudstackAPIException + - CloudStack records no pool with that name + - ONTAP: the pre-existing FlexVol is left in place + This test owns everything it creates and leaves no pool behind. + """ + self._require_ontap_client("create_flexvol", "offline_and_delete_volume") + + pool_name = "OntapZoneISCSIDup_%d" % random.randint(0, 99999) + size_bytes = self.testdata[TestData.primaryStorage]["capacitybytes"] + created_pool = None + try: + self.ontap.create_flexvol( + self.svm_name, pool_name, size_bytes, nas_path=False + ) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-created ONTAP FlexVol '%s' not visible; cannot test the " + "name collision" % pool_name + ) + + try: + created_pool = self._create_zone_pool(name=pool_name) + except CloudstackAPIException as exc: + log_progress( + logger, "info", + "createStoragePool rejected for existing FlexVol '%s': %s", + pool_name, exc, + ) + else: + self.fail( + "createStoragePool should have been rejected: ONTAP " + "FlexVol '%s' already exists" % pool_name + ) + + self._assert_no_cs_pool_named(pool_name) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-existing ONTAP FlexVol '%s' was removed by the rejected " + "create" % pool_name + ) + finally: + self._force_cleanup_zone_pool(created_pool) + self._force_delete_flexvol(pool_name) + + # ------------------------------------------------------------------ + # Step 02 — Create rejected when no aggregate has enough free space + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_02_create_zone_pool_rejected_when_no_aggregate_space(self): + """ + Ask for a pool 1 GiB larger than the free space of the roomiest + assigned online aggregate. + Verifies: + - createStoragePool raises CloudstackAPIException reporting + 'No suitable aggregates' + - CloudStack records no pool with that name + - ONTAP: no FlexVol of that name was left behind + Skipped when the SVM has more than 300 TiB free on one aggregate, + where the oversized request stops being meaningful. + """ + self._require_ontap_client( + "max_online_aggregate_available_bytes", "offline_and_delete_volume" + ) + + max_free = self.ontap.max_online_aggregate_available_bytes(self.svm_name) + if not max_free: + raise unittest.SkipTest( + "No assigned online aggregate with free space reported for " + "SVM '%s'; cannot build an over-capacity request" + % self.svm_name + ) + requested = int(max_free) + self.ONE_GIB + if requested > self.MAX_AGGREGATE_FREE_FOR_NO_SPACE_TEST: + raise unittest.SkipTest( + "Largest assigned online aggregate on SVM '%s' has %d B free; " + "the over-capacity request would exceed the %d B FlexVol limit" + % (self.svm_name, max_free, + self.MAX_AGGREGATE_FREE_FOR_NO_SPACE_TEST) + ) + + pool_name = "OntapZoneISCSINoSpace_%d" % random.randint(0, 99999) + created_pool = None + try: + try: + created_pool = self._create_zone_pool( + name=pool_name, capacitybytes=requested + ) + except CloudstackAPIException as exc: + error_text = str(exc) + log_progress( + logger, "info", + "createStoragePool rejected for %d B (max aggregate free " + "%d B): %s", requested, max_free, error_text, + ) + self.assertIn( + "No suitable aggregates", error_text, + "Expected the rejection to report 'No suitable " + "aggregates', got: %s" % error_text + ) + else: + self.fail( + "createStoragePool should have been rejected: requested " + "%d B but the roomiest aggregate has only %d B free" + % (requested, max_free) + ) + + self._assert_no_cs_pool_named(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' was left behind by the rejected create" + % pool_name + ) + finally: + self._force_cleanup_zone_pool(created_pool) + self._force_delete_flexvol(pool_name) + # ------------------------------------------------------------------ - # Step 01 — Create zone-scoped iSCSI pool + # Step 03 — Create zone-scoped iSCSI pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_01_create_zone_scoped_pool(self): + def test_03_create_zone_scoped_pool(self): """ Create a zone-scoped iSCSI primary storage pool (no clusterid/podid). - CloudStack calls attachZone(), which connects all eligible KVM hosts - in the zone and creates igroups for each host's IQN. Verifies: - pool.state is Up, type is OntapiSCSI - ONTAP: FlexVol is online - - ONTAP: igroup exists for each cluster host with the correct IQN + - ONTAP: pre-existing shared igroups are unchanged """ pool = self._create_zone_pool() self.__class__.pool = pool @@ -276,22 +521,26 @@ def test_01_create_zone_scoped_pool(self): "ONTAP FlexVol should be 'online', got '%s'" % ontap_vol.get("state") ) - # ONTAP: igroups must exist for each cluster host with IQN - self._assert_igroups_for_hosts(expect_present=True) + # ONTAP: the plugin creates an igroup only when a host is first + # granted access to a LUN, so creating an empty pool must not make + # new ones. + self._assert_igroup_baseline_unchanged( + "after creating an empty zone-scoped pool" + ) # ------------------------------------------------------------------ - # Step 02 — Disable zone-scoped pool + # Step 04 — Disable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_02_disable_zone_scoped_pool(self): + def test_04_disable_zone_scoped_pool(self): """ Disable the zone-scoped iSCSI pool. Verifies: - pool.state is Disabled - ONTAP: FlexVol still online; igroups unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -308,22 +557,24 @@ def test_02_disable_zone_scoped_pool(self): "ONTAP FlexVol should still be 'online' after disable" ) - # igroups must still be present after a simple disable - self._assert_igroups_for_hosts(expect_present=True) + # The pool has no volumes, so shared igroups must remain unchanged. + self._assert_igroup_baseline_unchanged( + "after disabling an empty zone-scoped pool" + ) # ------------------------------------------------------------------ - # Step 03 — Enable zone-scoped pool + # Step 05 — Enable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_03_enable_zone_scoped_pool(self): + def test_05_enable_zone_scoped_pool(self): """ Re-enable the zone-scoped iSCSI pool. Verifies: - pool.state is Up - ONTAP: FlexVol still online; igroups unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -340,23 +591,25 @@ def test_03_enable_zone_scoped_pool(self): "ONTAP FlexVol should be 'online' after enable" ) - # igroups must still be present after re-enable - self._assert_igroups_for_hosts(expect_present=True) + # The pool has no volumes, so shared igroups must remain unchanged. + self._assert_igroup_baseline_unchanged( + "after re-enabling an empty zone-scoped pool" + ) # ------------------------------------------------------------------ - # Step 04 — Delete zone-scoped pool + # Step 06 — Delete zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["iscsi_zone_pool"], required_hardware=True) - def test_04_delete_zone_scoped_pool(self): + def test_06_delete_zone_scoped_pool(self): """ Enter maintenance then delete the zone-scoped iSCSI pool. Verifies: - Pool is removed from CloudStack - ONTAP: FlexVol deleted - - ONTAP: igroups deleted for all cluster hosts + - ONTAP: this pool's maps are gone and shared igroups are restored """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") pool = self.__class__.pool pool_name = pool.name @@ -383,5 +636,144 @@ def test_04_delete_zone_scoped_pool(self): "ONTAP FlexVol '%s' still exists after pool deletion" % pool_name ) - # ONTAP: igroups for each cluster host must be deleted - self._assert_igroups_for_hosts(expect_present=False) + self._assert_no_lun_maps_for_volume( + pool_name, "after zone-scoped pool deletion" + ) + self._assert_igroup_baseline_unchanged( + "after zone-scoped pool deletion" + ) + + # ------------------------------------------------------------------ + # Step 07 — Delete a pool whose FlexVol was pre-deleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_07_delete_zone_pool_with_flexvol_predeleted(self): + """ + Create an empty zone pool, remove its FlexVol directly on ONTAP, then + delete the pool through CloudStack. + Verifies: + - deleteStoragePool succeeds and the pool leaves CloudStack + - ONTAP: the FlexVol stays gone + """ + self._require_ontap_client("offline_and_delete_volume") + + pool_name = "OntapZoneISCSINoFv_%d" % random.randint(0, 99999) + pool = self._create_isolated_zone_pool(pool_name) + try: + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' missing right after pool creation" + % pool_name + ) + + self.assertTrue( + self._enter_maintenance_quietly(pool.id), + "Pool '%s' did not enter Maintenance before backend mutation" + % pool_name + ) + + # Delete the FlexVol behind CloudStack's back. + self.ontap.offline_and_delete_volume(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' still present after the manual delete" + % pool_name + ) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool2 = None + + self.assertFalse( + self._cs_pool_exists(pool.id), + "Pool '%s' still listed in CloudStack after deletion with a " + "pre-deleted FlexVol" % pool_name + ) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' reappeared after pool deletion" % pool_name + ) + finally: + self._force_cleanup_zone_pool(pool) + self._force_delete_flexvol(pool_name) + + # ------------------------------------------------------------------ + # Step 08 — Delete a pool whose igroups were pre-deleted + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_zone_pool"], required_hardware=True) + def test_08_delete_zone_pool_with_igroups_predeleted(self): + """ + Create an empty zone pool, remove the per-host igroups directly on + ONTAP, then delete the pool through CloudStack. + + The plugin only creates an igroup when a host is first granted access + to a LUN, so an empty pool has none. The test seeds them under the + exact names the plugin would use, so the pre-deletion is a real + precondition rather than a no-op. + + Verifies: + - deleteStoragePool succeeds and the pool leaves CloudStack + - ONTAP: the FlexVol is deleted and the igroups stay gone + """ + self._require_ontap_client("create_igroup", "delete_igroup", + "offline_and_delete_volume") + other_pools = self._other_ontap_pools_on_svm(None) + if other_pools: + self.skipTest( + "Pre-deleting SVM-wide igroups requires exclusive SVM use; " + "found other ONTAP pool(s): %s" + % ", ".join(str(getattr(p, "name", p)) for p in other_pools) + ) + + pool_name = "OntapZoneISCSINoIg_%d" % random.randint(0, 99999) + pool = self._create_isolated_zone_pool(pool_name) + seeded = [] + try: + for host, iqn in self._hosts_with_iqn(): + igroup_name = self._igroup_name(host.id) + if self.ontap.get_igroup(self.svm_name, igroup_name) is None: + logger.info("Seeding ONTAP igroup '%s' with initiator '%s'", + igroup_name, iqn) + self.ontap.create_igroup(self.svm_name, igroup_name, iqn) + seeded.append(igroup_name) + self._assert_igroups_for_hosts(expect_present=True) + self.assertTrue( + self._enter_maintenance_quietly(pool.id), + "Pool '%s' did not enter Maintenance before backend mutation" + % pool_name + ) + + for host, _iqn in self._hosts_with_iqn(): + igroup_name = self._igroup_name(host.id) + self.ontap.delete_igroup(self.svm_name, igroup_name) + self.assertIsNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' still present after the manual delete" + % igroup_name + ) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool2 = None + + self.assertFalse( + self._cs_pool_exists(pool.id), + "Pool '%s' still listed in CloudStack after deletion with " + "pre-deleted igroups" % pool_name + ) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' still exists after pool deletion" + % pool_name + ) + self._assert_igroups_for_hosts(expect_present=False) + finally: + for igroup_name in seeded: + try: + self.ontap.delete_igroup(self.svm_name, igroup_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete seeded igroup '%s': %s", + igroup_name, exc) + self._force_cleanup_zone_pool(pool) + self._force_delete_flexvol(pool_name) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py index 5d1812cdad4f..4aed98462c5f 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -18,18 +18,22 @@ """ Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. -Tests are numbered test_01 ... test_08 and must run in that order. Each step +Tests are numbered test_01 ... test_12 and must run in that order. Each step builds on the shared state established by the previous step. Workflow: - 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Delete the storage pool - 07 Create fresh pool and allocate a CloudStack volume - 08 Delete volume then force-delete the pool + 01 Reject create when a FlexVol of that name already exists on ONTAP + 02 Reject create when no online assigned aggregate has enough free space + 03 Create primary storage pool + 04 Disable storage pool + 05 Enable storage pool + 06 Enter maintenance mode + 07 Cancel maintenance mode + 08 Delete the storage pool + 09 Create fresh pool and allocate a CloudStack volume + 10 Delete volume then force-delete the pool + 11 Delete an empty pool whose FlexVol was deleted directly on ONTAP + 12 Delete an empty pool whose NFS export policy was deleted on ONTAP Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -42,7 +46,7 @@ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v -Note: Tests 01-06 share class-level state (sequential). Running a single test +Note: Tests 03-08 share class-level state (sequential). Running a single test with -m "test_NN" will invoke setUpClass but the guard assertion will fail immediately if earlier steps have not yet run. Always run the full suite. """ @@ -96,6 +100,7 @@ class TestData: DETAIL_NFS_MOUNT_OPTS = "nfsmountopts" ONTAP_MIN_VOLUME_SIZE = 1677721600 + ONTAP_MAX_VOLUME_SIZE = 300 * 1024 ** 4 def __init__(self, storage_ip, svm_name, username, password, protocol="NFS3", scope="CLUSTER", provider="NetApp ONTAP", @@ -199,10 +204,14 @@ def setUpClass(cls): # Helpers # ------------------------------------------------------------------ - def _create_pool(self): + def _create_pool(self, pool_name=None, capacitybytes=None): + """Create a pool; name and capacity default to the suite's values.""" ps = self.testdata[TestData.primaryStorage] storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] - pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + if pool_name is None: + pool_name = "OntapNFS3_%d" % random.randint(0, 99999) + if capacitybytes is None: + capacitybytes = ps["capacitybytes"] cmd = createStoragePoolAPI.createStoragePoolCmd() cmd.name = pool_name @@ -213,7 +222,7 @@ def _create_pool(self): cmd.scope = ps[TestData.scope] cmd.provider = ps[TestData.provider] cmd.tags = ps[TestData.tags] - cmd.capacitybytes = ps["capacitybytes"] + cmd.capacitybytes = capacitybytes cmd.hypervisor = "KVM" cmd.managed = True @@ -401,7 +410,123 @@ def _delete_volume_then_force_delete_pool(self, pool, vol, ep_name): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_01_create_primary_storage_pool(self): + def test_01_reject_create_when_flexvol_name_exists(self): + """ + Pre-create a FlexVol on ONTAP, then ask CloudStack for a pool of the + same name. ONTAP refuses the duplicate, so the create must fail. + + Verifies: + - createStoragePool raises CloudstackAPIException + - no pool of that name is left in CloudStack + - the pre-existing FlexVol is untouched (the plugin must not + adopt or delete a volume it did not create) + """ + self._sweep_tracked_pool2() + pool_name = self._throwaway_pool_name("Dup") + try: + self.ontap.create_flexvol( + self.svm_name, pool_name, TestData.ONTAP_MIN_VOLUME_SIZE + ) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-created ONTAP FlexVol '%s' not found; cannot test the " + "duplicate-name rejection" % pool_name, + ) + log_progress( + logger, "info", + "Pre-created FlexVol '%s'; requesting a pool of the same name " + "(expect reject)", pool_name, + ) + with self.assertRaises(CloudstackAPIException) as caught: + self.__class__.pool2 = self._create_pool(pool_name=pool_name) + log_progress( + logger, "info", + "Rejected duplicate-name create for '%s': %s", + pool_name, caught.exception, + ) + self._assert_no_pool_named(pool_name) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-existing ONTAP FlexVol '%s' was removed by the failed " + "pool create" % pool_name, + ) + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool_name, + ep_name="cs-%s-%s" % (self.svm_name, pool_name), + ) + + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_02_reject_create_when_no_aggregate_space(self): + """ + Ask for 1 GiB more than the largest online aggregate assigned to the + SVM can provide, so no aggregate qualifies and the plugin refuses + before creating anything. + + Verifies: + - createStoragePool raises CloudstackAPIException + - the error names the aggregate shortage rather than some other + failure ('No suitable aggregates') + - no pool is left in CloudStack and no FlexVol on ONTAP + """ + self._sweep_tracked_pool2() + max_free = self.ontap.max_online_aggregate_available_bytes( + self.svm_name + ) + if not max_free: + self.skipTest( + "No online aggregate with reported free space is assigned to " + "SVM '%s'; cannot build an unsatisfiable request" + % self.svm_name + ) + requested = int(max_free) + 1024 ** 3 + if requested > TestData.ONTAP_MAX_VOLUME_SIZE: + self.skipTest( + "Largest aggregate free space (%d B) + 1 GiB exceeds the " + "ONTAP FlexVol maximum (%d B); the request would be refused " + "for the size limit rather than the aggregate shortage" + % (max_free, TestData.ONTAP_MAX_VOLUME_SIZE) + ) + + pool_name = self._throwaway_pool_name("NoSpace") + log_progress( + logger, "info", + "Requesting pool '%s' of %d B; largest online aggregate on SVM " + "'%s' has %d B free (expect reject)", + pool_name, requested, self.svm_name, max_free, + ) + try: + with self.assertRaises(CloudstackAPIException) as caught: + self.__class__.pool2 = self._create_pool( + pool_name=pool_name, capacitybytes=requested + ) + error_text = str(caught.exception) + log_progress( + logger, "info", + "Rejected no-space create for '%s': %s", pool_name, error_text, + ) + self.assertIn( + "No suitable aggregates", error_text, + "Expected the rejection to report 'No suitable aggregates', " + "got: %s" % error_text, + ) + self._assert_no_pool_named(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' was created despite the rejected pool " + "create" % pool_name, + ) + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool_name, + ep_name="cs-%s-%s" % (self.svm_name, pool_name), + ) + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_create_primary_storage_pool(self): """ Create an NFS3 primary storage pool and verify: - CloudStack state is Up, type is NetworkFilesystem @@ -462,13 +587,13 @@ def test_01_create_primary_storage_pool(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): + def test_04_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled - ONTAP: FlexVol is still online and export policy unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -498,13 +623,13 @@ def test_02_disable_storage_pool(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): + def test_05_enable_storage_pool(self): """ Re-enable the pool and verify: - CloudStack reports Up - ONTAP: FlexVol is still online and export policy unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -534,14 +659,14 @@ def test_03_enable_storage_pool(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): + def test_06_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance - ONTAP: FlexVol is still online and export policy unchanged (maintenance is a CS-only state change) """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() cmd.id = self.__class__.pool.id @@ -570,7 +695,7 @@ def test_04_enter_maintenance_mode(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): + def test_07_cancel_maintenance_mode(self): """ Cancel maintenance mode and verify the pool returns to Up. @@ -580,7 +705,7 @@ def test_05_cancel_maintenance_mode(self): - ONTAP: NFS export policy still present """ self.assertIsNotNone(self.__class__.pool, - "Pool absent — test_01 must pass first") + "Pool absent — test_03 must pass first") cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() cmd.id = self.__class__.pool.id @@ -620,7 +745,7 @@ def test_05_cancel_maintenance_mode(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_06_delete_pool_from_maintenance(self): + def test_08_delete_pool_from_maintenance(self): """ Enter maintenance mode then delete the storage pool. @@ -629,7 +754,7 @@ def test_06_delete_pool_from_maintenance(self): - ONTAP: FlexVol is deleted - ONTAP: NFS export policy is deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent — test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent — test_03 must pass first") pool = self.__class__.pool pool_name = pool.name ep_name = self.__class__.pool_ep_name @@ -671,7 +796,7 @@ def test_06_delete_pool_from_maintenance(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_07_create_volume_on_pool(self): + def test_09_create_volume_on_pool(self): """ Create a new NFS3 pool and allocate a CloudStack data volume. For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record @@ -737,7 +862,7 @@ def test_07_create_volume_on_pool(self): # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_08_delete_volume_and_pool(self): + def test_10_delete_volume_and_pool(self): """ Delete the volume from test_07, enter maintenance, then force-delete the pool. @@ -748,8 +873,8 @@ def test_08_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: export policy deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_09 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_09 must pass first") pool = self.__class__.pool pool_name = pool.name @@ -779,3 +904,240 @@ def test_08_delete_volume_and_pool(self): ) self.__class__.pool2 = None self.__class__.pool2_ep_name = None + + def _throwaway_pool_name(self, suffix): + return "OntapNFS3%s_%d" % (suffix, random.randint(0, 99999)) + + + def _cleanup_throwaway_pool(self, pool, flexvol_name=None, ep_name=None): + """Best-effort teardown for an isolated test: CloudStack, then ONTAP. + + Never raises, so a failed assertion in the test body is the error + that surfaces. The KVM unmount happens while the export is still + reachable, i.e. before the FlexVol goes away. + """ + backend_cleanup_safe = pool is None + if pool is not None: + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + listed = None + backend_cleanup_safe = not listed + if listed: + try: + if listed[0].state != "Maintenance": + maint_cmd = ( + enableStorageMaintenance + .enableStorageMaintenanceCmd() + ) + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state( + pool.id, "Maintenance", timeout=120 + ) + except Exception as exc: + logger.warning( + "cleanup: could not put pool '%s' into Maintenance: %s", + pool.name, exc, + ) + try: + self._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as exc: + logger.warning( + "cleanup: could not safely unmount pool '%s': %s", + pool.name, exc, + ) + return + backend_cleanup_safe = True + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "cleanup: could not delete pool '%s': %s", + pool.name, exc, + ) + if flexvol_name is None: + flexvol_name = pool.name + if backend_cleanup_safe and flexvol_name: + try: + if self.ontap.get_volume(flexvol_name) is not None: + self.ontap.offline_and_delete_volume(flexvol_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete ONTAP FlexVol '%s': %s", + flexvol_name, exc, + ) + if backend_cleanup_safe and ep_name: + try: + if self.ontap.get_export_policy(ep_name) is not None: + self.ontap.delete_export_policy(ep_name) + except Exception as exc: + logger.warning( + "cleanup: could not delete export policy '%s': %s", + ep_name, exc, + ) + if pool is None: + return + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except Exception: + remaining = None + if not remaining: + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + + def _assert_no_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with this name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except CloudstackAPIException: + listed = None + self.assertFalse( + listed, + "CloudStack should hold no pool named '%s' after a rejected " + "create, found: %s" % (pool_name, listed), + ) + + + def _sweep_tracked_pool2(self): + """Clean a prior leftover before reusing the shared pool2 slot.""" + if self.__class__.pool2 is None: + return + self._cleanup_throwaway_pool( + self.__class__.pool2, + ep_name=self.__class__.pool2_ep_name, + ) + if self.__class__.pool2 is not None: + self.skipTest("A previously tracked pool could not be cleaned up safely") + + + def _create_throwaway_pool(self, suffix): + """Create an isolated pool, park it in pool2, and return it.""" + self._sweep_tracked_pool2() + pool = self._create_pool(pool_name=self._throwaway_pool_name(suffix)) + self.__class__.pool2 = pool + ep_name = self._get_export_policy_name(pool) + self.__class__.pool2_ep_name = ep_name + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (pool.name, pool.state), + ) + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol missing for throwaway pool '%s'" % pool.name, + ) + return pool, ep_name + + + def _enter_maintenance(self, pool): + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool.id + self.apiClient.enableStorageMaintenance(maint_cmd) + self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_11_delete_pool_with_flexvol_predeleted(self): + """ + Delete the backing FlexVol directly on ONTAP, then delete the empty + pool through CloudStack. Deletion must tolerate the missing volume + rather than leaving an undeletable pool behind. + + Verifies: + - deleteStoragePool succeeds with the FlexVol already gone + - the pool is removed from CloudStack + - the export policy is cleaned up as well + """ + pool, ep_name = self._create_throwaway_pool("PreDelVol") + try: + self._enter_maintenance(pool) + # Unmount on the KVM hosts first: once the FlexVol is gone the + # export is unreachable and a stale mount can wedge the host. + self._cleanup_kvm_storage_pool_mounts(pool.id) + log_progress( + logger, "info", + "Deleting ONTAP FlexVol '%s' behind CloudStack's back", + pool.name, + ) + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still present after direct deletion" + % pool.name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' reappeared after pool deletion" + % pool.name, + ) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' still exists after pool deletion" + % ep_name, + ) + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool.name, + ep_name=ep_name, + ) + + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_12_delete_pool_with_export_policy_predeleted(self): + """ + Delete the NFS export policy directly on ONTAP, then delete the + empty pool through CloudStack. Deletion must tolerate the missing + policy and still remove the FlexVol. + + Verifies: + - deleteStoragePool succeeds with the export policy already gone + - the pool is removed from CloudStack + - ONTAP: the FlexVol is deleted + """ + pool, ep_name = self._create_throwaway_pool("PreDelEp") + try: + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' missing for the fresh throwaway pool" + % ep_name, + ) + self._enter_maintenance(pool) + # Dropping the policy revokes the hosts' NFS access, so unmount + # before it disappears. + self._cleanup_kvm_storage_pool_mounts(pool.id) + log_progress( + logger, "info", + "Deleting NFS export policy '%s' behind CloudStack's back", + ep_name, + ) + self.ontap.reassign_volume_export_policy(pool.name) + self.ontap.delete_export_policy(ep_name) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' still present after direct deletion" + % ep_name, + ) + + self._delete_pool(pool.id) + self._assert_pool_gone_from_cs(pool.id, pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol '%s' still exists after pool deletion" + % pool.name, + ) + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + finally: + self._cleanup_throwaway_pool( + self.__class__.pool2, + flexvol_name=pool.name, + ep_name=ep_name, + ) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py index b266c1920f9d..0253f23b90f0 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_with_volumes.py @@ -30,6 +30,14 @@ 06 Forced=False delete rejected — pool stays in Maintenance (negative) 07 Cleanup — cancel maintenance, delete volume, force-delete pool +Isolated tests (test_08 onwards) run after that workflow and share no state +with it. Each one builds its own pool plus CloudStack volume, breaks a single +ONTAP object behind CloudStack's back, and force-deletes the pool: + + 08 FlexVol pre-deleted on ONTAP, then force-delete pool with CS volume + 09 Export policy pre-deleted on ONTAP, then force-delete pool with CS volume + 10 Cancel maintenance once the CS volume has been deleted + Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed - KVM cluster registered in CloudStack @@ -69,6 +77,7 @@ cancelStorageMaintenance, createStoragePool as createStoragePoolAPI, deleteVolume as deleteVolumeAPI, + destroyVolume as destroyVolumeAPI, enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) @@ -150,6 +159,7 @@ class TestOntapNFS3PoolWithVolumes(OntapTestBase): """ pool_ep_name = None # NFS export policy name extracted at pool creation + pool2_ep_name = None # export policy of the pool an isolated test builds _vol_name_prefix = "OntapNFS3WV" @@ -749,3 +759,420 @@ def test_07_force_delete_pool_and_cleanup(self): policy, "NFS export policy '%s' should be removed after cleanup" % ep_name ) + + # ================================================================== + # Isolated tests — appended after the sequential workflow above. + # + # Each one creates its own pool and CloudStack volume in the pool2 / + # volume2 slots (which OntapTestBase.tearDownClass also sweeps), runs a + # single scenario, and cleans up in a finally block. They never reuse a + # pool destroyed by another test. + # ================================================================== + + def _enter_maintenance(self, pool): + """Put the pool into Maintenance and wait for the state to settle.""" + cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + cmd.id = pool.id + self.apiClient.enableStorageMaintenance(cmd) + return self._poll_pool_state(pool.id, "Maintenance", timeout=120) + + def _create_isolated_pool_with_volume(self, label): + """Build a fresh pool plus CS volume for one isolated scenario. + + Returns ``(pool, volume, export_policy_name)``. Any pool a previous + isolated test could not clean up is swept first so its ONTAP objects + are never orphaned by the overwrite of the pool2 slot. + """ + if self.__class__.pool2 is not None: + self._cleanup_isolated_pool( + self.__class__.pool2, self.__class__.pool2_ep_name, + "leftover-from-previous-isolated-test" + ) + + pool = self._create_pool() + self.__class__.pool2 = pool + ep_name = self._get_export_policy_name(pool) + self.__class__.pool2_ep_name = ep_name + logger.info("[%s] created isolated pool '%s' (ep '%s')", + label, pool.name, ep_name) + + self.assertEqual( + pool.state, "Up", + "[%s] new pool state should be 'Up', got '%s'" % (label, pool.state) + ) + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol not found for new pool '%s'" % (label, pool.name) + ) + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' not found for new pool '%s'" + % (label, ep_name, pool.name) + ) + + vol = self._create_volume(pool.id) + self.__class__.volume2 = vol + self.assertIsNotNone(vol, "[%s] createVolume returned None" % label) + return pool, vol, ep_name + + def _assert_pool_absent(self, pool, label, delete_error=None): + """Assert CloudStack no longer lists the pool.""" + try: + remaining = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + remaining = None + self.assertFalse( + remaining, + "[%s] pool '%s' is still listed after deleteStoragePool(forced=True)%s" + % (label, pool.name, + "; the API raised: %s" % delete_error if delete_error else "") + ) + + def _purge_cs_volume_record(self, vol, label): + """Remove a CS volume record the forced pool delete may have left. + + The backing storage is already gone at this point, so a failure here + only affects tidiness — the volume stays in the volume2 slot for + tearDownClass to retry and no exception is raised. + """ + if vol is None: + return + if self._volume_exists_in_cs(vol.id): + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + logger.info("[%s] deleted leftover CS volume record %s", + label, vol.id) + except Exception as exc: + logger.warning("[%s] could not delete leftover CS volume %s: %s", + label, vol.id, exc) + else: + logger.info("[%s] CS volume %s was removed along with the pool", + label, vol.id) + if not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + + def _exit_maintenance(self, pool, label): + """Bring a pool out of Maintenance so its volumes can be deleted.""" + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + return False + if not listed: + return False + if listed[0].state != "Maintenance": + return True + try: + cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cmd.id = pool.id + self.apiClient.cancelStorageMaintenance(cmd) + self._poll_pool_state(pool.id, "Up", timeout=120) + return True + except Exception as exc: + logger.warning("[%s] could not cancel maintenance on '%s': %s", + label, pool.name, exc) + return False + + def _enter_maintenance_quietly(self, pool, label): + """Enter Maintenance, tolerating a pool whose backend is already gone.""" + try: + self._enter_maintenance(pool) + except Exception as exc: + logger.warning("[%s] could not enter maintenance on '%s': %s", + label, pool.name, exc) + + DESTROYED_VOLUME_STATES = ("destroy", "destroyed", "expunging", "expunged") + + def _cs_volume_state(self, vol_id): + """Return the CloudStack volume state, or None when it is not listed.""" + vol = self._get_cs_volume(vol_id) + return getattr(vol, "state", None) if vol is not None else None + + def _volume_cleared_for_pool_delete(self, vol_id): + """True once the volume no longer blocks deleteStoragePool(forced).""" + state = self._cs_volume_state(vol_id) + return state is None or state.lower() in self.DESTROYED_VOLUME_STATES + + def _remove_cs_volume(self, pool, vol, label): + """Clear the CloudStack volume so the pool can be force-deleted. + + deleteStoragePool(forced=True) refuses while any volume on the pool is + in a state other than Destroy. deleteVolume is tried first because it + also reclaims the backing storage, but it expunges through libvirt and + fails when the FlexVol is already gone. destroyVolume(expunge=False) + is the fallback: it only moves the record to Destroy, which is all the + forced pool delete requires - it expunges the leftovers itself. + """ + if vol is None or not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + return True + self._exit_maintenance(pool, label) + try: + cmd = deleteVolumeAPI.deleteVolumeCmd() + cmd.id = vol.id + self.apiClient.deleteVolume(cmd) + except Exception as exc: + logger.warning("[%s] deleteVolume failed for %s (%s); falling back " + "to destroyVolume without expunge", + label, vol.id, exc) + try: + cmd = destroyVolumeAPI.destroyVolumeCmd() + cmd.id = vol.id + cmd.expunge = False + self.apiClient.destroyVolume(cmd) + except Exception as destroy_exc: + logger.warning("[%s] destroyVolume also failed for %s: %s", + label, vol.id, destroy_exc) + if not self._volume_cleared_for_pool_delete(vol.id): + return False + if not self._volume_exists_in_cs(vol.id): + self.__class__.volume2 = None + return True + + def _cleanup_isolated_pool(self, pool, ep_name, label): + """Best-effort teardown of one isolated pool and its ONTAP objects.""" + if pool is None: + return + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + listed = None + backend_cleanup_safe = not listed + if listed: + self._remove_cs_volume(pool, self.__class__.volume2, label) + try: + listed = list_storage_pools(self.apiClient, id=pool.id) or listed + if listed[0].state != "Maintenance": + self._enter_maintenance(pool) + self._cleanup_kvm_storage_pool_mounts(pool.id) + except Exception as exc: + logger.warning("[%s] could not safely unmount pool '%s': %s", + label, pool.name, exc) + return + backend_cleanup_safe = True + try: + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning("[%s] could not force-delete pool '%s': %s", + label, pool.name, exc) + if not backend_cleanup_safe: + return + try: + self.ontap.offline_and_delete_volume(pool.name) + except Exception as exc: + logger.warning("[%s] ONTAP FlexVol cleanup for '%s' failed: %s", + label, pool.name, exc) + if ep_name: + try: + self.ontap.delete_export_policy(ep_name) + except Exception as exc: + logger.warning( + "[%s] ONTAP export policy cleanup for '%s' failed: %s", + label, ep_name, exc) + try: + listed = list_storage_pools(self.apiClient, id=pool.id) + except CloudstackAPIException: + listed = None + if not listed: + self.__class__.pool2 = None + self.__class__.pool2_ep_name = None + + # ------------------------------------------------------------------ + # Step 08 — FlexVol deleted on ONTAP before the pool delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_08_delete_pool_with_volume_flexvol_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after its + ONTAP FlexVol has been removed behind CloudStack's back. + + Uses its own pool and volume, so the pool is known to be healthy up to + the point the FlexVol is destroyed. Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing FlexVol + - the CloudStack pool record is removed + - the leftover CS volume record can still be cleaned up + """ + label = "flexvol-missing" + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool) + + # Unmount on every KVM host while the NFS export is still + # reachable, before the FlexVol is destroyed underneath it. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self.ontap.offline_and_delete_volume(pool.name) + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should be gone before the pool delete" + % (label, pool.name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + self._enter_maintenance_quietly(pool, label) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate a missing " + "FlexVol, but raised: %s" % (label, delete_error) + ) + + self._purge_cs_volume_record(vol, label) + finally: + self._cleanup_isolated_pool(pool, ep_name, label) + + # ------------------------------------------------------------------ + # Step 09 — Export policy deleted on ONTAP before the delete (negative) + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_09_delete_pool_with_volume_export_policy_missing(self): + """ + Force-delete a pool that still owns a CloudStack volume after its NFS + export policy has been removed behind CloudStack's back. + + Uses its own pool and volume. Unlike test_08 the FlexVol is still + present, so the plugin is expected to remove it as part of the delete. + Verifies: + - deleteStoragePool is rejected while the CS volume still exists + - deleteStoragePool(forced=True) tolerates the missing export policy + - the CloudStack pool record is removed + - the ONTAP FlexVol is deleted despite the missing policy + """ + label = "export-policy-missing" + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + try: + self._enter_maintenance(pool) + + # Unmount before the export policy goes away, otherwise the KVM + # hosts are left holding a mount they can no longer reach. + self._cleanup_kvm_storage_pool_mounts(pool.id) + + self.ontap.reassign_volume_export_policy(pool.name) + self.ontap.delete_export_policy(ep_name) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' should be gone before the pool delete" + % (label, ep_name) + ) + + # CloudStack rejects deleteStoragePool while the pool still owns + # a volume, even with forced=True, so the volume goes first. + with self.assertRaises(CloudstackAPIException): + self._delete_pool(pool.id, forced=True) + self.assertTrue( + self._remove_cs_volume(pool, vol, label), + "[%s] CloudStack volume could not be deleted before the pool " + "delete" % label + ) + self._enter_maintenance_quietly(pool, label) + + delete_error = None + try: + self._delete_pool(pool.id, forced=True) + except CloudstackAPIException as exc: + delete_error = exc + + self._assert_pool_absent(pool, label, delete_error) + self.assertIsNone( + delete_error, + "[%s] deleteStoragePool(forced=True) should tolerate a missing " + "export policy, but raised: %s" % (label, delete_error) + ) + + self.assertIsNone( + self.ontap.get_volume(pool.name), + "[%s] ONTAP FlexVol '%s' should have been deleted with the pool" + % (label, pool.name) + ) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' reappeared during the pool delete" + % (label, ep_name) + ) + + self._purge_cs_volume_record(vol, label) + finally: + self._cleanup_isolated_pool(pool, ep_name, label) + + # ------------------------------------------------------------------ + # Step 10 — Cancel maintenance once the CS volume has been deleted + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_with_volumes"], required_hardware=True) + def test_10_cancel_maintenance_after_volume_deleted(self): + """ + Cancel maintenance on a pool whose CloudStack volume has been deleted. + + Complements test_05, which cancels maintenance with the volume still + present. The volume is deleted while the pool is Up because on NFS3 + the KVM agent cannot service a deleteVolume for a pool already in + Maintenance. Verifies: + - the pool returns to Up + - the ONTAP FlexVol is still online and the export policy intact + """ + label = "cancel-maintenance-no-volume" + pool, vol, ep_name = self._create_isolated_pool_with_volume(label) + try: + del_cmd = deleteVolumeAPI.deleteVolumeCmd() + del_cmd.id = vol.id + self.apiClient.deleteVolume(del_cmd) + self.assertFalse( + self._volume_exists_in_cs(vol.id), + "[%s] CS volume %s should be gone before entering Maintenance" + % (label, vol.id) + ) + self.__class__.volume2 = None + vol = None + + self._enter_maintenance(pool) + + cancel_cmd = cancelStorageMaintenance.cancelStorageMaintenanceCmd() + cancel_cmd.id = pool.id + self.apiClient.cancelStorageMaintenance(cancel_cmd) + + result = self._poll_pool_state(pool.id, "Up", timeout=120) + self.assertEqual( + result.state, "Up", + "[%s] pool should be 'Up' after cancel maintenance, got '%s'" + % (label, result.state) + ) + + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol '%s' disappeared after cancel maintenance" + % (label, pool.name) + ) + self.assertEqual( + ontap_vol.get("state"), "online", + "[%s] ONTAP FlexVol should be 'online' after cancel " + "maintenance, got '%s'" % (label, ontap_vol.get("state")) + ) + self.assertIsNotNone( + self.ontap.get_export_policy(ep_name), + "[%s] export policy '%s' should survive cancel maintenance" + % (label, ep_name) + ) + finally: + self._purge_cs_volume_record(vol, label) + self._cleanup_isolated_pool(pool, ep_name, label) diff --git a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py index 88a6309f1ee1..22e2cd86eec7 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py @@ -23,12 +23,18 @@ hosts in the zone to the pool and creates an NFS export policy covering their IPs. -Workflow: - 01 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; - export policy has all cluster host IPs - 02 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged - 03 Enable zone-scoped pool — pool.state Up; FlexVol unchanged - 04 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted +Test order — 03-06 are a sequential workflow that must run in order; 01, 02, +07 and 08 are isolated negative/recovery cases, each owning the pool it +creates, so they can be run on their own: + 01 Create rejected when a FlexVol of the same name already exists + 02 Create rejected when no assigned online aggregate has enough free space + 03 Create zone-scoped NFS3 pool — pool.state Up; ONTAP FlexVol online; + export policy has all cluster host IPs + 04 Disable zone-scoped pool — pool.state Disabled; FlexVol unchanged + 05 Enable zone-scoped pool — pool.state Up; FlexVol unchanged + 06 Delete zone-scoped pool — pool gone; FlexVol deleted; export policy deleted + 07 Delete an empty pool whose FlexVol was removed behind CloudStack's back + 08 Delete an empty pool whose NFS export policy was removed beforehand Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -41,7 +47,7 @@ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ test/integration/plugins/ontap/nfs3/pool/test_zone_scoped_pool.py -v -Note: Tests 01-04 share class-level state (sequential). Running a single test +Note: Tests 03-06 share class-level state (sequential). Running a single test with -m "test_NN" will invoke setUpClass but the guard assertion will fail immediately if earlier steps have not yet run. Always run the full suite. """ @@ -49,6 +55,7 @@ import base64 import logging import random +import time import unittest from nose.plugins.attrib import attr @@ -58,10 +65,17 @@ enableStorageMaintenance, updateStoragePool as updateStoragePoolAPI, ) +from marvin.cloudstackException import CloudstackAPIException from marvin.lib.base import StoragePool from marvin.lib.common import list_storage_pools -from ontap_test_base import OntapRestClient, OntapTestBase, _parse_pool_details, get_datacenter_config +from ontap_test_base import ( + OntapRestClient, + OntapTestBase, + _parse_pool_details, + get_datacenter_config, + log_progress, +) logger = logging.getLogger("TestOntapZoneScopedPool") @@ -136,6 +150,11 @@ class TestOntapZoneScopedPool(OntapTestBase): _vol_name_prefix = "OntapZoneVol" + ONE_GIB = 1024 ** 3 + # Above this much free aggregate space, asking for "max free + 1 GiB" + # stops being a meaningful request, so the no-space test skips instead. + MAX_AGGREGATE_FREE_FOR_NO_SPACE_TEST = 300 * 1024 ** 4 + @classmethod def setUpClass(cls): super(TestOntapZoneScopedPool, cls).setUpClass() @@ -186,11 +205,17 @@ def setUpClass(cls): # Helpers # ------------------------------------------------------------------ - def _create_zone_pool(self): - """Create a zone-scoped NFS3 pool (no clusterid / podid).""" + def _create_zone_pool(self, name=None, capacitybytes=None): + """Create a zone-scoped NFS3 pool (no clusterid / podid). + + ``name`` and ``capacitybytes`` let the isolated negative tests drive + the pool name (to collide with a pre-created FlexVol) and the + requested size (to exceed every aggregate) without touching the + shared test data. + """ ps = self.testdata[TestData.primaryStorage] storage_ip = self.testdata[TestData.ontap][TestData.DETAIL_STORAGE_IP] - pool_name = "OntapZoneNFS3_%d" % random.randint(0, 99999) + pool_name = name or "OntapZoneNFS3_%d" % random.randint(0, 99999) cmd = createStoragePoolAPI.createStoragePoolCmd() cmd.name = pool_name @@ -200,7 +225,7 @@ def _create_zone_pool(self): cmd.scope = "ZONE" cmd.provider = ps[TestData.provider] cmd.tags = ps[TestData.tags] - cmd.capacitybytes = ps["capacitybytes"] + cmd.capacitybytes = capacitybytes or ps["capacitybytes"] cmd.hypervisor = "KVM" cmd.managed = True @@ -240,12 +265,259 @@ def _assert_export_policy_has_host_ips(self, ep_name): % (ip, ep_name, all_clients) ) + # ---- helpers for the isolated tests (01, 02, 09, 10) --------------- + + def _require_ontap_client(self, *method_names): + """Skip when the shared OntapRestClient lacks a backend helper.""" + missing = [n for n in method_names if not hasattr(self.ontap, n)] + if missing: + raise unittest.SkipTest( + "OntapRestClient does not provide %s; update " + "ontap_test_base.py before running this test" + % ", ".join(missing) + ) + + def _create_isolated_zone_pool(self, name): + """Create a throwaway zone pool and register it for class teardown.""" + pool = self._create_zone_pool(name=name) + self.__class__.pool2 = pool + self.assertEqual( + pool.state, "Up", + "Throwaway pool '%s' should be 'Up', got '%s'" + % (name, pool.state) + ) + return pool + + def _wait_for_pool_state_quietly(self, pool_id, target_state, + timeout=120, interval=5): + """Poll for a pool state, returning False instead of failing the test. + + Used on the recovery paths where the backend has deliberately been + broken, so entering Maintenance is allowed to fail. + """ + deadline = time.time() + timeout + while time.time() < deadline: + pools = list_storage_pools(self.apiClient, id=pool_id) + if not pools or pools[0].state == target_state: + return True + time.sleep(interval) + return False + + def _enter_maintenance_quietly(self, pool_id): + """Request Maintenance without failing when the backend is broken.""" + try: + maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() + maint_cmd.id = pool_id + self.apiClient.enableStorageMaintenance(maint_cmd) + except Exception as exc: + logger.warning( + "enableStorageMaintenance failed for pool %s: %s" + % (pool_id, exc) + ) + return False + return self._wait_for_pool_state_quietly(pool_id, "Maintenance") + + def _cs_pool_exists(self, pool_id): + try: + return bool(list_storage_pools(self.apiClient, id=pool_id)) + except Exception: + return False + + def _assert_no_cs_pool_named(self, pool_name): + """Assert CloudStack holds no storage pool with the given name.""" + try: + listed = list_storage_pools(self.apiClient, name=pool_name) + except Exception: + listed = None + self.assertFalse( + listed, + "CloudStack should not have created pool '%s' after the " + "rejected request" % pool_name + ) + + def _force_cleanup_zone_pool(self, pool): + """Best-effort removal of a throwaway pool left behind by a failure. + + Unmounts on the KVM hosts first so a stale NFS mount can never + outlive the ONTAP export and trip KVMHAMonitor. + """ + if pool is None: + return True + if not self._cs_pool_exists(pool.id): + self.__class__.pool2 = None + return True + try: + self._enter_maintenance_quietly(pool.id) + self._cleanup_kvm_storage_pool_mounts(pool.id) + self._delete_pool(pool.id, forced=True) + except Exception as exc: + logger.warning( + "could not force-delete throwaway pool %s: %s" + % (pool.id, exc) + ) + return False + if self._cs_pool_exists(pool.id): + logger.warning( + "throwaway pool %s still exists after force-delete", pool.id + ) + return False + self.__class__.pool2 = None + return True + + def _force_delete_flexvol(self, vol_name): + """Best-effort ONTAP FlexVol removal for a throwaway volume.""" + try: + self.ontap.offline_and_delete_volume(vol_name) + except Exception as exc: + logger.warning( + "could not delete ONTAP FlexVol '%s': %s" % (vol_name, exc) + ) + + def _force_delete_export_policy(self, ep_name): + """Best-effort ONTAP export policy removal for a throwaway pool.""" + if not ep_name: + return + try: + self.ontap.delete_export_policy(ep_name) + except Exception as exc: + logger.warning( + "could not delete export policy '%s': %s" % (ep_name, exc) + ) + + # ------------------------------------------------------------------ + # Step 01 — Create rejected when a FlexVol of the same name exists + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_01_create_zone_pool_rejected_when_flexvol_exists(self): + """ + Pre-create a FlexVol on the SVM, then ask CloudStack for a + zone-scoped pool with that exact name. + Verifies: + - createStoragePool raises CloudstackAPIException + - CloudStack records no pool with that name + - ONTAP: the pre-existing FlexVol is left in place + This test owns everything it creates and leaves no pool behind. + """ + self._require_ontap_client("create_flexvol", "offline_and_delete_volume") + + pool_name = "OntapZoneNFS3Dup_%d" % random.randint(0, 99999) + size_bytes = self.testdata[TestData.primaryStorage]["capacitybytes"] + created_pool = None + try: + self.ontap.create_flexvol(self.svm_name, pool_name, size_bytes) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-created ONTAP FlexVol '%s' not visible; cannot test the " + "name collision" % pool_name + ) + + try: + created_pool = self._create_zone_pool(name=pool_name) + except CloudstackAPIException as exc: + log_progress( + logger, "info", + "createStoragePool rejected for existing FlexVol '%s': %s", + pool_name, exc, + ) + else: + self.fail( + "createStoragePool should have been rejected: ONTAP " + "FlexVol '%s' already exists" % pool_name + ) + + self._assert_no_cs_pool_named(pool_name) + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "Pre-existing ONTAP FlexVol '%s' was removed by the rejected " + "create" % pool_name + ) + finally: + if self._force_cleanup_zone_pool(created_pool): + self._force_delete_flexvol(pool_name) + self._force_delete_export_policy( + "cs-%s-%s" % (self.svm_name, pool_name) + ) + # ------------------------------------------------------------------ - # Step 01 — Create zone-scoped pool + # Step 02 — Create rejected when no aggregate has enough free space # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_01_create_zone_scoped_pool(self): + def test_02_create_zone_pool_rejected_when_no_aggregate_space(self): + """ + Ask for a pool 1 GiB larger than the free space of the roomiest + assigned online aggregate. + Verifies: + - createStoragePool raises CloudstackAPIException reporting + 'No suitable aggregates' + - CloudStack records no pool with that name + - ONTAP: no FlexVol of that name was left behind + Skipped when the SVM has more than 300 TiB free on one aggregate, + where the oversized request stops being meaningful. + """ + self._require_ontap_client( + "max_online_aggregate_available_bytes", "offline_and_delete_volume" + ) + + max_free = self.ontap.max_online_aggregate_available_bytes(self.svm_name) + if not max_free: + raise unittest.SkipTest( + "No assigned online aggregate with free space reported for " + "SVM '%s'; cannot build an over-capacity request" + % self.svm_name + ) + requested = int(max_free) + self.ONE_GIB + if requested > self.MAX_AGGREGATE_FREE_FOR_NO_SPACE_TEST: + raise unittest.SkipTest( + "Largest assigned online aggregate on SVM '%s' has %d B free; " + "the over-capacity request would exceed the %d B FlexVol limit" + % (self.svm_name, max_free, + self.MAX_AGGREGATE_FREE_FOR_NO_SPACE_TEST) + ) + + pool_name = "OntapZoneNFS3NoSpace_%d" % random.randint(0, 99999) + created_pool = None + try: + try: + created_pool = self._create_zone_pool( + name=pool_name, capacitybytes=requested + ) + except CloudstackAPIException as exc: + error_text = str(exc) + log_progress( + logger, "info", + "createStoragePool rejected for %d B (max aggregate free " + "%d B): %s", requested, max_free, error_text, + ) + self.assertIn( + "No suitable aggregates", error_text, + "Expected the rejection to report 'No suitable " + "aggregates', got: %s" % error_text + ) + else: + self.fail( + "createStoragePool should have been rejected: requested " + "%d B but the roomiest aggregate has only %d B free" + % (requested, max_free) + ) + + self._assert_no_cs_pool_named(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' was left behind by the rejected create" + % pool_name + ) + finally: + if self._force_cleanup_zone_pool(created_pool): + self._force_delete_flexvol(pool_name) + + # ------------------------------------------------------------------ + # Step 03 — Create zone-scoped pool + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_03_create_zone_scoped_pool(self): """ Create a zone-scoped NFS3 primary storage pool (no clusterid/podid). CloudStack calls attachZone(), which connects all eligible KVM hosts @@ -288,18 +560,18 @@ def test_01_create_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 02 — Disable zone-scoped pool + # Step 04 — Disable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_02_disable_zone_scoped_pool(self): + def test_04_disable_zone_scoped_pool(self): """ Disable the zone-scoped pool. Verifies: - pool.state is Disabled - ONTAP: FlexVol still online; export policy unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -325,18 +597,18 @@ def test_02_disable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 03 — Enable zone-scoped pool + # Step 05 — Enable zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_03_enable_zone_scoped_pool(self): + def test_05_enable_zone_scoped_pool(self): """ Re-enable the zone-scoped pool. Verifies: - pool.state is Up - ONTAP: FlexVol still online; export policy unchanged """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") cmd = updateStoragePoolAPI.updateStoragePoolCmd() cmd.id = self.__class__.pool.id @@ -362,11 +634,11 @@ def test_03_enable_zone_scoped_pool(self): ) # ------------------------------------------------------------------ - # Step 04 — Delete zone-scoped pool + # Step 06 — Delete zone-scoped pool # ------------------------------------------------------------------ @attr(tags=["zone_pool"], required_hardware=True) - def test_04_delete_zone_scoped_pool(self): + def test_06_delete_zone_scoped_pool(self): """ Enter maintenance then delete the zone-scoped pool. Verifies: @@ -374,7 +646,7 @@ def test_04_delete_zone_scoped_pool(self): - ONTAP: FlexVol deleted - ONTAP: export policy deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_01 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_03 must pass first") pool = self.__class__.pool pool_name = pool.name @@ -417,6 +689,127 @@ def test_04_delete_zone_scoped_pool(self): "Export policy '%s' still exists after pool deletion" % ep_name ) + # ------------------------------------------------------------------ + # Step 07 — Delete a pool whose FlexVol was pre-deleted + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_07_delete_zone_pool_with_flexvol_predeleted(self): + """ + Create an empty zone pool, remove its FlexVol directly on ONTAP, then + delete the pool through CloudStack. + Verifies: + - deleteStoragePool succeeds and the pool leaves CloudStack + - ONTAP: the FlexVol stays gone + The KVM hosts are unmounted before the FlexVol is removed so the pool + never becomes a stale NFS mount. + """ + self._require_ontap_client("offline_and_delete_volume") + + pool_name = "OntapZoneNFS3NoFv_%d" % random.randint(0, 99999) + pool = self._create_isolated_zone_pool(pool_name) + ep_name = self._get_export_policy_name(pool) + try: + self.assertIsNotNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' missing right after pool creation" + % pool_name + ) + + self.assertTrue( + self._enter_maintenance_quietly(pool.id), + "Pool '%s' did not enter Maintenance before backend mutation" + % pool_name + ) + + # Unmount while the export is still reachable, then delete the + # FlexVol behind CloudStack's back. + self._cleanup_kvm_storage_pool_mounts(pool.id) + self.ontap.offline_and_delete_volume(pool_name) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' still present after the manual delete" + % pool_name + ) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool2 = None + + self.assertFalse( + self._cs_pool_exists(pool.id), + "Pool '%s' still listed in CloudStack after deletion with a " + "pre-deleted FlexVol" % pool_name + ) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' reappeared after pool deletion" % pool_name + ) + finally: + if self._force_cleanup_zone_pool(pool): + self._force_delete_flexvol(pool_name) + self._force_delete_export_policy(ep_name) + + # ------------------------------------------------------------------ + # Step 08 — Delete a pool whose export policy was pre-deleted + # ------------------------------------------------------------------ + + @attr(tags=["zone_pool"], required_hardware=True) + def test_08_delete_zone_pool_with_export_policy_predeleted(self): + """ + Create an empty zone pool, remove its NFS export policy directly on + ONTAP, then delete the pool through CloudStack. + Verifies: + - deleteStoragePool succeeds and the pool leaves CloudStack + - ONTAP: the FlexVol is deleted and the export policy stays gone + The KVM hosts are unmounted before the export policy is removed so + the pool never becomes a stale NFS mount. + """ + self._require_ontap_client("offline_and_delete_volume") + + pool_name = "OntapZoneNFS3NoEp_%d" % random.randint(0, 99999) + pool = self._create_isolated_zone_pool(pool_name) + ep_name = self._get_export_policy_name(pool) + try: + self._assert_export_policy_has_host_ips(ep_name) + self.assertTrue( + self._enter_maintenance_quietly(pool.id), + "Pool '%s' did not enter Maintenance before backend mutation" + % pool_name + ) + + # Unmount before pulling the export policy out from under the + # hosts, otherwise the mount goes stale and KVMHAMonitor reboots. + self._cleanup_kvm_storage_pool_mounts(pool.id) + self.ontap.reassign_volume_export_policy(pool.name) + self.ontap.delete_export_policy(ep_name) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' still present after the manual delete" + % ep_name + ) + + self._delete_pool(pool.id, forced=True) + self.__class__.pool2 = None + + self.assertFalse( + self._cs_pool_exists(pool.id), + "Pool '%s' still listed in CloudStack after deletion with a " + "pre-deleted export policy" % pool_name + ) + self.assertIsNone( + self.ontap.get_volume(pool_name), + "ONTAP FlexVol '%s' still exists after pool deletion" + % pool_name + ) + self.assertIsNone( + self.ontap.get_export_policy(ep_name), + "Export policy '%s' reappeared after pool deletion" % ep_name + ) + finally: + if self._force_cleanup_zone_pool(pool): + self._force_delete_flexvol(pool_name) + self._force_delete_export_policy(ep_name) + # ------------------------------------------------------------------ # Class-level teardown # ------------------------------------------------------------------ diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 4f60dbf9433f..6ed1a62f589a 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -27,6 +27,7 @@ import logging import random +import re import requests import sys import time @@ -144,7 +145,248 @@ def _delete(self, path, params=None): url = self._base + path resp = requests.delete(url, auth=self._auth, params=params, verify=False, timeout=30) - resp.raise_for_status() + self._raise_http(resp) + return self._response_data(resp) + + @staticmethod + def _response_data(resp): + """Return response JSON, including async job UUID from Location.""" + if resp.content: + try: + return resp.json() + except ValueError: + pass + location = resp.headers.get("Location", "") + marker = "/cluster/jobs/" + if marker in location: + return {"job": {"uuid": location.split(marker, 1)[1].split("?", 1)[0]}} + return None + + def _raise_http(self, resp): + if resp.ok: + return + body = "" + try: + body = resp.text + except Exception: + body = "" + raise requests.HTTPError( + "%s Client Error: %s for url: %s body: %s" + % (resp.status_code, resp.reason, resp.url, body), + response=resp, + ) + + def _patch(self, path, params=None, json_body=None, timeout=60): + url = self._base + path + resp = requests.patch( + url, auth=self._auth, params=params, json=json_body, + verify=False, timeout=timeout, + ) + self._raise_http(resp) + return self._response_data(resp) + + def _post(self, path, params=None, data=None, json_body=None, timeout=60, + headers=None, files=None): + url = self._base + path + resp = requests.post( + url, auth=self._auth, params=params, data=data, json=json_body, + headers=headers, files=files, verify=False, timeout=timeout, + ) + self._raise_http(resp) + return self._response_data(resp) + + def _wait_for_job(self, response, timeout=120): + """Wait for an asynchronous ONTAP response, if it contains a job.""" + job = (response or {}).get("job") or {} + job_uuid = job.get("uuid") + if not job_uuid: + return response + deadline = time.time() + timeout + while time.time() < deadline: + current = self._get("/cluster/jobs/%s" % job_uuid) + state = (current.get("state") or "").lower() + if state == "success": + return current + if state in ("failure", "failed", "error"): + message = current.get("message") or "unknown ONTAP job failure" + raise RuntimeError("ONTAP job %s failed: %s" % (job_uuid, message)) + time.sleep(2) + raise RuntimeError("Timed out waiting for ONTAP job %s" % job_uuid) + + def _svm_aggregates(self, svm_name): + """Return detailed aggregate records assigned to an SVM.""" + svms = self._get( + "/svm/svms", + params={"name": svm_name, "fields": "aggregates"}, + ).get("records", []) + if not svms: + raise RuntimeError("ONTAP SVM '%s' was not found" % svm_name) + aggregates = [] + for aggregate in svms[0].get("aggregates", []): + uuid = aggregate.get("uuid") + if not uuid: + continue + aggregates.append(self._get( + "/storage/aggregates/%s" % uuid, + params={"fields": "name,uuid,state,space.block_storage.available"}, + )) + return aggregates + + def max_online_aggregate_available_bytes(self, svm_name): + """Return the largest free-space value among assigned online aggregates.""" + available = [] + for aggregate in self._svm_aggregates(svm_name): + if (aggregate.get("state") or "").lower() != "online": + continue + free = (aggregate.get("space", {}) + .get("block_storage", {}).get("available")) + if free is not None: + available.append(int(float(free))) + if not available: + raise RuntimeError( + "SVM '%s' has no online aggregate with space data" % svm_name + ) + return max(available) + + def create_flexvol(self, svm_name, volume_name, size_bytes, nas_path=True): + """Create a thin FlexVol directly on a suitable SVM aggregate.""" + suitable = [] + for aggregate in self._svm_aggregates(svm_name): + free = (aggregate.get("space", {}) + .get("block_storage", {}).get("available")) + if ((aggregate.get("state") or "").lower() == "online" + and free is not None and int(float(free)) > int(size_bytes)): + suitable.append((int(float(free)), aggregate)) + if not suitable: + raise RuntimeError( + "No ONTAP aggregate can hold FlexVol '%s'" % volume_name + ) + aggregate = max(suitable, key=lambda item: item[0])[1] + request = { + "name": volume_name, + "svm": {"name": svm_name}, + "size": int(size_bytes), + "aggregates": [{"name": aggregate.get("name")}], + "guarantee": {"type": "none"}, + } + if nas_path: + request["nas"] = {"path": "/" + volume_name} + response = self._post( + "/storage/volumes", + params={"return_timeout": 15}, + json_body=request, + ) + self._wait_for_job(response) + deadline = time.time() + 120 + while time.time() < deadline: + volume = self.get_volume(volume_name) + if volume is not None: + return volume + time.sleep(2) + raise RuntimeError( + "FlexVol '%s' was not visible after creation" % volume_name + ) + + def offline_and_delete_volume(self, name): + """Offline and delete a FlexVol directly; no-op when already absent.""" + volume = self.get_volume(name) + if not volume: + return + uuid = volume.get("uuid") + if not uuid: + raise RuntimeError("FlexVol '%s' has no UUID" % name) + if (volume.get("nas") or {}).get("path"): + response = self._patch( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + json_body={"nas": {"path": ""}}, + ) + self._wait_for_job(response) + if (volume.get("state") or "").lower() != "offline": + response = self._patch( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + json_body={"state": "offline"}, + ) + self._wait_for_job(response) + response = self._delete( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + ) + self._wait_for_job(response) + deadline = time.time() + 120 + while time.time() < deadline: + if self.get_volume(name) is None: + return + time.sleep(2) + raise RuntimeError("FlexVol '%s' still exists after deletion" % name) + + def reassign_volume_export_policy(self, volume_name, policy_name="default"): + """Assign a FlexVol to another export policy before deleting its policy.""" + volume = self.get_volume(volume_name) + if not volume: + raise RuntimeError("FlexVol '%s' was not found" % volume_name) + uuid = volume.get("uuid") + if not uuid: + raise RuntimeError("FlexVol '%s' has no UUID" % volume_name) + response = self._patch( + "/storage/volumes/%s" % uuid, + params={"return_timeout": 15}, + json_body={"nas": {"export_policy": {"name": policy_name}}}, + ) + self._wait_for_job(response) + + def create_igroup(self, svm_name, igroup_name, initiator_iqn): + """Create an ONTAP igroup holding a single initiator.""" + self._post( + "/protocols/san/igroups", + json_body={ + "svm": {"name": svm_name}, + "name": igroup_name, + "os_type": "linux", + "protocol": "iscsi", + "initiators": [{"name": initiator_iqn}], + }, + ) + igroup = self.get_igroup(svm_name, igroup_name) + if igroup is None: + raise RuntimeError( + "ONTAP igroup '%s' absent right after creation" % igroup_name + ) + return igroup + + def delete_igroup(self, svm_name, igroup_name): + """Delete an ONTAP igroup by name; no-op when already absent.""" + igroup = self.get_igroup(svm_name, igroup_name) + if not igroup: + return + uuid = igroup.get("uuid") + if not uuid: + raise RuntimeError("ONTAP igroup '%s' has no UUID" % igroup_name) + self._delete("/protocols/san/igroups/%s" % uuid) + + def create_lun_map(self, svm_name, lun_path, igroup_name): + """Map an existing LUN to an existing igroup.""" + response = self._post( + "/protocols/san/lun-maps", + params={"return_timeout": 15}, + json_body={ + "svm": {"name": svm_name}, + "lun": {"name": lun_path}, + "igroup": {"name": igroup_name}, + }, + ) + self._wait_for_job(response) + + def delete_lun_map(self, lun_map): + """Delete one LUN map returned by list_lun_maps_for_volume.""" + lun_uuid = lun_map.get("lun", {}).get("uuid") + igroup_uuid = lun_map.get("igroup", {}).get("uuid") + if not lun_uuid or not igroup_uuid: + raise RuntimeError("ONTAP LUN map is missing LUN or igroup UUID") + self._delete( + "/protocols/san/lun-maps/%s/%s" % (lun_uuid, igroup_uuid) + ) def delete_volume(self, name): """Delete the ONTAP FlexVol with the given name. No-op if not found.""" @@ -175,7 +417,7 @@ def get_volume(self, name): uuid = records[0].get("uuid") if uuid: return self._get("/storage/volumes/%s" % uuid, - params={"fields": "name,uuid,state,space"}) + params={"fields": "name,uuid,state,space,nas.path,nas.export_policy"}) return records[0] # -- NFS helpers --------------------------------------------------------- @@ -237,7 +479,7 @@ def list_lun_maps_for_volume(self, svm_name, vol_name): prefix = "/vol/%s/" % vol_name data = self._get("/protocols/san/lun-maps", params={"svm.name": svm_name, - "fields": "lun.name,igroup.name"}) + "fields": "lun.name,lun.uuid,igroup.name,igroup.uuid"}) return [r for r in data.get("records", []) if r.get("lun", {}).get("name", "").startswith(prefix)] @@ -290,6 +532,8 @@ class OntapTestBase(cloudstackTestCase): svm_name = None cluster_hosts = None kvm_hosts_ssh_creds = [] # [{'host': '10.x.x.x', 'user': 'root', 'password': '...'}] + _host_iqn_cache = {} + igroup_baseline = {} ontap = None testdata = None zone = None @@ -601,6 +845,130 @@ def _poll_pool_state(self, pool_id, target_state, timeout=120, interval=5): % (pool_id, target_state, timeout, current_state) ) + + @classmethod + def host_iqn(cls, host): + """The iSCSI initiator IQN for a cluster host, or None.""" + host_ip = getattr(host, "ipaddress", None) + if not host_ip: + return None + if host_ip in cls._host_iqn_cache: + return cls._host_iqn_cache[host_ip] + iqn = None + creds = next( + (c for c in cls.kvm_hosts_ssh_creds if c["host"] == host_ip), None + ) + if creds is not None: + try: + ssh = SshClient(host_ip, 22, creds["user"], creds["password"], + retries=3, delay=3, timeout=15.0) + out = ssh.execute( + "awk -F= '/^InitiatorName=/{print $2}' " + "/etc/iscsi/initiatorname.iscsi 2>/dev/null" + ) + for line in out or []: + line = line.strip() + if line.startswith("iqn."): + iqn = line + break + except Exception as ex: + logger.warning("host_iqn: SSH to %s failed: %s", host_ip, ex) + cls._host_iqn_cache[host_ip] = iqn + return iqn + + @classmethod + def _igroup_name(cls, host_uuid): + """Return the igroup name used by OntapStorageUtils.""" + sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", str(host_uuid)) + return ("cs_%s_%s" % (sanitized, cls.svm_name))[:96] + + @classmethod + def _iscsi_host_specs(cls): + """Return (igroup name, initiator IQN) for iSCSI cluster hosts.""" + specs = [] + for host in cls.cluster_hosts or []: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + or cls.host_iqn(host) + ) + host_uuid = getattr(host, "id", None) + if not iqn or not iqn.startswith("iqn.") or not host_uuid: + continue + specs.append((cls._igroup_name(host_uuid), iqn)) + return specs + + @staticmethod + def _igroup_initiators(igroup): + if igroup is None: + return None + return tuple(sorted( + i.get("name", "") for i in igroup.get("initiators", []) + )) + + @classmethod + def _capture_igroup_baseline(cls): + """Snapshot shared host igroups before an iSCSI suite creates a pool.""" + cls.igroup_baseline = {} + for igroup_name, _ in cls._iscsi_host_specs(): + igroup = cls.ontap.get_igroup(cls.svm_name, igroup_name) + cls.igroup_baseline[igroup_name] = cls._igroup_initiators(igroup) + logger.info( + "Captured iSCSI igroup baseline for SVM '%s': %s", + cls.svm_name, cls.igroup_baseline, + ) + + def _assert_igroup_baseline_unchanged(self, context): + """Assert an operation did not change pre-existing shared igroups.""" + for igroup_name, expected_initiators in self.igroup_baseline.items(): + igroup = self.ontap.get_igroup(self.svm_name, igroup_name) + actual_initiators = self._igroup_initiators(igroup) + self.assertEqual( + actual_initiators, expected_initiators, + "ONTAP igroup '%s' changed %s: expected initiators %s, got %s" + % (igroup_name, context, expected_initiators, + actual_initiators), + ) + + def _assert_no_lun_maps_for_volume(self, volume_name, context): + """Assert no LUN in a test FlexVol remains mapped to any igroup.""" + maps = self.ontap.list_lun_maps_for_volume( + self.svm_name, volume_name + ) + self.assertFalse( + maps, + "LUN maps for FlexVol '%s' remain %s: %s" + % (volume_name, context, maps), + ) + + def _get_cs_volume(self, vol_id): + """Return the CloudStack volume object, or None if it is gone.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return vols[0] if vols else None + + def _other_ontap_pools_on_svm(self, current_pool_id): + """Return other CloudStack ONTAP pools that use this suite's SVM.""" + try: + pools = list_storage_pools(self.apiClient) or [] + except Exception: + return ["unable to list storage pools"] + others = [] + for pool in pools: + if str(getattr(pool, "id", "")) == str(current_pool_id): + continue + details = _parse_pool_details(pool) + if details.get("svmName") == getattr(self, "svm_name", None): + others.append(pool) + continue + provider = (getattr(pool, "provider", "") or "").lower() + if not details and "netapp" in provider and "ontap" in provider: + others.append(pool) + return others + def _create_volume(self, pool_id): """Create a data volume on the given pool; uses _vol_name_prefix.""" cmd = createVolumeAPI.createVolumeCmd()