diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..5008ddfc Binary files /dev/null and b/.DS_Store differ diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 094f0e79..50eda4dc 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -2158,14 +2158,20 @@ def switch_group_guideline_check(fabric_nodes, **kwargs): @check_wrapper(check_title="Switch Node /bootflash usage") -def switch_bootflash_usage_check(tversion, **kwargs): +def switch_bootflash_usage_check(sw_cversion, tversion, **kwargs): result = FAIL_UF msg = '' - headers = ["Pod-ID", "Node-ID", "Utilization"] + headers = ["Pod-ID", "Node-ID", "Avail (MB)", "Required (MB)"] data = [] - recommended_action = "Over 50% usage! Contact Cisco TAC for Support" + recommended_action = "Insufficient free space to download and extract the target image! Contact Cisco TAC for Support" doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-node-bootflash-usage" + if not sw_cversion: + return Result(result=MANUAL, msg="Current switch version not found. Check switch health.", doc_url=doc_url) + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING, doc_url=doc_url) + partitions_api = 'eqptcapacityFSPartition.json' partitions_api += '?query-target-filter=eq(eqptcapacityFSPartition.path,"/bootflash")' @@ -2175,7 +2181,7 @@ def switch_bootflash_usage_check(tversion, **kwargs): partitions = icurl('class', partitions_api) if not partitions: - return Result(result=MANUAL, msg='bootflash objects not found. Check switch health.', doc_url=doc_url) + return Result(result=MANUAL, msg='/bootflash directory not found. Check switch health.', doc_url=doc_url) predownloaded_nodes = [] try: @@ -2183,26 +2189,92 @@ def switch_bootflash_usage_check(tversion, **kwargs): except OldVerPropNotFound: # Older versions don't have 'dnldStatus' param download_sts = [] - + for maintUpgJob in download_sts: dn = re.search(node_regex, maintUpgJob['maintUpgJob']['attributes']['dn']) - node = dn.group("node") - predownloaded_nodes.append(node) + if dn: + predownloaded_nodes.append(dn.group("node")) + + # Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit + # isos (`-cs_64` suffix for 64-bit). Below that, only a single 32-bit iso exists. + boundary_version = "6.0(2a)" + switch_target_version = "aci-n9000-dk9.1{}.bin".format(tversion.dot_version) + switch_target_version_64 = "aci-n9000-dk9.1{}-cs_64.bin".format(tversion.dot_version) + + firmware_api = 'firmwareFirmware.json?query-target-filter=eq(firmwareFirmware.type,"switch")' + firmwares = icurl('class', firmware_api) + fw_sizes = {} + for firmware in firmwares: + fw_attr = firmware['firmwareFirmware']['attributes'] + fw_sizes[fw_attr['isoname']] = int(fw_attr['size']) + + target_size_32 = fw_sizes.get(switch_target_version) + target_size_64 = fw_sizes.get(switch_target_version_64) + + # sw_cversion (lowest switch version), not the APIC cversion, drives the boundary + # decision: the upgrade guide has APICs reach 6.0(2a)+ before the switches, so the + # switches can still be pre-boundary while the APIC cluster is already post-boundary. + target_is_legacy = tversion.older_than(boundary_version) + current_is_legacy = sw_cversion.older_than(boundary_version) + + if target_is_legacy: + # Only the 32-bit image is ever used for a pre-6.0(2a) target. + if target_size_32 is None: + msg = 'Target switch image ({}) not found in Firmware Repository.'.format(switch_target_version) + return Result(result=MANUAL, msg=msg, doc_url=doc_url) + required_space = 2 * target_size_32 + downloaded_required_space = target_size_32 + + else: + # The larger image is used as a conservative estimate, so both sizes must be + # known; a missing one can't be assumed to be the smaller (or zero-byte) one. + if target_size_32 is None and target_size_64 is None: + msg = 'Target switch images ({}, {}) not found in Firmware Repository.'.format(switch_target_version, switch_target_version_64) + return Result(result=MANUAL, msg=msg, doc_url=doc_url) + elif target_size_32 is None: + msg = '32-bit target switch image ({}) not found in Firmware Repository.'.format(switch_target_version) + return Result(result=MANUAL, msg=msg, doc_url=doc_url) + elif target_size_64 is None: + msg = '64-bit target switch image ({}) not found in Firmware Repository.'.format(switch_target_version_64) + return Result(result=MANUAL, msg=msg, doc_url=doc_url) + + downloaded_required_space = max(target_size_32, target_size_64) + + if current_is_legacy: + # Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a + # 32-bit image, so its size is freed once removed during the upgrade. + switch_current_version = "aci-n9000-dk9.1{}.bin".format(sw_cversion.dot_version) + current_size = fw_sizes.get(switch_current_version) + if current_size is None: + msg = 'Current switch image ({}) not found in Firmware Repository.'.format(switch_current_version) + return Result(result=MANUAL, msg=msg, doc_url=doc_url) + if target_size_32 > current_size: + required_space = 2 * (target_size_32 + target_size_64 - current_size) + else: + required_space = 2 * max(target_size_32, target_size_64) + else: + required_space = 2 * max(target_size_32, target_size_64) + + required_space_kb = required_space / 1024.0 # eqptcapacityFSPartition avail/used are in KB + downloaded_required_space_kb = downloaded_required_space / 1024.0 for eqptcapacityFSPartition in partitions: dn = re.search(node_regex, eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['dn']) pod = dn.group("pod") node = dn.group("node") avail = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['avail']) - used = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['used']) - usage = (used / (avail + used)) * 100 - if (usage >= 50) and (node not in predownloaded_nodes): - data.append([pod, node, usage]) + # dnldStatus == downloaded only proves the image was delivered, not that + # extraction (which still consumes bootflash) has completed, so a downloaded + # node is still checked, just against the smaller extraction-only requirement. + node_required_space_kb = downloaded_required_space_kb if node in predownloaded_nodes else required_space_kb + + if avail < node_required_space_kb: + data.append([pod, node, round(avail / 1024.0, 2), round(node_required_space_kb / 1024.0, 2)]) if not data: result = PASS - msg = 'All below 50% or pre-downloaded' + msg = 'All nodes have sufficient bootflash space' return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 717c1757..25463e1d 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -651,67 +651,16 @@ To prevent this, check the `/bootflash` prior to an upgrade and take the necessa The pre-upgrade validation built into Cisco APIC upgrade workflow monitors the fault F1821, which can capture the high utilization of any partition. When this fault is present, we recommend that you resolve it prior to the upgrade even if the fault is not for bootflash. -The ACI Pre-Upgrade Validation script (this script) focuses on the utilization of bootflash on each switch specifically to see if there are any issues with bootflash where the usage is more than 50%, which might trigger the internal cleanup script. - -!!! example "Example of a query used by this script" - The script is calculating the bootflash usage using `avail` and `used` in the object `eqptcapacityFSPartition` for each switch. - ``` - f2-apic1# moquery -c eqptcapacityFSPartition -f 'eqptcapacity.FSPartition.path=="/bootflash"' - Total Objects shown: 6 - - # eqptcapacity.FSPartition - name : bootflash - avail : 7214920 - childAction : - dn : topology/pod-1/node-101/sys/eqptcapacity/fspartition-bootflash - memAlert : normal - modTs : never - monPolDn : uni/fabric/monfab-default - path : /bootflash - rn : fspartition-bootflash - status : - used : 4320184 - --- omit --- - ``` +The ACI Pre-Upgrade Validation script (this script) dynamically calculates the actual bootflash space required for the target upgrade, rather than relying on a fixed usage threshold, and compares it against each switch's available `/bootflash` space: -!!! tip - Alternatively you can log into a leaf switch CLI, and check `/bootflash` usage `df -h` - ``` - leaf1# df -h - Filesystem Size Used Avail Use% Mounted on - rootfs 2.5G 935M 1.6G 38% /bin - /dev/sda4 12G 5.7G 4.9G 54% /bootflash - /dev/sda2 4.7G 9.6M 4.4G 1% /recovery - /dev/mapper/map-sda9 11G 5.7G 4.2G 58% /isan/lib - none 3.0G 602M 2.5G 20% /dev/shm - none 50M 3.4M 47M 7% /etc - /dev/sda6 56M 1.3M 50M 3% /mnt/cfg/1 - /dev/sda5 56M 1.3M 50M 3% /mnt/cfg/0 - /dev/sda8 15G 140M 15G 1% /mnt/ifc/log - /dev/sda3 115M 52M 54M 50% /mnt/pss - none 1.5G 2.3M 1.5G 1% /tmp - none 50M 240K 50M 1% /var/log - /dev/sda7 12G 1.4G 9.3G 13% /logflash - none 350M 54M 297M 16% /var/log/dme/log/dme_logs - none 512M 24M 489M 5% /var/sysmgr/mem_logs - none 40M 4.0K 40M 1% /var/sysmgr/startup-cfg - none 500M 0 500M 0% /volatile - ``` +* Required space is based on the target image size(s) needed to download and extract on top of the existing content. Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit isos, so both current and target version determine whether one or both images apply: + * Both versions pre-6.0(2a): only the single 32-bit image size is used. + * Both versions post-6.0(2a): the larger of the 32-bit/64-bit target images is used. + * Crossing the 6.0(2a) boundary: target image downloaded while the current is removed, freeing its space for successful extraction. -!!! note - If you suspect that the auto cleanup removed some files within `/bootflash`, you can review a log to validate this: +* Nodes that already downloaded the exact target version (`maintUpgJob.dnldStatus == downloaded` and `desiredVersion` matching target) are still evaluated, just against a smaller, extraction-only requirement (the larger of the 32-bit/64-bit target image sizes, without doubling for the download), since extraction and later upgrade stages can still require additional bootflash space. - ``` - leaf1# egrep "higher|removed" /mnt/pss/core_control.log - [2020-07-22 16:52:08.928318] Bootflash Usage is higher than 50%!! - [2020-07-22 16:52:08.931990] File: MemoryLog.65%_usage removed !! - [2020-07-22 16:52:08.943914] File: mem_log.txt.old.gz removed !! - [2020-07-22 16:52:08.955376] File: libmon.logs removed !! - [2020-07-22 16:52:08.966686] File: urib_api_log.txt removed !! - [2020-07-22 16:52:08.977832] File: disk_log.txt removed !! - [2020-07-22 16:52:08.989102] File: mem_log.txt removed !! - [2020-07-22 16:52:09.414572] File: aci-n9000-dk9.13.2.1m.bin removed !! - ``` +* If a required firmware image isn't found in the Firmware Repository, the check reports a manual review. ### APIC SSD Health diff --git a/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_602.json b/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_602.json new file mode 100644 index 00000000..4c12c879 --- /dev/null +++ b/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_602.json @@ -0,0 +1,4 @@ +[ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h.bin", "size": "2000000000"}}}, + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h-cs_64.bin", "size": "3000000000"}}} +] diff --git a/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_insufficient.json b/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_insufficient.json new file mode 100644 index 00000000..0200064e --- /dev/null +++ b/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_insufficient.json @@ -0,0 +1,35 @@ +[ + { + "firmwareFirmware": { + "attributes": { + "dn": "fwrepo/fw-aci-n9000-system.16.1.5e.bin", + "fullVersion": "n9000-16.1(5e)", + "isoname": "aci-n9000-dk9.16.1.5e.bin", + "name": "aci-n9000-system.16.1.5e.bin", + "size": "3000000000" + } + } + }, + { + "firmwareFirmware": { + "attributes": { + "dn": "fwrepo/fw-aci-n9000-system.16.1.5e-cs_64.bin", + "fullVersion": "n9000-16.1(5e)", + "isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin", + "name": "aci-n9000-system.16.1.5e-cs_64.bin", + "size": "3000000000" + } + } + }, + { + "firmwareFirmware": { + "attributes": { + "dn": "fwrepo/fw-aci-n9000-system.15.2.8h.bin", + "fullVersion": "n9000-15.2(8h)", + "isoname": "aci-n9000-dk9.15.2.8h.bin", + "name": "aci-n9000-system.15.2.8h.bin", + "size": "2000000000" + } + } + } +] diff --git a/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_sufficient.json b/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_sufficient.json new file mode 100644 index 00000000..652f5768 --- /dev/null +++ b/tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_sufficient.json @@ -0,0 +1,35 @@ +[ + { + "firmwareFirmware": { + "attributes": { + "dn": "fwrepo/fw-aci-n9000-system.16.1.5e.bin", + "fullVersion": "n9000-16.1(5e)", + "isoname": "aci-n9000-dk9.16.1.5e.bin", + "name": "aci-n9000-system.16.1.5e.bin", + "size": "1500000000" + } + } + }, + { + "firmwareFirmware": { + "attributes": { + "dn": "fwrepo/fw-aci-n9000-system.16.1.5e-cs_64.bin", + "fullVersion": "n9000-16.1(5e)", + "isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin", + "name": "aci-n9000-system.16.1.5e-cs_64.bin", + "size": "1500000000" + } + } + }, + { + "firmwareFirmware": { + "attributes": { + "dn": "fwrepo/fw-aci-n9000-system.15.2.8h.bin", + "fullVersion": "n9000-15.2(8h)", + "isoname": "aci-n9000-dk9.15.2.8h.bin", + "name": "aci-n9000-system.15.2.8h.bin", + "size": "2000000000" + } + } + } +] diff --git a/tests/checks/switch_bootflash_usage_check/maintUpgJob_all_downloaded.json b/tests/checks/switch_bootflash_usage_check/maintUpgJob_all_downloaded.json new file mode 100644 index 00000000..207a2544 --- /dev/null +++ b/tests/checks/switch_bootflash_usage_check/maintUpgJob_all_downloaded.json @@ -0,0 +1,13 @@ +[ + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-205/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-206/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2003/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2010/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-101/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}} +] diff --git a/tests/checks/switch_bootflash_usage_check/maintUpgJob_all_of_failing.json b/tests/checks/switch_bootflash_usage_check/maintUpgJob_all_of_failing.json new file mode 100644 index 00000000..f0ca5167 --- /dev/null +++ b/tests/checks/switch_bootflash_usage_check/maintUpgJob_all_of_failing.json @@ -0,0 +1,10 @@ +[ + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-205/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-206/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-101/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}} +] diff --git a/tests/checks/switch_bootflash_usage_check/maintUpgJob_not_downloaded.json b/tests/checks/switch_bootflash_usage_check/maintUpgJob_not_downloaded.json deleted file mode 100644 index 0637a088..00000000 --- a/tests/checks/switch_bootflash_usage_check/maintUpgJob_not_downloaded.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/tests/checks/switch_bootflash_usage_check/maintUpgJob_old_ver_no_prop.json b/tests/checks/switch_bootflash_usage_check/maintUpgJob_old_ver_no_prop.json index 9f74f58d..d9f93d7e 100644 --- a/tests/checks/switch_bootflash_usage_check/maintUpgJob_old_ver_no_prop.json +++ b/tests/checks/switch_bootflash_usage_check/maintUpgJob_old_ver_no_prop.json @@ -1,10 +1,10 @@ [ { - "error": { - "attributes": { - "code": "121", - "text": "Prop 'dnldStatus' not found in class 'maintUpgJob' property table" - } + "error": { + "attributes": { + "code": "400", + "text": "Prop 'dnldStatus' not found in class 'maintUpgJob' property table" } + } } -] \ No newline at end of file +] diff --git a/tests/checks/switch_bootflash_usage_check/maintUpgJob_partial_missing_empty.json b/tests/checks/switch_bootflash_usage_check/maintUpgJob_partial_missing_empty.json new file mode 100644 index 00000000..552d314d --- /dev/null +++ b/tests/checks/switch_bootflash_usage_check/maintUpgJob_partial_missing_empty.json @@ -0,0 +1,5 @@ +[ + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, + {"maintUpgJob": {"attributes": {"dn": "", "dnldStatus": "downloaded", "dnldPercent": "100"}}} +] diff --git a/tests/checks/switch_bootflash_usage_check/maintUpgJob_pre_downloaded.json b/tests/checks/switch_bootflash_usage_check/maintUpgJob_pre_downloaded.json deleted file mode 100644 index efc36e70..00000000 --- a/tests/checks/switch_bootflash_usage_check/maintUpgJob_pre_downloaded.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "maintUpgJob": { - "attributes": { - "desiredVersion": "n9000-16.0(2h)", - "dn": "topology/pod-1/node-101/sys/fwstatuscont/upgjob", - "dnldPercent": "100", - "dnldStatus": "downloaded", - "startDate": "2023-11-16T10:15:22.894-08:00", - "status": "", - "upgradeStatus": "scheduled", - "upgradeStatusStr": "Scheduled" - } - } - } -] \ No newline at end of file diff --git a/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py b/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py index 18e9c1c3..9d025874 100644 --- a/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py +++ b/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py @@ -15,48 +15,371 @@ partitions = "eqptcapacityFSPartition.json" partitions += '?query-target-filter=eq(eqptcapacityFSPartition.path,"/bootflash")' -download_sts = "maintUpgJob.json" -download_sts += '?query-target-filter=and(eq(maintUpgJob.dnldStatus,"downloaded")' -download_sts += ',eq(maintUpgJob.desiredVersion,"n9000-16.0(2h)"))' +firmware = 'firmwareFirmware.json?query-target-filter=eq(firmwareFirmware.type,"switch")' + +download_sts_602 = 'maintUpgJob.json' +download_sts_602 += '?query-target-filter=and(eq(maintUpgJob.dnldStatus,"downloaded")' +download_sts_602 += ',eq(maintUpgJob.desiredVersion,"n9000-16.0(2h)"))' + +download_sts_528 = 'maintUpgJob.json' +download_sts_528 += '?query-target-filter=and(eq(maintUpgJob.dnldStatus,"downloaded")' +download_sts_528 += ',eq(maintUpgJob.desiredVersion,"n9000-15.2(8h)"))' + +download_sts_615 = 'maintUpgJob.json' +download_sts_615 += '?query-target-filter=and(eq(maintUpgJob.dnldStatus,"downloaded")' +download_sts_615 += ',eq(maintUpgJob.desiredVersion,"n9000-16.1(5e)"))' + +# No pre-downloaded nodes unless a test overrides this key. +no_predownload = [] + +# Older versions don't have `dnldStatus`/`dnldPercent` props on `maintUpgJob`. +old_ver_no_prop = read_data(dir, "maintUpgJob_old_ver_no_prop.json") + +# Firmware images for target 6.0(2h): both current/target are >= 6.0(2a) so both +# 32/64-bit isos are considered. The 64-bit image is the larger of the two. +# Of all nodes in eqptcapacityFSPartition.json, only node-101 (avail 5347648 KB) +# falls below the resulting required space (~5859375 KB) and thus fails. +firmware_dual_602 = read_data(dir, "firmwareFirmware_dual_602.json") + +# Only the 64-bit target image (6.0(2h)) is missing from the Firmware Repository. +firmware_602_missing_64 = [ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h.bin", "size": "2000000000"}}}, +] + +# Only the 32-bit target image (6.0(2h)) is missing from the Firmware Repository. +firmware_602_missing_32 = [ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h-cs_64.bin", "size": "3000000000"}}}, +] + +# Crossing 6.0(2a): current (5.2(8h)) image plus only the 32-bit target (6.1(5e)); the +# 64-bit target image is missing from the Firmware Repository. +firmware_crossing_missing_64 = [ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.15.2.8h.bin", "size": "2000000000"}}}, + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.1.5e.bin", "size": "3000000000"}}}, +] + +# Crossing 6.0(2a): current (5.2(8h)) image plus only the 64-bit target (6.1(5e)); the +# 32-bit target image is missing from the Firmware Repository. +firmware_crossing_missing_32 = [ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.15.2.8h.bin", "size": "2000000000"}}}, + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin", "size": "3000000000"}}}, +] + +# node-101 has fully downloaded/extracted the target image already. +maintUpgJob_node_101_downloaded = [ + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-101/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, +] + +# node-999 (not present in eqptcapacityFSPartition.json) has pre-downloaded the image. +maintUpgJob_node_999_downloaded = [ + {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-999/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, +] + +# Every node in eqptcapacityFSPartition.json has fully pre-downloaded the target image. +maintUpgJob_all_downloaded = read_data(dir, "maintUpgJob_all_downloaded.json") + +# `dn` doesn't match `node_regex` (unparseable): skipped gracefully, not added to the map. +maintUpgJob_malformed_dn = [ + {"maintUpgJob": {"attributes": {"dn": "uni/some/unexpected/format", "dnldStatus": "downloaded", "dnldPercent": "100"}}}, +] + +# Only node-101 (one of several failing nodes with the insufficient-space fixture) is pre-downloaded. +maintUpgJob_partial_of_failing = maintUpgJob_node_101_downloaded + +# All 8 nodes that fail with the insufficient-space fixture are pre-downloaded; the +# remaining 3 nodes (2002, 2003, 2010) already have enough free space on their own. +maintUpgJob_all_of_failing = read_data(dir, "maintUpgJob_all_of_failing.json") + +# Mixed real-world response: node-102 and node-103 are present and downloaded, the +# other failing nodes (205, 206, 1002, 1001, 2001, 101) are simply missing from the +# response, and one entry has an empty-string `dn` (unparseable, skipped gracefully). +maintUpgJob_partial_missing_empty = read_data(dir, "maintUpgJob_partial_missing_empty.json") + +# APIC (cversion) reaches 6.0(2a)+ before the switches per the documented upgrade +# sequence, so sw_cversion (5.2(8h), pre-boundary) must still drive the crossing decision +# even though the APIC cluster (cversion 6.0(3a)) is already post-boundary. cversion is +# passed here (as it would be via query_common_data() in production) solely to prove the +# check ignores it: switch_bootflash_usage_check() only declares sw_cversion/tversion, so +# cversion lands in **kwargs and has no effect on the result. Sizes are chosen so +# target_size_32 (3 GiB) exceeds current_size (2 GiB), forcing the full crossing formula: +# 2 * (3 GiB + 3 GiB - 2 GiB) = 8 GiB required. +apic_post_boundary_switch_pre_boundary_case = [ + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: no_predownload, + firmware: [ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.15.2.8h.bin", "size": str(2 * 1024 ** 3)}}}, + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.1.5e.bin", "size": str(3 * 1024 ** 3)}}}, + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin", "size": str(3 * 1024 ** 3)}}}, + ], + }, +] + + +@pytest.mark.parametrize("icurl_outputs", apic_post_boundary_switch_pre_boundary_case) +def test_apic_post_boundary_switch_pre_boundary_uses_crossing_formula(run_check, mock_icurl): + result = run_check( + cversion=script.AciVersion("6.0(3a)"), + sw_cversion=script.AciVersion("5.2(8h)"), + tversion=script.AciVersion("6.1(5e)"), + ) + assert result.result == script.FAIL_UF + assert result.data + required_mb = str(8 * 1024.0) # "8192.0" MB == 8 GiB + assert all(row[3] == required_mb for row in result.data) + + +#`dnldStatus == "downloaded"` only proves the image was delivered, not +# that extraction (which still consumes bootflash) succeeded, so node-101 having +# pre-downloaded the exact target must still fail when its remaining space (1 KB) can't +# fit the extraction-only requirement (max(target_size_32, target_size_64) == 3 GB). +exact_target_downloaded_insufficient_case = [ + { + partitions: [ + {"eqptcapacityFSPartition": {"attributes": {"dn": "topology/pod-1/node-101/sys/eqptcapacity/fspartition-bootflash", "avail": "1", "used": "999999999"}}}, + ], + download_sts_602: maintUpgJob_node_101_downloaded, + firmware: firmware_dual_602, + }, +] + + +@pytest.mark.parametrize("icurl_outputs", exact_target_downloaded_insufficient_case) +def test_exact_target_downloaded_still_fails_on_insufficient_extraction_space(run_check, mock_icurl): + result = run_check( + sw_cversion=script.AciVersion("6.0(3a)"), + tversion=script.AciVersion("6.0(2h)"), + ) + assert result.result == script.FAIL_UF + assert result.data == [["1", "101", "0.0", "2861.02"]] @pytest.mark.parametrize( - "icurl_outputs, tversion, expected_result", + "icurl_outputs, sw_cversion, tversion, expected_result", [ + # No tversion provided. + ( + {}, + None, + None, + script.MANUAL, + ), + # No sw_cversion (lowest switch version) found. + ( + {}, + None, + "6.0(2h)", + script.MANUAL, + ), + # /bootflash partition objects not found at all. Returns before maintUpgJob is queried. ( { partitions: [], - download_sts: [], }, + "6.0(3a)", "6.0(2h)", script.MANUAL, ), + # Baseline failure: node-101 lacks sufficient space and is not pre-downloaded. ( { partitions: read_data(dir, "eqptcapacityFSPartition.json"), - download_sts: read_data(dir, "maintUpgJob_not_downloaded.json"), + download_sts_602: no_predownload, + firmware: firmware_dual_602, }, + "6.0(3a)", "6.0(2h)", script.FAIL_UF, ), + # Both current and target are pre-6.0(2a): only the single 32-bit iso matters. ( { partitions: read_data(dir, "eqptcapacityFSPartition.json"), - download_sts: read_data(dir, "maintUpgJob_pre_downloaded.json"), + download_sts_528: no_predownload, + firmware: [ + {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.15.2.8h.bin", "size": "1000000000"}}}, + ], }, + "5.2(8h)", + "5.2(8h)", + script.PASS, + ), + # Crossing 6.0(2a): current 32-bit image size is deducted from the two target isos. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: no_predownload, + firmware: read_data(dir, "firmwareFirmware_dual_image_insufficient.json"), + }, + "5.2(8h)", + "6.1(5e)", + script.FAIL_UF, + ), + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: no_predownload, + firmware: read_data(dir, "firmwareFirmware_dual_image_sufficient.json"), + }, + "5.2(8h)", + "6.1(5e)", + script.PASS, + ), + # Target image not yet uploaded to the Firmware Repository. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: no_predownload, + firmware: [], + }, + "6.0(3a)", + "6.0(2h)", + script.MANUAL, + ), + # Post-6.0(2a): only the 64-bit target image is missing from the Firmware Repository. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: no_predownload, + firmware: firmware_602_missing_64, + }, + "6.0(3a)", + "6.0(2h)", + script.MANUAL, + ), + # Post-6.0(2a): only the 32-bit target image is missing from the Firmware Repository. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: no_predownload, + firmware: firmware_602_missing_32, + }, + "6.0(3a)", + "6.0(2h)", + script.MANUAL, + ), + # Crossing 6.0(2a): only the 64-bit target image is missing from the Firmware Repository. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: no_predownload, + firmware: firmware_crossing_missing_64, + }, + "5.2(8h)", + "6.1(5e)", + script.MANUAL, + ), + # Crossing 6.0(2a): only the 32-bit target image is missing from the Firmware Repository. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: no_predownload, + firmware: firmware_crossing_missing_32, + }, + "5.2(8h)", + "6.1(5e)", + script.MANUAL, + ), + # node-101 (the only node that would otherwise fail) already fully downloaded + # the target image, so it's excluded from the check and the result is PASS. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: maintUpgJob_node_101_downloaded, + firmware: firmware_dual_602, + }, + "6.0(3a)", + "6.0(2h)", + script.PASS, + ), + # A pre-downloaded node unrelated to the failing node doesn't change the outcome. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: maintUpgJob_node_999_downloaded, + firmware: firmware_dual_602, + }, + "6.0(3a)", + "6.0(2h)", + script.FAIL_UF, + ), + # Every node has pre-downloaded the image, so all are skipped and the result is PASS. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: maintUpgJob_all_downloaded, + firmware: firmware_dual_602, + }, + "6.0(3a)", "6.0(2h)", script.PASS, ), + # Older versions don't have `dnldStatus`/`dnldPercent` on `maintUpgJob`: `OldVerPropNotFound` + # is caught and treated as no pre-downloaded nodes found (node-101 still fails). ( { partitions: read_data(dir, "eqptcapacityFSPartition.json"), - download_sts: read_data(dir, "maintUpgJob_old_ver_no_prop.json"), + download_sts_602: old_ver_no_prop, + firmware: firmware_dual_602, }, + "6.0(3a)", "6.0(2h)", script.FAIL_UF, ), + # A `maintUpgJob` entry with an unparseable `dn` is skipped gracefully (no crash, + # no false skip): node-101 is not excluded and the check still fails. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_602: maintUpgJob_malformed_dn, + firmware: firmware_dual_602, + }, + "6.0(3a)", + "6.0(2h)", + script.FAIL_UF, + ), + # Multiple nodes fail with the insufficient-space fixture; pre-downloading only + # one of them still leaves the rest failing. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: maintUpgJob_partial_of_failing, + firmware: read_data(dir, "firmwareFirmware_dual_image_insufficient.json"), + }, + "5.2(8h)", + "6.1(5e)", + script.FAIL_UF, + ), + # Pre-downloading every node that would otherwise fail with the insufficient-space + # fixture leaves only the already-sufficient nodes, so the result is PASS. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: maintUpgJob_all_of_failing, + firmware: read_data(dir, "firmwareFirmware_dual_image_insufficient.json"), + }, + "5.2(8h)", + "6.1(5e)", + script.PASS, + ), + # Mixed response: only node-102/node-103 are covered, the rest of the failing + # nodes are missing from `maintUpgJob`, and an empty-`dn` entry is ignored. + # The uncovered failing nodes (205, 206, 1002, 1001, 2001, 101) still fail. + ( + { + partitions: read_data(dir, "eqptcapacityFSPartition.json"), + download_sts_615: maintUpgJob_partial_missing_empty, + firmware: read_data(dir, "firmwareFirmware_dual_image_insufficient.json"), + }, + "5.2(8h)", + "6.1(5e)", + script.FAIL_UF, + ), ], ) -def test_logic(run_check, mock_icurl, tversion, expected_result): - result = run_check(tversion=script.AciVersion(tversion)) +def test_logic(run_check, mock_icurl, sw_cversion, tversion, expected_result): + result = run_check( + sw_cversion=script.AciVersion(sw_cversion) if sw_cversion else None, + tversion=script.AciVersion(tversion) if tversion else None, + ) assert result.result == expected_result