434-dynamically-calculate-the-required-free-space-in-switch-bootflash

aci-preupgrade-validation-script.py

@@ -2158,14 +2158,20 @@ def switch_group_guideline_check(fabric_nodes, **kwargs):
 
 
 @check_wrapper(check_title="Switch Node /bootflash usage")
-def switch_bootflash_usage_check(tversion, **kwargs):
+def switch_bootflash_usage_check(sw_cversion, tversion, **kwargs):
     result = FAIL_UF
     msg = ''
-    headers = ["Pod-ID", "Node-ID", "Utilization"]
+    headers = ["Pod-ID", "Node-ID", "Avail (MB)", "Required (MB)"]
     data = []
-    recommended_action = "Over 50% usage! Contact Cisco TAC for Support"
+    recommended_action = "Insufficient free space to download and extract the target image! Contact Cisco TAC for Support"
     doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-node-bootflash-usage"
 
+    if not sw_cversion:
+        return Result(result=MANUAL, msg="Current switch version not found. Check switch health.", doc_url=doc_url)
+
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING, doc_url=doc_url)
+
     partitions_api = 'eqptcapacityFSPartition.json'
     partitions_api += '?query-target-filter=eq(eqptcapacityFSPartition.path,"/bootflash")'
 
@@ -2175,34 +2181,100 @@ def switch_bootflash_usage_check(tversion, **kwargs):
 
     partitions = icurl('class', partitions_api)
     if not partitions:
-        return Result(result=MANUAL, msg='bootflash objects not found. Check switch health.', doc_url=doc_url)
+        return Result(result=MANUAL, msg='/bootflash directory not found. Check switch health.', doc_url=doc_url)
 
     predownloaded_nodes = []
     try:
         download_sts = icurl('class', download_sts_api)
     except OldVerPropNotFound:
         # Older versions don't have 'dnldStatus' param
         download_sts = []
-
+   
     for maintUpgJob in download_sts:
         dn = re.search(node_regex, maintUpgJob['maintUpgJob']['attributes']['dn'])
-        node = dn.group("node")
-        predownloaded_nodes.append(node)
+        if dn:
+            predownloaded_nodes.append(dn.group("node"))
+
+    # Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit
+    # isos (`-cs_64` suffix for 64-bit). Below that, only a single 32-bit iso exists.
+    boundary_version = "6.0(2a)"
+    switch_target_version = "aci-n9000-dk9.1{}.bin".format(tversion.dot_version)
+    switch_target_version_64 = "aci-n9000-dk9.1{}-cs_64.bin".format(tversion.dot_version)
+
+    firmware_api = 'firmwareFirmware.json?query-target-filter=eq(firmwareFirmware.type,"switch")'
+    firmwares = icurl('class', firmware_api)
+    fw_sizes = {}
+    for firmware in firmwares:
+        fw_attr = firmware['firmwareFirmware']['attributes']
+        fw_sizes[fw_attr['isoname']] = int(fw_attr['size'])
+
+    target_size_32 = fw_sizes.get(switch_target_version)
+    target_size_64 = fw_sizes.get(switch_target_version_64)
+
+    # sw_cversion (lowest switch version), not the APIC cversion, drives the boundary
+    # decision: the upgrade guide has APICs reach 6.0(2a)+ before the switches, so the
+    # switches can still be pre-boundary while the APIC cluster is already post-boundary.
+    target_is_legacy = tversion.older_than(boundary_version)
+    current_is_legacy = sw_cversion.older_than(boundary_version)
+
+    if target_is_legacy:
+        # Only the 32-bit image is ever used for a pre-6.0(2a) target.
+        if target_size_32 is None:
+            msg = 'Target switch image ({}) not found in Firmware Repository.'.format(switch_target_version)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+        required_space = 2 * target_size_32
+        downloaded_required_space = target_size_32
+
+    else:
+        # The larger image is used as a conservative estimate, so both sizes must be
+        # known; a missing one can't be assumed to be the smaller (or zero-byte) one.
+        if target_size_32 is None and target_size_64 is None:
+            msg = 'Target switch images ({}, {}) not found in Firmware Repository.'.format(switch_target_version, switch_target_version_64)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+        elif target_size_32 is None:
+            msg = '32-bit target switch image ({}) not found in Firmware Repository.'.format(switch_target_version)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+        elif target_size_64 is None:
+            msg = '64-bit target switch image ({}) not found in Firmware Repository.'.format(switch_target_version_64)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+
+        downloaded_required_space = max(target_size_32, target_size_64)
+
+        if current_is_legacy:
+            # Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a
+            # 32-bit image, so its size is freed once removed during the upgrade.
+            switch_current_version = "aci-n9000-dk9.1{}.bin".format(sw_cversion.dot_version)
+            current_size = fw_sizes.get(switch_current_version)
+            if current_size is None:
+                msg = 'Current switch image ({}) not found in Firmware Repository.'.format(switch_current_version)
+                return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+            if target_size_32 > current_size:
+                required_space = 2 * (target_size_32 + target_size_64 - current_size)
+            else:
+                required_space = 2 * max(target_size_32, target_size_64)
+        else:
+            required_space = 2 * max(target_size_32, target_size_64)
+
+    required_space_kb = required_space / 1024.0  # eqptcapacityFSPartition avail/used are in KB
+    downloaded_required_space_kb = downloaded_required_space / 1024.0
 
     for eqptcapacityFSPartition in partitions:
         dn = re.search(node_regex, eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['dn'])
         pod = dn.group("pod")
         node = dn.group("node")
         avail = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['avail'])
-        used = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['used'])
 
-        usage = (used / (avail + used)) * 100
-        if (usage >= 50) and (node not in predownloaded_nodes):
-            data.append([pod, node, usage])
+        # dnldStatus == downloaded only proves the image was delivered, not that
+        # extraction (which still consumes bootflash) has completed, so a downloaded
+        # node is still checked, just against the smaller extraction-only requirement.
+        node_required_space_kb = downloaded_required_space_kb if node in predownloaded_nodes else required_space_kb
+
+        if avail < node_required_space_kb:
+            data.append([pod, node, round(avail / 1024.0, 2), round(node_required_space_kb / 1024.0, 2)])
 
     if not data:
         result = PASS
-        msg = 'All below 50% or pre-downloaded'
+        msg = 'All nodes have sufficient bootflash space'
     return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 

docs/docs/validations.md

@@ -651,67 +651,16 @@ To prevent this, check the `/bootflash` prior to an upgrade and take the necessa
 
 The pre-upgrade validation built into Cisco APIC upgrade workflow monitors the fault F1821, which can capture the high utilization of any partition. When this fault is present, we recommend that you resolve it prior to the upgrade even if the fault is not for bootflash.
 
-The ACI Pre-Upgrade Validation script (this script) focuses on the utilization of bootflash on each switch specifically to see if there are any issues with bootflash where the usage is more than 50%, which might trigger the internal cleanup script.
-
-!!! example "Example of a query used by this script"
-    The script is calculating the bootflash usage using `avail` and `used` in the object `eqptcapacityFSPartition` for each switch.
-    ```
-    f2-apic1# moquery -c eqptcapacityFSPartition -f 'eqptcapacity.FSPartition.path=="/bootflash"'
-    Total Objects shown: 6
-
-    # eqptcapacity.FSPartition
-    name            : bootflash
-    avail           : 7214920
-    childAction     :
-    dn              : topology/pod-1/node-101/sys/eqptcapacity/fspartition-bootflash
-    memAlert        : normal
-    modTs           : never
-    monPolDn        : uni/fabric/monfab-default
-    path            : /bootflash
-    rn              : fspartition-bootflash
-    status          :
-    used            : 4320184
-    --- omit ---
-    ```
+The ACI Pre-Upgrade Validation script (this script) dynamically calculates the actual bootflash space required for the target upgrade, rather than relying on a fixed usage threshold, and compares it against each switch's available `/bootflash` space:
 
-!!! tip
-    Alternatively you can log into a leaf switch CLI, and check `/bootflash` usage `df -h`
-    ```
-    leaf1# df -h
-    Filesystem             Size    Used   Avail   Use%   Mounted on
-    rootfs                 2.5G    935M   1.6G    38%    /bin
-    /dev/sda4               12G    5.7G   4.9G    54%    /bootflash
-    /dev/sda2              4.7G    9.6M   4.4G     1%    /recovery
-    /dev/mapper/map-sda9    11G    5.7G   4.2G    58%    /isan/lib
-    none                   3.0G    602M   2.5G    20%    /dev/shm
-    none                    50M    3.4M    47M     7%    /etc
-    /dev/sda6               56M    1.3M    50M     3%    /mnt/cfg/1
-    /dev/sda5               56M    1.3M    50M     3%    /mnt/cfg/0
-    /dev/sda8               15G    140M    15G     1%    /mnt/ifc/log
-    /dev/sda3              115M     52M    54M    50%    /mnt/pss
-    none                   1.5G    2.3M   1.5G     1%    /tmp
-    none                    50M    240K    50M     1%    /var/log
-    /dev/sda7               12G    1.4G   9.3G    13%    /logflash
-    none                   350M     54M   297M    16%    /var/log/dme/log/dme_logs
-    none                   512M     24M   489M     5%    /var/sysmgr/mem_logs
-    none                    40M    4.0K    40M     1%    /var/sysmgr/startup-cfg
-    none                   500M     0     500M     0%    /volatile
-    ```
+* Required space is based on the target image size(s) needed to download and extract on top of the existing content. Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit isos, so both current and target version determine whether one or both images apply:
+    * Both versions pre-6.0(2a): only the single 32-bit image size is used.
+    * Both versions post-6.0(2a): the larger of the 32-bit/64-bit target images is used.
+    * Crossing the 6.0(2a) boundary: target image downloaded while the current is removed, freeing its space for successful extraction.
 
-!!! note
-    If you suspect that the auto cleanup removed some files within `/bootflash`, you can review a log to validate this:
+* Nodes that already downloaded the exact target version (`maintUpgJob.dnldStatus == downloaded` and `desiredVersion` matching target) are still evaluated, just against a smaller, extraction-only requirement (the larger of the 32-bit/64-bit target image sizes, without doubling for the download), since extraction and later upgrade stages can still require additional bootflash space.
 
-    ```
-    leaf1# egrep "higher|removed" /mnt/pss/core_control.log
-    [2020-07-22 16:52:08.928318] Bootflash Usage is higher than 50%!!
-    [2020-07-22 16:52:08.931990] File: MemoryLog.65%_usage removed !!
-    [2020-07-22 16:52:08.943914] File: mem_log.txt.old.gz removed !!
-    [2020-07-22 16:52:08.955376] File: libmon.logs removed !!
-    [2020-07-22 16:52:08.966686] File: urib_api_log.txt removed !!
-    [2020-07-22 16:52:08.977832] File: disk_log.txt removed !!
-    [2020-07-22 16:52:08.989102] File: mem_log.txt removed !!
-    [2020-07-22 16:52:09.414572] File: aci-n9000-dk9.13.2.1m.bin removed !!
-    ```
+* If a required firmware image isn't found in the Firmware Repository, the check reports a manual review.
 
 
 ### APIC SSD Health

tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_602.json

@@ -0,0 +1,4 @@
+[
+  {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h.bin", "size": "2000000000"}}},
+  {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h-cs_64.bin", "size": "3000000000"}}}
+]

tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_insufficient.json

@@ -0,0 +1,35 @@
+[
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e.bin",
+        "name": "aci-n9000-system.16.1.5e.bin",
+        "size": "3000000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e-cs_64.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin",
+        "name": "aci-n9000-system.16.1.5e-cs_64.bin",
+        "size": "3000000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.15.2.8h.bin",
+        "fullVersion": "n9000-15.2(8h)",
+        "isoname": "aci-n9000-dk9.15.2.8h.bin",
+        "name": "aci-n9000-system.15.2.8h.bin",
+        "size": "2000000000"
+      }
+    }
+  }
+]

tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_sufficient.json

@@ -0,0 +1,35 @@
+[
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e.bin",
+        "name": "aci-n9000-system.16.1.5e.bin",
+        "size": "1500000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e-cs_64.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin",
+        "name": "aci-n9000-system.16.1.5e-cs_64.bin",
+        "size": "1500000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.15.2.8h.bin",
+        "fullVersion": "n9000-15.2(8h)",
+        "isoname": "aci-n9000-dk9.15.2.8h.bin",
+        "name": "aci-n9000-system.15.2.8h.bin",
+        "size": "2000000000"
+      }
+    }
+  }
+]

tests/checks/switch_bootflash_usage_check/maintUpgJob_all_downloaded.json

@@ -0,0 +1,13 @@
+[
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-205/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-206/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2003/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2010/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-101/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}
+]

tests/checks/switch_bootflash_usage_check/maintUpgJob_all_of_failing.json

@@ -0,0 +1,10 @@
+[
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-205/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-206/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-101/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}
+]

tests/checks/switch_bootflash_usage_check/maintUpgJob_old_ver_no_prop.json

@@ -1,10 +1,10 @@
 [
   {
-      "error": {
-          "attributes": {
-              "code": "121",
-              "text": "Prop 'dnldStatus' not found in class 'maintUpgJob' property table"
-          }
+    "error": {
+      "attributes": {
+        "code": "400",
+        "text": "Prop 'dnldStatus' not found in class 'maintUpgJob' property table"
       }
+    }
   }
-]
+]

tests/checks/switch_bootflash_usage_check/maintUpgJob_partial_missing_empty.json

@@ -0,0 +1,5 @@
+[
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "", "dnldStatus": "downloaded", "dnldPercent": "100"}}}
+]