Merge apache/main into cks-rolling-update

Resolve the conflict in schema-42300to2400.sql: main's nic_network_map,
disk_offering error_policy and FlashArray rename hunk and this branch's
per-node-type root disk size columns are independent additions; keep
both, with this branch's columns after main's.

Signed-off-by: Brad House <bhouse@nexthop.ai>

Author: bhouse-nexthop

.github/workflows/sbom.yml

@@ -35,15 +35,18 @@ jobs:
         with:
           persist-credentials: false
 
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 24
+
       - name: Generate UI SBOM
-        uses: sbomify/sbomify-action@master
-        env:
-          LOCK_FILE: ui/package-lock.json
-          OUTPUT_FILE: sbom.ui.cdx.json
-          COMPONENT_NAME: cloudstack-ui
-          COMPONENT_VERSION: ${{ github.ref_name }}
-          UPLOAD: false
-          ENRICH: true
+        # the generator is installed outside the checkout without lifecycle scripts;
+        # dev dependencies are build tooling and are not shipped in the UI bundle
+        run: |
+          npm install --no-save --no-audit --no-fund --ignore-scripts \
+            --prefix "$RUNNER_TEMP/cdx" @cyclonedx/cyclonedx-npm@6.0.1
+          "$RUNNER_TEMP/cdx/node_modules/.bin/cyclonedx-npm" --package-lock-only --omit dev \
+            --spec-version 1.6 --output-format JSON --output-file sbom.ui.cdx.json ui/package.json
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -62,7 +65,7 @@ jobs:
         uses: ./.github/actions/setup-env
 
       - name: Generate Java SBOM
-        run: mvn -B -ntp -Dnoredist org.cyclonedx:cyclonedx-maven-plugin:makeAggregateBom
+        run: mvn -B -ntp -Dnoredist -Psbom org.cyclonedx:cyclonedx-maven-plugin:makeAggregateBom
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
@@ -100,17 +103,14 @@ jobs:
           " > marvin-install-requires.txt
           python3 -m venv marvin-venv
           marvin-venv/bin/pip install --quiet -r marvin-install-requires.txt
-          marvin-venv/bin/pip freeze > marvin-requirements.txt
 
       - name: Generate marvin SBOM
-        uses: sbomify/sbomify-action@master
-        env:
-          LOCK_FILE: marvin-requirements.txt
-          OUTPUT_FILE: sbom.marvin.cdx.json
-          COMPONENT_NAME: cloudstack-marvin
-          COMPONENT_VERSION: ${{ github.ref_name }}
-          UPLOAD: false
-          ENRICH: true
+        # the generator gets its own venv so it doesn't end up in marvin's SBOM
+        run: |
+          python3 -m venv sbom-venv
+          sbom-venv/bin/pip install --quiet cyclonedx-bom==7.5.0
+          sbom-venv/bin/cyclonedx-py environment --spec-version 1.6 --output-format JSON \
+            --output-file sbom.marvin.cdx.json marvin-venv/bin/python
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:

PendingReleaseNotes

@@ -91,3 +91,36 @@ example.ver.1 > example.ver.2:
     no device-mapper multipath configuration is required. A new
     Storage.StoragePoolType.NVMeTCP + MultipathNVMeOFAdapterBase /
     NVMeTCPAdapter on the KVM side back the new pool type.
+
+  * New VM allocation algorithm 'balancedweighted' for vm.allocation.algorithm. Existing
+    algorithms and the default are unchanged; the new one is opt-in.
+
+    Existing algorithms rank hosts on allocated capacity alone. Under a large overprovisioning
+    factor that reads badly: allocation is measured against a total that has already been
+    multiplied by the factor, so a host under real strain can still report a low percentage and
+    keep attracting new VMs. Concurrent deployments make it worse, since they all read the same
+    figures before any of them is accounted for.
+
+    'balancedweighted' ranks hosts on a blend of allocated CPU and memory, measured CPU and
+    memory utilisation, VM count, and how many VMs started on the host recently. It holds back
+    hosts that are measurably too busy, and chooses at random from among the best scoring hosts
+    so that simultaneous deployments do not all pick the same one.
+
+    It also holds back hosts that already have host.weighted.starting.vms.threshold VMs sitting
+    in the Starting state, so that a burst of deployments does not queue up behind itself on one
+    host. This is the only signal that reacts within a single deployment: capacity is not charged
+    until a VM reaches Starting and a utilisation average is a sample interval behind, so neither
+    notices a burst until it is over. Set the threshold to 0 to disable it.
+
+    Held-back hosts are never excluded, only moved to the back of the list, so no threshold can
+    turn a deployment that would have succeeded into an InsufficientServerCapacity failure.
+
+    Tuned with the host.weighted.* settings, most of which are cluster scoped. Measured
+    utilisation comes from a moving average configured with host.load.sample.interval and
+    host.load.half.life. Until a management server has collected samples, ranking falls back to
+    allocation figures alone.
+
+KVM console:
+  * Added the global setting `consoleproxy.kvm.multiple.viewers.enabled` to allow
+    multiple users to simultaneously access the console of the same KVM user VM.
+    The setting defaults to false to preserve the existing single-viewer behavior.

agent/conf/agent.properties

@@ -318,6 +318,80 @@ iscsi.session.cleanup.enabled=false
 # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat.
 #reboot.host.and.alert.management.on.heartbeat.timeout=false
 
+# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh / kvmsmpheartbeat.sh when a storage heartbeat
+# write fails persistently. Only consulted when fencing is enabled, i.e.
+# 'reboot.host.and.alert.management.on.heartbeat.timeout' is true: that boolean remains
+# the global on/off switch for fencing, this setting only chooses WHAT the fence does.
+#
+# Allowed values:
+#   hard-reboot     - immediate sysrq-trigger reboot (default).
+#                     Required default for setups where a stale NFSv3 mount can prevent
+#                     a graceful shutdown from completing.
+#   graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly.
+#                     Use only if a stale storage mount cannot block shutdown.
+#   restart-agent   - stop every VM on the host, then restart cloudstack-agent; the
+#                     host itself stays up. If it already fired within
+#                     kvm.heartbeat.fence.restart.agent.cooldown, hard-reboot instead.
+#   log-only        - stop every VM on the host, then only write a message to the
+#                     host's syslog (no reboot, no agent restart, no management server
+#                     alert). The operator must watch syslog and investigate.
+#
+# Any other value falls back to hard-reboot. restart-agent and graceful-reboot need
+# systemctl; if it is missing or the systemctl call fails, the fence hard-reboots.
+#
+# restart-agent and log-only stop the VMs because, once the heartbeat is stale, the
+# management server may declare the host Down and HA may start the same VMs on another
+# host; two copies would then write to the same disks. If any VM cannot be stopped,
+# the fence falls back to hard-reboot.
+#
+# The non-default values are recommended for setups using LINSTOR/DRBD or any local
+# storage with replication, where transient I/O contention can cause a heartbeat
+# write to time out without the host actually being unhealthy.
+#kvm.heartbeat.fence.action=hard-reboot
+
+# Set to true to let restart-agent and log-only leave the VMs running. This is only
+# safe when no VM on this host can be started on another host by HA (VM HA off for all
+# of them and host HA not enabled). Otherwise a host the management server declares
+# Down can end up with the same VM running twice and corrupting its disks.
+# Has no effect on hard-reboot and graceful-reboot.
+# This is the master opt-out: when true, nothing is stopped and
+# kvm.heartbeat.fence.include.system.vms is ignored (the syslog warning says so).
+#kvm.heartbeat.fence.keep.vms.running=false
+
+# Whether restart-agent and log-only also stop system VMs (virtual routers, console
+# proxy, secondary storage and internal LB VMs, recognised by their r-, v-, s- and b-
+# instance names). With true (default) every VM on the host is stopped. With false
+# only user VMs are stopped; system VMs keep running and a system VM that is still
+# running does not trigger the hard-reboot fallback.
+# Risk of false: if the management server declares this host Down it restarts the
+# system VMs on another host, so a virtual router could run twice with the same IP
+# and MAC addresses on the guest network, and a second console proxy or secondary
+# storage VM could run with the same addresses. Only set false if you accept that.
+# Ignored when kvm.heartbeat.fence.keep.vms.running is true. Has no effect on
+# hard-reboot and graceful-reboot.
+#kvm.heartbeat.fence.include.system.vms=true
+
+# Seconds restart-agent and log-only may spend stopping VMs before the host is
+# hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that
+# keeps running even if the agent kills the heartbeat script when
+# heartbeat.update.timeout runs out (stopping a VM can block for a long time on hung
+# storage). If the VMs are not all confirmed stopped within this budget, the deadman
+# hard-reboots the host. Each virsh call is also limited to 10 seconds (or this value
+# if lower) when coreutils timeout is installed. Keep it below heartbeat.update.timeout
+# (60 seconds by default). A value that is not a positive integer means 30.
+# Has no effect on hard-reboot and graceful-reboot, or when
+# kvm.heartbeat.fence.keep.vms.running is true.
+#kvm.heartbeat.fence.stop.vms.timeout=30
+
+# Loop guard for restart-agent, in seconds. While storage keeps failing, the restarted
+# agent runs the heartbeat again, it fails again and restart-agent would restart the
+# agent again, over and over. Each restart-agent fence records its time under /run
+# (kept across agent restarts, cleared by a reboot); if restart-agent already fired less
+# than this many seconds ago, the fence hard-reboots the host instead. 0 turns the guard
+# off. A value that is not a whole number of seconds means 600.
+# Has no effect on hard-reboot, graceful-reboot and log-only.
+#kvm.heartbeat.fence.restart.agent.cooldown=600
+
 # Enables manually setting CPU's topology on KVM's VM.
 #enable.manually.setting.cpu.topology.on.kvm.vm=true
 

agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Properties;
 
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -59,6 +60,8 @@ public synchronized void persist(String key, String value) {
             _properties.store(output, _name);
             output.flush();
             output.close();
+            AgentPropertiesFileHandler.clearCache();
+            logger.debug("Cleared agent properties cache after persisting key: {}", key);
         } catch (IOException e) {
             logger.error("Uh-oh: ", e);
         } finally {

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -622,6 +622,96 @@ public class AgentProperties{
     public static final Property<Boolean> REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT
         = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", false);
 
+    /**
+     * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh, kvmspheartbeat.sh
+     * and kvmsmpheartbeat.sh) when a heartbeat write fails persistently. Only consulted when
+     * {@link #REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT} is true: that boolean remains the
+     * on/off switch for fencing, this setting only chooses what the fence does. Allowed values:
+     * <ul>
+     *   <li>{@code hard-reboot} (default): immediate sysrq-trigger reboot. Required default for setups
+     *       where a stale NFSv3 mount can prevent a graceful shutdown from completing.</li>
+     *   <li>{@code graceful-reboot}: {@code systemctl reboot} instead of sysrq; allows VMs
+     *       to stop cleanly. Use only if a stale storage mount cannot block shutdown.</li>
+     *   <li>{@code restart-agent}: stop every VM on the host, then restart cloudstack-agent; the host stays up. If it
+     *       already fired within {@link #KVM_HEARTBEAT_FENCE_RESTART_AGENT_COOLDOWN}, hard-reboot instead.</li>
+     *   <li>{@code log-only}: stop every VM on the host, then only write a message to the host's syslog (no
+     *       reboot, no agent restart, no management server alert); the operator must investigate.</li>
+     * </ul>
+     * Any other value falls back to {@code hard-reboot}. {@code restart-agent} and {@code graceful-reboot} need
+     * systemctl; if it is missing or the systemctl call fails, the fence hard-reboots.<br>
+     * {@code restart-agent} and {@code log-only} stop the VMs because, once the heartbeat is stale, the management
+     * server may declare the host Down and HA may start the same VMs on another host, where both copies would write
+     * to the same disks. If any VM cannot be stopped the fence falls back to {@code hard-reboot}. See
+     * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING}.<br>
+     * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated
+     * local storage, where transient I/O contention can cause a heartbeat write to time out
+     * without the host actually being unhealthy.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: String.<br>
+     * Default value: {@code hard-reboot}
+     */
+    public static final Property<String> KVM_HEARTBEAT_FENCE_ACTION
+        = new Property<>("kvm.heartbeat.fence.action", "hard-reboot");
+
+    /**
+     * When true, the {@code restart-agent} and {@code log-only} fence actions leave the host's VMs running instead of
+     * stopping them first. Only safe when no VM on the host can be started on another host by HA (VM HA off for all
+     * of them and host HA not enabled); otherwise a host the management server declares Down can end up with the
+     * same VM running twice, writing to the same disks. No effect on {@code hard-reboot} and {@code graceful-reboot}.
+     * This is the master opt-out: when true nothing is stopped and {@link #KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS}
+     * is ignored.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>false</code>
+     */
+    public static final Property<Boolean> KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING
+        = new Property<>("kvm.heartbeat.fence.keep.vms.running", false);
+
+    /**
+     * Whether the {@code restart-agent} and {@code log-only} fence actions also stop system VMs (virtual routers,
+     * console proxy, secondary storage and internal LB VMs, recognised by their {@code r-}, {@code v-}, {@code s-} and
+     * {@code b-} instance names). When true every VM on the host is stopped. When false only user VMs are stopped;
+     * system VMs keep running and do not trigger the {@code hard-reboot} fallback. Risk of false: if the management
+     * server declares the host Down it restarts the system VMs on another host, so a virtual router could run twice
+     * with the same IP and MAC addresses. Ignored when {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true. No
+     * effect on {@code hard-reboot} and {@code graceful-reboot}.
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>true</code>
+     */
+    public static final Property<Boolean> KVM_HEARTBEAT_FENCE_INCLUDE_SYSTEM_VMS
+        = new Property<>("kvm.heartbeat.fence.include.system.vms", true);
+
+    /**
+     * Seconds the {@code restart-agent} and {@code log-only} fence actions may spend stopping VMs before the host is
+     * hard-rebooted instead. Before stopping any VM the fence arms a detached deadman that keeps running even if the
+     * agent kills the heartbeat script when {@link #HEARTBEAT_UPDATE_TIMEOUT} runs out (stopping a VM can block for a
+     * long time on hung storage). If the VMs are not all confirmed stopped within this budget, the deadman
+     * hard-reboots the host. Each virsh call is also limited to 10 seconds, or this value if lower, when coreutils
+     * {@code timeout} is installed. Keep it below {@link #HEARTBEAT_UPDATE_TIMEOUT}. A value that is not a positive
+     * integer falls back to the default. No effect on {@code hard-reboot} and {@code graceful-reboot}, or when
+     * {@link #KVM_HEARTBEAT_FENCE_KEEP_VMS_RUNNING} is true.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Integer.<br>
+     * Default value: <code>30</code>
+     */
+    public static final Property<Integer> KVM_HEARTBEAT_FENCE_STOP_VMS_TIMEOUT
+        = new Property<>("kvm.heartbeat.fence.stop.vms.timeout", 30);
+
+    /**
+     * Loop guard for the {@code restart-agent} fence action, in seconds. While storage keeps failing, the restarted
+     * agent runs the heartbeat again, it fails again and {@code restart-agent} would restart the agent again, over and
+     * over. Each {@code restart-agent} fence records its time under /run (kept across agent restarts, cleared by a
+     * reboot); if {@code restart-agent} already fired less than this many seconds ago, the fence hard-reboots the host
+     * instead. 0 turns the guard off. A value that is not a whole number of seconds falls back to the default. No
+     * effect on {@code hard-reboot}, {@code graceful-reboot} and {@code log-only}.<br>
+     * Read by the heartbeat shell scripts directly from agent.properties.<br>
+     * Data type: Integer.<br>
+     * Default value: <code>600</code>
+     */
+    public static final Property<Integer> KVM_HEARTBEAT_FENCE_RESTART_AGENT_COOLDOWN
+        = new Property<>("kvm.heartbeat.fence.restart.agent.cooldown", 600);
+
     /**
      * Enables manually setting CPU's topology on KVM's VM. <br>
      * Data type: Boolean.<br>