CKS: restart the nodes of a cluster with new node settings, a few at a time

A CloudManaged Kubernetes cluster could get a new service offering only
through a scale, which changes every node at once with no drain, its
affinity groups only while stopped, and its root disk size not at all.

updateKubernetesCluster sets, for each node type (control, worker,
etcd), the service offering, the root disk size and the affinity
groups. The nodes created from then on get them. A cluster created
before its node types had settings of their own first gets the service
offering of the cluster copied to them; templates are left as they are.

With rollout=true, the nodes that differ from the settings of their
type are restarted with them: the external etcd nodes, then the control
nodes, one at a time, then the worker nodes, maxunavailable at a time
(1 by default, and always leaving a worker running). For each batch:

- every other node must be running and Ready, and a control or etcd
  node is only taken down if the others keep a quorum
- each Kubernetes node is annotated with its boot ID, unless someone
  else cordoned it, then cordoned and drained
- its VM is stopped, given the offering, a larger root disk and the
  affinity groups, and started, which places it by its new groups;
  cloud-init grows the root partition and file system at boot
- once it is Ready with another boot ID, and etcd is healthy, it is
  uncordoned and the annotation removed

A control or etcd node type with too few nodes to lose one (a single
control node, a 2-node stacked control plane, 2 etcd nodes) can not be
rolled: a rollout=true that would make one of its nodes differ from the
settings is refused, and its nodes that differ anyway are left out and
reported in lasterror, while the other types are still rolled.

Nodes are restarted rather than changed live, which CloudStack can not
do reliably for a KVM VM, and which would leave the guest and kubelet
unaware of the new resources.

Nothing else is recorded: the nodes that still differ, are stopped or
carry the annotation are the work left, and go first when the rollout
is run again, which is how one that failed is resumed. However a rollout
ends, it uncordons the nodes that are back, leaving the others cordoned,
counts the CPU cores and memory of the cluster from its VMs, and moves
the cluster from the new Updating state to Running, or to Alert if it is
not healthy. Why it stopped is shown as lasterror. cancelrollout=true
stops a rollout between two batches. Other operations are refused while
a cluster is Updating, and the state scanner moves a cluster left
Updating by a management server that stopped to Alert.

The response also shows the number of nodes of each type that differ
from the settings of their type.

Signed-off-by: Brad House <bhouse@nexthop.ai>

Author: bhouse-nexthop

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java

@@ -47,6 +47,7 @@ enum Event {
         AddNodeRequested,
         RemoveNodeRequested,
         UpgradeRequested,
+        UpdateRequested,
         OperationSucceeded,
         OperationFailed,
         CreateFailed,
@@ -62,6 +63,7 @@ enum State {
         Scaling("Transient state in which resources are either getting scaled up/down"),
         ScalingStoppedCluster("Transient state in which the service offerings of stopped clusters are getting scaled"),
         Upgrading("Transient state in which cluster is getting upgraded"),
+        Updating("Transient state in which the nodes of the cluster are restarted with their stored settings, a few at a time"),
         Importing("Transient state in which additional nodes are added as worker nodes to a cluster"),
         RemovingNodes("Transient state in which additional nodes are removed from a cluster"),
         Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."),
@@ -108,6 +110,12 @@ enum State {
             s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert);
 
+            // a rollout ends in Running if the cluster is healthy, and in Alert otherwise
+            s_fsm.addTransition(State.Running, Event.UpdateRequested, State.Updating);
+            s_fsm.addTransition(State.Alert, Event.UpdateRequested, State.Updating);
+            s_fsm.addTransition(State.Updating, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.Updating, Event.OperationFailed, State.Alert);
+
             s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing);
             s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing);
             s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running);
@@ -155,6 +163,9 @@ enum State {
     long getCores();
     long getMemory();
     long getNodeRootDiskSize();
+    Long getControlNodeRootDiskSize();
+    Long getWorkerNodeRootDiskSize();
+    Long getEtcdNodeRootDiskSize();
     String getEndpoint();
     boolean isCheckForGc();
     @Override

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -1324,6 +1324,16 @@ public class ApiConstants {
     public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings";
     public static final String NODE_TYPE_TEMPLATE_MAP = "nodetemplates";
     public static final String NODE_TYPE_AFFINITY_GROUP_MAP = "nodeaffinitygroups";
+    public static final String NODE_TYPE_ROOT_DISK_SIZE_MAP = "noderootdisksizes";
+    public static final String CONTROL_ROOT_DISK_SIZE = "controlrootdisksize";
+    public static final String WORKER_ROOT_DISK_SIZE = "workerrootdisksize";
+    public static final String ETCD_ROOT_DISK_SIZE = "etcdrootdisksize";
+    public static final String CONTROL_NODES_OUT_OF_DATE = "controlnodesoutofdate";
+    public static final String WORKER_NODES_OUT_OF_DATE = "workernodesoutofdate";
+    public static final String ETCD_NODES_OUT_OF_DATE = "etcdnodesoutofdate";
+    public static final String ROLLOUT = "rollout";
+    public static final String CANCEL_ROLLOUT = "cancelrollout";
+    public static final String MAX_UNAVAILABLE = "maxunavailable";
     public static final String CONTROL_AFFINITY_GROUP_IDS = "controlaffinitygroupids";
     public static final String CONTROL_AFFINITY_GROUP_NAMES = "controlaffinitygroupnames";
     public static final String WORKER_AFFINITY_GROUP_IDS = "workeraffinitygroupids";

engine/schema/src/main/resources/META-INF/db/schema-42300to2400.sql

@@ -239,3 +239,8 @@ CREATE TABLE IF NOT EXISTS `cloud`.`instance_boot_group_details` (
     PRIMARY KEY (`id`),
     CONSTRAINT `fk_instance_boot_group_details__group_id` FOREIGN KEY (`boot_group_id`) REFERENCES `instance_boot_group` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- CKS: root disk size per node type. When not set, the cluster wide node_root_disk_size is used, as before.
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'control_node_root_disk_size', 'bigint unsigned DEFAULT NULL COMMENT "root disk size in GB of the control nodes, node_root_disk_size when not set"');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'worker_node_root_disk_size', 'bigint unsigned DEFAULT NULL COMMENT "root disk size in GB of the worker nodes, node_root_disk_size when not set"');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster', 'etcd_node_root_disk_size', 'bigint unsigned DEFAULT NULL COMMENT "root disk size in GB of the etcd nodes, the size of the template when not set"');

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java

@@ -26,4 +26,5 @@ public class KubernetesClusterEventTypes {
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD";
     public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE";
     public static final String EVENT_KUBERNETES_CLUSTER_AFFINITY_UPDATE = "KUBERNETES.CLUSTER.AFFINITY.UPDATE";
+    public static final String EVENT_KUBERNETES_CLUSTER_UPDATE = "KUBERNETES.CLUSTER.UPDATE";
 }