CKS: open the cluster public IP on VPC tiers when the VPC has the Firewall service

Since #12706 a VPC offering can provide the Firewall service for public
IPs. The VPC virtual router then drops ingress traffic to a public IP
that has no firewall rule. CKS only created network ACL items, port
forwarding and the API load balancer rule on VPC tiers, so the
management server could not reach the nodes (SSH 2222+, API 6443), the
control node could not reach its own API through the public IP, and
cluster creation failed after the start timeout.

When the VPC virtual router provides the Firewall service of the
cluster's VPC (the check that sets vpc_firewall_enabled on the VR), CKS
now creates the same ingress firewall rules as on isolated networks on
create (including etcd nodes), scale and node add/remove. In a VPC the
firewall rules are created after the load balancing and port forwarding
rules: applying firewall rules on a VPC public IP does not associate the
IP with the VR, and if they come first the IP is never associated.
Firewall rules on VPC public IPs have no network_id, so per-node rules
are looked up by public IP and ports. On delete the rules go with the
release of the cluster public IP.

Isolated networks, VPCs without the Firewall service and the NSX/Netris
paths are unchanged.

Author: adestis-mf

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

@@ -51,6 +51,7 @@
 import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
 import com.cloud.network.firewall.FirewallService;
 import com.cloud.network.rules.FirewallRule;
+import com.cloud.network.rules.FirewallRuleVO;
 import com.cloud.network.rules.PortForwardingRuleVO;
 import com.cloud.network.rules.RulesService;
 import com.cloud.network.rules.dao.PortForwardingRulesDao;
@@ -59,6 +60,7 @@
 import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.db.TransactionCallbackWithException;
 import com.cloud.utils.net.Ip;
+import com.cloud.utils.net.NetUtils;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicVO;
 import com.cloud.vm.VirtualMachine;
@@ -107,10 +109,12 @@
 import com.cloud.network.Network;
 import com.cloud.network.NetworkModel;
 import com.cloud.network.NetworkService;
+import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.vpc.VpcService;
+import com.cloud.network.vpc.dao.VpcServiceMapDao;
 import com.cloud.projects.ProjectService;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.Storage;
@@ -214,12 +218,16 @@ public class KubernetesClusterActionWorker {
     @Inject
     public VpcService vpcService;
     @Inject
+    protected VpcServiceMapDao vpcServiceMapDao;
+    @Inject
     public PortForwardingRulesDao portForwardingRulesDao;
     @Inject
     protected RulesService rulesService;
     @Inject
     protected FirewallService firewallService;
     @Inject
+    protected FirewallRulesDao firewallRulesDao;
+    @Inject
     private NicDao nicDao;
     @Inject
     protected AffinityGroupDao affinityGroupDao;
@@ -1113,6 +1121,49 @@ protected String updateKubeConfigWithRegistryDetails(String k8sConfig) {
         return k8sConfig;
     }
 
+    /**
+     * When the VPC virtual router provides the Firewall service of a VPC, it drops ingress traffic to
+     * public IPs that has no firewall rule (in addition to the network ACL of each tier). The ports
+     * used by CKS must then be opened on the cluster public IP, as for isolated networks. This is the
+     * same check that sets vpc_firewall_enabled for the VPC virtual router.
+     *
+     * @param network the network of the Kubernetes cluster
+     * @return true if the network is a VPC tier and the VPC virtual router provides the Firewall service
+     */
+    protected boolean isVpcFirewallEnabled(Network network) {
+        if (network == null || network.getVpcId() == null) {
+            return false;
+        }
+        return vpcServiceMapDao.canProviderSupportServiceInVpc(network.getVpcId(), Network.Service.Firewall, Network.Provider.VPCVirtualRouter);
+    }
+
+    /**
+     * @param network the network of the Kubernetes cluster
+     * @return true if ingress firewall rules are needed on the cluster public IP: always for isolated
+     * networks, and for VPC tiers when the VPC uses the Firewall service
+     */
+    protected boolean isPublicIpFirewallRequired(Network network) {
+        return network.getVpcId() == null || isVpcFirewallEnabled(network);
+    }
+
+    /**
+     * Finds the ingress firewall rule for the given public port range of the cluster. Firewall rules on a
+     * public IP of a VPC are stored with the VPC and without a network, so they are looked up by the IP.
+     *
+     * @return the rule, or null if none exists
+     */
+    protected FirewallRuleVO findIngressFirewallRuleForPorts(Network network, Long publicIpId, int startPort, int endPort) {
+        if (network.getVpcId() == null) {
+            return firewallRulesDao.findByNetworkIdAndPorts(network.getId(), startPort, endPort);
+        }
+        if (publicIpId == null) {
+            return null;
+        }
+        List<FirewallRuleVO> rules = firewallRulesDao.listByIpPurposePortsProtocolAndNotRevoked(publicIpId, startPort, endPort,
+                NetUtils.TCP_PROTO, FirewallRule.Purpose.Firewall);
+        return CollectionUtils.isEmpty(rules) ? null : rules.get(0);
+    }
+
     public Map<Long, Integer> addFirewallRulesForNodes(IpAddress publicIp, int size) throws ManagementServerException {
         Map<Long, Integer> vmIdPortMap = new HashMap<>();
         CallContext.register(CallContext.current(), null);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java

@@ -31,7 +31,6 @@
 import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
 import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
-import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.rules.FirewallRuleVO;
 import com.cloud.network.rules.PortForwardingRuleVO;
 import com.cloud.service.ServiceOfferingVO;
@@ -49,7 +48,6 @@
 import org.apache.commons.codec.binary.Base64;
 import org.apache.logging.log4j.Level;
 
-import javax.inject.Inject;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
@@ -60,8 +58,6 @@
 
 public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
 
-    @Inject
-    private FirewallRulesDao firewallRulesDao;
     private long addNodeTimeoutTime;
 
     List<Long> finalNodeIds = new ArrayList<>();
@@ -229,9 +225,13 @@ private Pair<Boolean, Integer> validateAndSetupNode(Network network, IpAddress p
                 provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort);
             }
             provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT);
+            if (isVpcFirewallEnabled(network)) {
+                // In a VPC the firewall rule follows the port forwarding rule, see setupKubernetesClusterVpcTierRules
+                provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort);
+            }
             boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort);
             if (!isCompatible) {
-                revertNetworkRules(network, nodeId, sshStartPort);
+                revertNetworkRules(network, publicIp, nodeId, sshStartPort);
                 return new Pair<>(false, nodeIndex);
             }
 
@@ -249,13 +249,13 @@ private Pair<Boolean, Integer> validateAndSetupNode(Network network, IpAddress p
                  InsufficientCapacityException | IllegalAccessException e) {
             logger.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()));
             // remove added Firewall and PF rules
-            revertNetworkRules(network, nodeId, sshStartPort);
+            revertNetworkRules(network, publicIp, nodeId, sshStartPort);
             return new Pair<>( false, nodeIndex);
         } catch (Exception e) {
             String errMsg = String.format("Unexpected exception while trying to add the external node %s to the Kubernetes cluster %s: %s",
                     nodeId, kubernetesCluster.getName(), e.getMessage());
             logger.error(errMsg, e);
-            revertNetworkRules(network, nodeId, sshStartPort);
+            revertNetworkRules(network, publicIp, nodeId, sshStartPort);
             throw new CloudRuntimeException(e);
         }
         return new Pair<>(true, ++nodeIndex);
@@ -310,12 +310,14 @@ private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File p
         }
     }
 
-    private void revertNetworkRules(Network network, long vmId, int port) {
+    private void revertNetworkRules(Network network, IpAddress publicIp, long vmId, int port) {
         logger.debug(String.format("Reverting network rules for VM ID %s on network %s", vmId, network.getName()));
-        FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port);
-        if (Objects.isNull(network.getVpcId())) {
-            logger.debug(String.format("Removing firewall rule %s", ruleVO.getId()));
-            firewallService.revokeIngressFirewallRule(ruleVO.getId(), true);
+        if (isPublicIpFirewallRequired(network)) {
+            FirewallRuleVO ruleVO = findIngressFirewallRuleForPorts(network, publicIp.getId(), port, port);
+            if (ruleVO != null) {
+                logger.debug(String.format("Removing firewall rule %s", ruleVO.getId()));
+                firewallService.revokeIngressFirewallRule(ruleVO.getId(), true);
+            }
         }
         List<PortForwardingRuleVO> pfRules = portForwardingRulesDao.listByVm(vmId);
         for (PortForwardingRuleVO pfRule : pfRules) {

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java

@@ -26,7 +26,6 @@
 import com.cloud.kubernetes.cluster.KubernetesClusterVO;
 import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
-import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.rules.FirewallRuleVO;
 import com.cloud.network.rules.PortForwardingRuleVO;
 import com.cloud.service.ServiceOfferingVO;
@@ -37,7 +36,6 @@
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.context.CallContext;
 
-import javax.inject.Inject;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
@@ -47,9 +45,6 @@
 
 public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker {
 
-    @Inject
-    private FirewallRulesDao firewallRulesDao;
-
     private long removeNodeTimeoutTime;
 
     public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) {
@@ -128,9 +123,13 @@ protected boolean removeNodePortForwardingRules(Long nodeId, Network network, Us
         for (PortForwardingRuleVO pfRule : pfRules) {
             try {
                 result &= rulesService.revokePortForwardingRule(pfRule.getId(), true);
-                if (Objects.isNull(network.getVpcId())) {
-                    FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd());
-                    result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true);
+                if (isPublicIpFirewallRequired(network)) {
+                    FirewallRuleVO ruleVO = findIngressFirewallRuleForPorts(network, pfRule.getSourceIpAddressId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd());
+                    if (ruleVO != null) {
+                        result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true);
+                    } else {
+                        logger.warn("Firewall rule for port {} of node {} not found, it may have been removed manually", pfRule.getSourcePortStart(), vm.getName());
+                    }
                 }
             } catch (Exception e) {
                 String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage());

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java

@@ -82,7 +82,6 @@
 import com.cloud.kubernetes.cluster.KubernetesClusterVO;
 import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
-import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.LoadBalancerDao;
 import com.cloud.network.dao.LoadBalancerVO;
 import com.cloud.network.lb.LoadBalancingRulesService;
@@ -130,8 +129,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
     @Inject
     protected HostDao hostDao;
     @Inject
-    protected FirewallRulesDao firewallRulesDao;
-    @Inject
     protected NetworkACLService networkACLService;
     @Inject
     protected  NetworkACLItemDao networkACLItemDao;
@@ -832,6 +829,14 @@ protected void setupKubernetesClusterVpcTierRules(IpAddress publicIp, Network ne
         } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
             throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
         }
+
+        // The VPC firewall drops traffic to the public IP unless a firewall rule allows it, open the
+        // API and SSH ports as for isolated networks. Create them after the load balancing and port
+        // forwarding rules: applying firewall rules on a VPC public IP does not associate the IP with
+        // the VPC virtual router, the first load balancing or port forwarding rule does.
+        if (isVpcFirewallEnabled(network)) {
+            createFirewallRules(publicIp, clusterVMIds, true);
+        }
     }
 
 

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java

@@ -173,6 +173,14 @@ private void scaleKubernetesClusterVpcTierRules(final List<Long> clusterVMIds) t
         if (publicIp == null) {
             throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
         }
+        boolean vpcFirewallEnabled = isVpcFirewallEnabled(network);
+        if (vpcFirewallEnabled) {
+            // Remove the existing SSH firewall rule, it is recreated below for the new node count
+            FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId());
+            if (firewallRule == null) {
+                logger.warn("SSH firewall rule not found for Kubernetes cluster: {}. It may have been manually deleted or modified.", kubernetesCluster.getName());
+            }
+        }
         try {
             removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1);
         } catch (ResourceUnavailableException e) {
@@ -184,6 +192,10 @@ private void scaleKubernetesClusterVpcTierRules(final List<Long> clusterVMIds) t
         } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
             throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
         }
+        if (vpcFirewallEnabled) {
+            // After the port forwarding rules, as when the cluster is created
+            addFirewallRulesForNodes(publicIp, clusterVMIds.size());
+        }
     }
 
     /**

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java

@@ -691,6 +691,14 @@ protected void setupKubernetesEtcdNetworkRules(List<UserVm> etcdVms, Network net
                 throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
             }
             provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), etcdStartPort, DEFAULT_SSH_PORT);
+            if (isVpcFirewallEnabled(network)) {
+                // In a VPC the firewall rule follows the port forwarding rule, see setupKubernetesClusterVpcTierRules
+                try {
+                    provisionFirewallRules(publicIp, owner, etcdStartPort, etcdStartPort);
+                } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
+                    throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+                }
+            }
         }
     }