From edc7c2e100a64346de15e92e5f149f927206a0e7 Mon Sep 17 00:00:00 2001 From: stardom3645 Date: Fri, 14 Aug 2026 09:58:06 +0900 Subject: [PATCH] =?UTF-8?q?[Mold][Genie]=20[=EB=B2=84=EA=B7=B8=20=EC=88=98?= =?UTF-8?q?=EC=A0=95]=20=EC=82=AD=EC=A0=9C=20=EB=B2=84=EA=B7=B8=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/AutomationController.java | 9 +- .../AutomationControllerManagerImpl.java | 138 +++++++--- .../controller/AutomationControllerVO.java | 3 +- .../AutomationControllerActionWorker.java | 53 ++-- .../AutomationControllerDestroyWorker.java | 123 ++++----- ...ontrollerResourceModifierActionWorker.java | 35 +-- .../AutomationControllerStartWorker.java | 245 +++++++++--------- .../src/main/resources/conf/genie.yml | 25 +- .../AutomationControllerStateTest.java | 53 ++++ 9 files changed, 391 insertions(+), 293 deletions(-) create mode 100644 plugins/integrations/automation-service/src/test/java/com/cloud/automation/controller/AutomationControllerStateTest.java diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationController.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationController.java index 5e86f55653fa..8ea74f870b28 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationController.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationController.java @@ -68,16 +68,19 @@ enum State { static { s_fsm.addTransition(State.Created, Event.StartRequested, State.Starting); + s_fsm.addTransition(State.Created, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Starting, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Starting, Event.OperationFailed, State.Alert); s_fsm.addTransition(State.Starting, Event.CreateFailed, State.Error); s_fsm.addTransition(State.Starting, Event.StopRequested, State.Stopping); + s_fsm.addTransition(State.Starting, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Running, Event.StopRequested, State.Stopping); s_fsm.addTransition(State.Alert, Event.StopRequested, State.Stopping); s_fsm.addTransition(State.Stopping, Event.OperationSucceeded, State.Stopped); s_fsm.addTransition(State.Stopping, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Stopping, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Stopped, Event.StartRequested, State.Starting); @@ -87,19 +90,23 @@ enum State { s_fsm.addTransition(State.Running, Event.ScaleDownRequested, State.Scaling); s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Scaling, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Scaling, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Running, Event.UpgradeRequested, State.Upgrading); s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Upgrading, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering); s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Recovering, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Running, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Stopped, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Alert, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Error, Event.DestroyRequested, State.Destroying); + s_fsm.addTransition(State.Destroying, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Destroying, Event.OperationSucceeded, State.Destroyed); @@ -128,4 +135,4 @@ enum State { State getState(); Date getCreated(); Date getRemoved(); -} \ No newline at end of file +} diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerManagerImpl.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerManagerImpl.java index ae75e431fc2e..e49bc89f9279 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerManagerImpl.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerManagerImpl.java @@ -38,14 +38,12 @@ import com.cloud.automation.version.AutomationControllerVersion; import com.cloud.automation.version.AutomationControllerVersionVO; import com.cloud.automation.version.dao.AutomationControllerVersionDao; -import com.cloud.dc.DataCenter; import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.Network; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; -import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -175,7 +173,9 @@ public AutomationControllerResponse addAutomationControllerResponse(long automat } NetworkVO ntwk = networkDao.findByIdIncludingRemoved(automationController.getNetworkId()); - response.setNetworkId(ntwk.getUuid()); + if (ntwk != null) { + response.setNetworkId(ntwk.getUuid()); + } response.setAutomationControllerIp(automationController.getAutomationControllerIp()); response.setRemoved(automationController.getRemoved()); DataCenterVO zone = dataCenterDao.findById(automationController.getZoneId()); @@ -184,24 +184,29 @@ public AutomationControllerResponse addAutomationControllerResponse(long automat response.setZoneName(zone.getName()); } - if (ntwk.getGuestType() == Network.GuestType.Isolated) { + if (ntwk != null && ntwk.getGuestType() == Network.GuestType.Isolated) { List ipAddresses = ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true); - if (ipAddresses != null && ipAddresses.size() == 1) { - response.setIpAddress(ipAddresses.get(0).getAddress().addr()); - response.setIpAddressId(ipAddresses.get(0).getUuid()); + IPAddressVO sourceNatIp = findSourceNatIp(ipAddresses); + if (sourceNatIp != null) { + response.setIpAddress(sourceNatIp.getAddress().addr()); + response.setIpAddressId(sourceNatIp.getUuid()); } } ServiceOfferingVO offering = serviceOfferingDao.findById(automationController.getServiceOfferingId()); - response.setServiceOfferingId(offering.getUuid()); - response.setServiceOfferingName(offering.getName()); + if (offering != null) { + response.setServiceOfferingId(offering.getUuid()); + response.setServiceOfferingName(offering.getName()); + } Account account = ApiDBUtils.findAccountById(automationController.getAccountId()); - if (account.getType() == Account.Type.PROJECT) { + if (account != null && account.getType() == Account.Type.PROJECT) { Project project = ApiDBUtils.findProjectByProjectAccountId(account.getId()); - response.setProjectId(project.getUuid()); - response.setProjectName(project.getName()); - } else { + if (project != null) { + response.setProjectId(project.getUuid()); + response.setProjectName(project.getName()); + } + } else if (account != null) { response.setAccountName(account.getAccountName()); } @@ -222,18 +227,15 @@ public AutomationControllerResponse addAutomationControllerResponse(long automat UserVmResponse cvmResponse = ApiDBUtils.newUserVmResponse(respView, responseName, userVM, EnumSet.of(ApiConstants.VMDetails.nics), caller); automationControllerVmResponses.add(cvmResponse); response.setAutomationControllerIp(userVM.getIpAddress()); - automationController.setAutomationControllerIp(userVM.getIpAddress()); - } - try { GuestOS guestOS = ApiDBUtils.findGuestOSById(userVM.getGuestOsId()); if (guestOS != null) { response.setOsDisplayName(guestOS.getDisplayName()); } - } catch (NullPointerException e) { - deleteAutomationController(automationController.getId()); - logger.warn(String.format("Failed to run Automation controller Alert state scanner on Automation controller : %s status scanner", automationController.getName()), e); + response.setHostName(userVM.getHostName()); + } else { + logger.warn(String.format("VM %d mapped to automation controller %s no longer exists", + vmMapVO.getVmId(), automationController.getName())); } - response.setHostName(userVM.getHostName()); } } @@ -262,7 +264,12 @@ private String resolveAutomationControllerResponseState(AutomationControllerVO a final String address = getAutomationControllerPublicIpAddress(automationController); if (address != null && isAutomationControllerHttpReady(address, AUTOMATION_CONTROLLER_HTTP_PORT)) { - return AutomationController.State.Running.toString(); + if (stateTransitTo(automationController.getId(), AutomationController.Event.RecoveryRequested) + && stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded)) { + logger.info(String.format("Recovered automation controller %s from Alert after its HTTP endpoint became ready", + automationController.getName())); + return AutomationController.State.Running.toString(); + } } return automationController.getState().toString(); @@ -274,12 +281,20 @@ private String getAutomationControllerPublicIpAddress(AutomationControllerVO aut return null; } - List ipAddresses = ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true); - if (ipAddresses == null || ipAddresses.size() != 1) { + IPAddressVO sourceNatIp = findSourceNatIp(ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true)); + return sourceNatIp == null ? null : sourceNatIp.getAddress().addr(); + } + + private IPAddressVO findSourceNatIp(List ipAddresses) { + if (ipAddresses == null) { return null; } - - return ipAddresses.get(0).getAddress().addr(); + for (IPAddressVO ipAddress : ipAddresses) { + if (ipAddress != null && ipAddress.isSourceNat()) { + return ipAddress; + } + } + return null; } private boolean isAutomationControllerHttpReady(String address, int port) { @@ -307,7 +322,6 @@ public ListResponse listAutomationController(ListA if (!AutomationServiceEnabled.value()) { throw new CloudRuntimeException("Automation Service plugin is disabled"); } - final Long versionId = cmd.getId(); final Long zoneId = cmd.getZoneId(); final CallContext ctx = CallContext.current(); final Account caller = ctx.getCallingAccount(); @@ -343,9 +357,6 @@ public ListResponse listAutomationController(ListA if (name != null) { sc.setParameters("name", name); } - if (versionId != null) { - sc.setParameters("id", versionId); - } if (zoneId != null) { SearchCriteria scc = automationControllerDao.createSearchCriteria(); scc.addOr("zoneId", SearchCriteria.Op.EQ, zoneId); @@ -369,6 +380,11 @@ public ListResponse listAutomationController(ListA protected boolean stateTransitTo(long automationControllerId, AutomationController.Event e) { AutomationControllerVO automationController = automationControllerDao.findById(automationControllerId); + if (automationController == null) { + logger.warn(String.format("Failed to transition missing automation controller %d on event %s", + automationControllerId, e)); + return false; + } try { return _stateMachine.transitTo(automationController, e, null, automationControllerDao); } catch (NoTransitionException nte) { @@ -385,16 +401,18 @@ public AutomationController addAutomationController(final AddAutomationControlle } validateAutomationControllerCreateParameters(cmd); - final String L2Type = "internal"; - final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final AutomationControllerVersion automationControllerVersion = automationControllerVersionDao.findById(cmd.getAutomationTemplateId()); - AutomationControllerResponse response = new AutomationControllerResponse(); - Long instanceId = Long.valueOf(3); + if (owner == null) { + throw new InvalidParameterValueException("Unable to find the owner account for the Automation controller"); + } + if (automationControllerVersion == null) { + throw new InvalidParameterValueException("Unable to find the requested Automation controller template version"); + } final AutomationControllerVO controller = Transaction.execute(new TransactionCallback() { @Override public AutomationControllerVO doInTransaction(TransactionStatus status) { - AutomationControllerVO newController = new AutomationControllerVO(automationControllerVersion.getId(), cmd.getName(), cmd.getDescription(), cmd.getAutomationTemplateId(), cmd.getZoneId(), + AutomationControllerVO newController = new AutomationControllerVO(cmd.getName(), cmd.getDescription(), automationControllerVersion.getId(), cmd.getZoneId(), cmd.getServiceOfferingId(), cmd.getNetworkId(), cmd.getNetworkName(), owner.getAccountId(), cmd.getDomainId(), AutomationController.State.Created, cmd.getAutomationControllerIp()); automationControllerDao.persist(newController); return newController; @@ -410,10 +428,8 @@ public AutomationControllerVO doInTransaction(TransactionStatus status) { private void validateAutomationControllerCreateParameters(final AddAutomationControllerCmd cmd) throws CloudRuntimeException { final String name = cmd.getName(); final String description = cmd.getDescription(); - final Long accountId = cmd.getAccountId(); final Long networkId = cmd.getNetworkId(); final String networkName = cmd.getNetworkName(); - final String ipAddress = cmd.getAutomationControllerIp(); if (name == null || name.isEmpty()) { throw new InvalidParameterValueException("Invalid name for the Automation controller name:" + name); @@ -422,6 +438,33 @@ private void validateAutomationControllerCreateParameters(final AddAutomationCon throw new InvalidParameterValueException("Invalid name. Automation controller name can contain ASCII letters 'a' through 'z', the digits '0' through '9', " + "and the hyphen ('-'), and can't start or end with \"-\" and can't start with digit"); } + if (networkId == null) { + throw new InvalidParameterValueException("Automation controller network ID is required"); + } + NetworkVO network = networkDao.findById(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find the requested Automation controller network"); + } + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + throw new InvalidParameterValueException("Automation controllers require an isolated network with a source NAT IP"); + } + if (cmd.getZoneId() == null || dataCenterDao.findById(cmd.getZoneId()) == null) { + throw new InvalidParameterValueException("Unable to find the requested Automation controller zone"); + } + if (network.getDataCenterId() != cmd.getZoneId()) { + throw new InvalidParameterValueException("Automation controller network and zone do not match"); + } + if (cmd.getServiceOfferingId() == null || serviceOfferingDao.findById(cmd.getServiceOfferingId()) == null) { + throw new InvalidParameterValueException("Unable to find the requested Automation controller service offering"); + } + AutomationControllerVersion version = cmd.getAutomationTemplateId() == null ? null + : automationControllerVersionDao.findById(cmd.getAutomationTemplateId()); + if (version == null) { + throw new InvalidParameterValueException("Unable to find the requested Automation controller template version"); + } + if (version.getZoneId() == null || !version.getZoneId().equals(cmd.getZoneId())) { + throw new InvalidParameterValueException("Automation controller template version and zone do not match"); + } final List controllers = automationControllerDao.listAll(); for (final AutomationControllerVO controller : controllers) { final String otherName = controller.getName(); @@ -433,7 +476,7 @@ private void validateAutomationControllerCreateParameters(final AddAutomationCon if (otherNetwork.equals(networkId)){ throw new InvalidParameterValueException("Automation controller network id '" + networkId + "' already deployed."); } - if (otherNetworkName.equals(networkName)){ + if (otherNetworkName != null && otherNetworkName.equals(networkName)){ throw new InvalidParameterValueException("Automation controller network name '" + networkName + "' already deployed."); } } @@ -445,7 +488,7 @@ private void validateAutomationControllerCreateParameters(final AddAutomationCon @Override public boolean startAutomationController(long automationControllerId, boolean onCreate) throws CloudRuntimeException { if (!AutomationServiceEnabled.value()) { -// logAndThrow(Level.ERROR, "Automation Service plugin is disabled"); + throw new CloudRuntimeException("Automation Service plugin is disabled"); } final AutomationControllerVO automationController = automationControllerDao.findById(automationControllerId); if (automationController == null) { @@ -467,10 +510,14 @@ public boolean startAutomationController(long automationControllerId, boolean on } return true; } - final AutomationControllerVersion AutomationControllerVersion = automationControllerVersionDao.findById(automationController.getAutomationTemplateId()); - final DataCenter zone = dataCenterDao.findById(AutomationControllerVersion.getZoneId()); - if (zone == null) { -// logAndThrow(Level.WARN, String.format("Unable to find zone for Automation Controller : %s", automationController.getName())); + if (onCreate && !AutomationController.State.Created.equals(automationController.getState())) { + throw new InvalidParameterValueException(String.format("Automation Controller %s cannot be created from state %s", + automationController.getName(), automationController.getState())); + } + if (!onCreate && !(AutomationController.State.Stopped.equals(automationController.getState()) + || AutomationController.State.Alert.equals(automationController.getState()))) { + throw new InvalidParameterValueException(String.format("Automation Controller %s cannot be started from state %s", + automationController.getName(), automationController.getState())); } AutomationControllerStartWorker startWorker = new AutomationControllerStartWorker(automationController, this); @@ -487,7 +534,7 @@ public boolean startAutomationController(long automationControllerId, boolean on @Override public boolean deleteAutomationController(Long automationControllerId) throws CloudRuntimeException { if (!AutomationServiceEnabled.value()) { -// logAndThrow(Level.ERROR, "Automation Service plugin is disabled"); + throw new CloudRuntimeException("Automation Service plugin is disabled"); } AutomationControllerVO cluster = automationControllerDao.findById(automationControllerId); if (cluster == null) { @@ -524,6 +571,11 @@ public boolean stopAutomationController(long automationControllerId) throws Clou } return true; } + if (!(AutomationController.State.Running.equals(automationController.getState()) + || AutomationController.State.Alert.equals(automationController.getState()))) { + throw new InvalidParameterValueException(String.format("Automation Controller %s cannot be stopped from state %s", + automationController.getName(), automationController.getState())); + } AutomationControllerStopWorker stopWorker = new AutomationControllerStopWorker(automationController, this); stopWorker = ComponentContext.inject(stopWorker); return stopWorker.stop(); diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerVO.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerVO.java index ddf09b461bcd..bea3301726a5 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerVO.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerVO.java @@ -218,9 +218,8 @@ public boolean isDisplay() { // public boolean isCheckForGc() { // return checkForGc; // } - public AutomationControllerVO(long id, String name, String description, Long automationTemplateId, Long zoneId, Long serviceOfferingId, long networkId, String networkName, long accountId, long domainId, State state, String automationControllerIp) { + public AutomationControllerVO(String name, String description, Long automationTemplateId, Long zoneId, Long serviceOfferingId, long networkId, String networkName, long accountId, long domainId, State state, String automationControllerIp) { this.uuid = UUID.randomUUID().toString(); - this.id = id; this.name = name; this.description = description; this.automationTemplateId = automationTemplateId; diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerActionWorker.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerActionWorker.java index c473e67c5236..cf168aad4284 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerActionWorker.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerActionWorker.java @@ -89,22 +89,16 @@ import javax.inject.Inject; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; public class AutomationControllerActionWorker { - public static final int CLUSTER_USER_PORTAL_PORT = 8080; - public static final int CLUSTER_ADMIN_PORTAL_PORT = 8081; - public static final int CLUSTER_API_PORT = 8082; - public static final int CLUSTER_SAMBA_PORT = 9017; - public static final Integer AUTOMATION_CONTROLLER_PORT = null; - protected Logger logger = LogManager.getLogger(getClass()); protected StateMachine2 _stateMachine = AutomationController.State.getStateMachine(); @@ -195,7 +189,14 @@ protected void init() { } protected String readResourceFile(String resource) throws IOException { - return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), StringUtils.getPreferredCharset()); + String normalizedResource = resource.startsWith("/") ? resource.substring(1) : resource; + InputStream resourceStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(normalizedResource); + if (resourceStream == null) { + throw new IOException(String.format("Automation controller resource not found: %s", resource)); + } + try (InputStream inputStream = resourceStream) { + return IOUtils.toString(inputStream, StringUtils.getPreferredCharset()); + } } protected void logMessage(final Level logLevel, final String message, final Exception e) { @@ -267,7 +268,7 @@ public AutomationControllerVmMapVO doInTransaction(TransactionStatus status) { protected List getControlVMMaps() { List automationControllerVMs = automationControllerVmMapDao.listByAutomationControllerId(automationController.getId()); if (!CollectionUtils.isEmpty(automationControllerVMs)) { - automationControllerVMs.sort((t1, t2) -> (int)((t1.getId() - t2.getId())/Math.abs(t1.getId() - t2.getId()))); + automationControllerVMs.sort((t1, t2) -> Long.compare(t1.getId(), t2.getId())); } return automationControllerVMs; } @@ -285,6 +286,11 @@ protected List getAutomationControllerVMs() { protected boolean stateTransitTo(long automationControllerId, AutomationController.Event e) { AutomationControllerVO automationController = automationControllerDao.findById(automationControllerId); + if (automationController == null) { + logger.warn(String.format("Failed to transition missing automation controller %d on event %s", + automationControllerId, e)); + return false; + } try { return _stateMachine.transitTo(automationController, e, null, automationControllerDao); } catch (NoTransitionException nte) { @@ -321,15 +327,9 @@ protected IpAddress getAutomationControllerServerIp() { protected void removeFirewallIngressRule(final IpAddress publicIp) { List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() != null && firewallRule.getSourcePortEnd() != null) { - if (firewallRule.getSourcePortStart() == CLUSTER_USER_PORTAL_PORT && - firewallRule.getSourcePortEnd() == CLUSTER_API_PORT && firewallRule.getTrafficType() == FirewallRule.TrafficType.Ingress) { - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - } - if (firewallRule.getSourcePortStart() == CLUSTER_SAMBA_PORT && - firewallRule.getSourcePortEnd() == CLUSTER_SAMBA_PORT && firewallRule.getTrafficType() == FirewallRule.TrafficType.Ingress) { - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - } + if (FirewallRule.TrafficType.Ingress.equals(firewallRule.getTrafficType()) + && isAutomationControllerFirewallRule(firewallRule)) { + firewallService.revokeIngressFwRule(firewallRule.getId(), true); } } } @@ -337,14 +337,23 @@ protected void removeFirewallIngressRule(final IpAddress publicIp) { protected void removeFirewallEgressRule(final Network network) { List firewallRules = firewallRulesDao.listByNetworkAndPurposeAndNotRevoked(network.getId(), FirewallRule.Purpose.Firewall); for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() != null && firewallRule.getSourcePortEnd() != null) { - if (firewallRule.getSourcePortStart() == CLUSTER_USER_PORTAL_PORT && firewallRule.getSourcePortEnd() == CLUSTER_ADMIN_PORTAL_PORT && firewallRule.getTrafficType() == FirewallRule.TrafficType.Egress) { - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - } + if (FirewallRule.TrafficType.Egress.equals(firewallRule.getTrafficType()) + && isAutomationControllerFirewallRule(firewallRule)) { + firewallService.revokeEgressFirewallRule(firewallRule.getId(), true); } } } + private boolean isAutomationControllerFirewallRule(FirewallRuleVO firewallRule) { + String protocol = firewallRule.getProtocol(); + if (!("tcp".equalsIgnoreCase(protocol) || "udp".equalsIgnoreCase(protocol) || "icmp".equalsIgnoreCase(protocol))) { + return false; + } + Integer startPort = firewallRule.getSourcePortStart(); + Integer endPort = firewallRule.getSourcePortEnd(); + return (startPort == null && endPort == null) || (Integer.valueOf(1).equals(startPort) && Integer.valueOf(65535).equals(endPort)); + } + protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List removedVMIds) throws ResourceUnavailableException { if (!CollectionUtils.isEmpty(removedVMIds)) { for (Long vmId : removedVMIds) { diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerDestroyWorker.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerDestroyWorker.java index 80c32f669b09..e3d53e5f8ebf 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerDestroyWorker.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerDestroyWorker.java @@ -19,48 +19,33 @@ import com.cloud.automation.controller.AutomationController; import com.cloud.automation.controller.AutomationControllerManagerImpl; -import com.cloud.automation.controller.AutomationControllerVO; import com.cloud.automation.controller.AutomationControllerVmMap; import com.cloud.automation.controller.AutomationControllerVmMapVO; import com.cloud.automation.resource.AutomationDeployedResourceVO; -import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; -import com.cloud.tags.dao.ResourceTagDao; -import com.cloud.user.AccountManager; import com.cloud.uservm.UserVm; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmVO; import org.apache.commons.collections.CollectionUtils; -import org.apache.logging.log4j.Level; -import javax.inject.Inject; import java.util.ArrayList; import java.util.List; public class AutomationControllerDestroyWorker extends AutomationControllerActionWorker { - @Inject - protected AccountManager accountManager; - @Inject - protected ResourceTagDao resourceTagDao; - - private List AutomationControllerVMs; + private List automationControllerVMs; public AutomationControllerDestroyWorker(final AutomationController automationController, final AutomationControllerManagerImpl clusterManager) { super(automationController, clusterManager); } private void validateControllerState() { - if (!(automationController.getState().equals(AutomationController.State.Running) - || automationController.getState().equals(AutomationController.State.Stopped) - || automationController.getState().equals(AutomationController.State.Alert) - || automationController.getState().equals(AutomationController.State.Error) - || automationController.getState().equals(AutomationController.State.Destroying))) { + if (AutomationController.State.Enabled.equals(automationController.getState())) { String msg = String.format("Cannot perform delete operation on controller : %s in state: %s", automationController.getName(), automationController.getState()); logger.warn(msg); @@ -82,39 +67,37 @@ private void validateDeployedPackages() { } private boolean destroyAutomationControllerVMs() { - boolean vmDestroyed = true; - //ControlVM removed - if (!CollectionUtils.isEmpty(AutomationControllerVMs)) { - for (AutomationControllerVmMapVO AutomationControllerVM : AutomationControllerVMs) { - long vmID = AutomationControllerVM.getVmId(); + boolean allVmsDestroyed = true; + if (!CollectionUtils.isEmpty(automationControllerVMs)) { + for (AutomationControllerVmMapVO automationControllerVM : automationControllerVMs) { + long vmID = automationControllerVM.getVmId(); - // delete only if VM exists and is not removed UserVmVO userVM = userVmDao.findById(vmID); if (userVM == null || userVM.isRemoved()) { + automationControllerVmMapDao.expunge(automationControllerVM.getId()); continue; } try { UserVm vm = userVmService.destroyVm(vmID, true); if (!userVmManager.expunge(userVM)) { - logger.warn(String.format("Unable to expunge VM %s : %s, destroying automation controller will probably fail", - vm.getInstanceName() , vm.getUuid())); + logger.warn(String.format("Unable to expunge VM %s while destroying automation controller %s", + userVM.getUuid(), automationController.getName())); + allVmsDestroyed = false; + continue; } - automationControllerVmMapDao.expunge(AutomationControllerVM.getId()); + automationControllerVmMapDao.expunge(automationControllerVM.getId()); if (logger.isInfoEnabled()) { - logger.info(String.format("Destroyed VM : %s as part of automation controller : %s cleanup", vm.getDisplayName(), automationController.getName())); + String vmName = vm == null ? userVM.getDisplayName() : vm.getDisplayName(); + logger.info(String.format("Destroyed VM : %s as part of automation controller : %s cleanup", vmName, automationController.getName())); } - } catch (ResourceUnavailableException | ConcurrentOperationException e) { - logger.warn(String.format("Failed to destroy VM : %s part of the automation controller : %s cleanup. Moving on with destroying remaining resources provisioned for the automation controller", userVM.getDisplayName(), automationController.getName()), e); - return false; + } catch (ResourceUnavailableException | CloudRuntimeException e) { + logger.warn(String.format("Failed to destroy VM : %s as part of automation controller : %s cleanup", + userVM.getDisplayName(), automationController.getName()), e); + allVmsDestroyed = false; } } } - return vmDestroyed; - } - - private boolean updateAutomationControllerEntryForGC() { - AutomationControllerVO automationControllerVO = automationControllerDao.findById(automationController.getId()); - return automationControllerDao.update(automationController.getId(), automationControllerVO); + return allVmsDestroyed; } @@ -124,46 +107,52 @@ private void deleteAutomationControllerNetworkRules() throws ManagementServerExc return; } List removedVmIds = new ArrayList<>(); - if (!CollectionUtils.isEmpty(AutomationControllerVMs)) { - for (AutomationControllerVmMapVO AutomationControllerVM : AutomationControllerVMs) { - removedVmIds.add(AutomationControllerVM.getVmId()); + if (!CollectionUtils.isEmpty(automationControllerVMs)) { + for (AutomationControllerVmMapVO automationControllerVM : automationControllerVMs) { + removedVmIds.add(automationControllerVM.getVmId()); } } IpAddress publicIp = getSourceNatIp(network); if (publicIp == null) { - throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s", network.getName())); + logger.warn(String.format("Source NAT IP for network %s is already absent; skipping automation controller network-rule cleanup", + network.getName())); + return; } removeFirewallIngressRule(publicIp); removeFirewallEgressRule(network); try { removePortForwardingRules(publicIp, network, owner, removedVmIds); } catch (ResourceUnavailableException e) { - // throw new ManagementServerException(String.format("Failed to automation controller port forwarding rules for network : %s", network.getName())); + throw new ManagementServerException(String.format("Failed to remove automation controller port forwarding rules for network : %s", network.getName()), e); } } - private void validateClusterVMsDestroyed() { - if(AutomationControllerVMs!=null && !AutomationControllerVMs.isEmpty()) { // Wait for few seconds to get all VMs really expunged + private boolean validateControllerVMsDestroyed() { + if (automationControllerVMs != null && !automationControllerVMs.isEmpty()) { final int maxRetries = 3; int retryCounter = 0; while (retryCounter < maxRetries) { boolean allVMsRemoved = true; - for (AutomationControllerVmMap AutomationControllerVM : AutomationControllerVMs) { - UserVmVO userVM = userVmDao.findById(AutomationControllerVM.getVmId()); + for (AutomationControllerVmMap automationControllerVM : automationControllerVMs) { + UserVmVO userVM = userVmDao.findById(automationControllerVM.getVmId()); if (userVM != null && !userVM.isRemoved()) { allVMsRemoved = false; break; } } if (allVMsRemoved) { - break; + return true; } try { Thread.sleep(10000); - } catch (InterruptedException ie) {} + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } retryCounter++; } } + return CollectionUtils.isEmpty(automationControllerVMs); } private void checkForRulesToDelete() throws ManagementServerException { @@ -177,36 +166,36 @@ public boolean destroy() throws CloudRuntimeException { init(); validateControllerState(); validateDeployedPackages(); - this.AutomationControllerVMs = automationControllerVmMapDao.listByAutomationControllerId(automationController.getId()); + this.automationControllerVMs = automationControllerVmMapDao.listByAutomationControllerId(automationController.getId()); if (logger.isInfoEnabled()) { logger.info(String.format("Destroying automation controller : %s", automationController.getName())); } - stateTransitTo(automationController.getId(), AutomationController.Event.DestroyRequested); + if (!AutomationController.State.Destroyed.equals(automationController.getState()) + && !stateTransitTo(automationController.getId(), AutomationController.Event.DestroyRequested)) { + throw new CloudRuntimeException(String.format("Failed to move automation controller %s into Destroying state", + automationController.getName())); + } boolean vmsDestroyed = destroyAutomationControllerVMs(); - // if there are VM's that were not expunged, we can not delete the network - if (vmsDestroyed) { - validateClusterVMsDestroyed(); - try { - checkForRulesToDelete(); - } catch (ManagementServerException e) { - String msg = String.format("Failed to remove network rules of automation controller : %s", automationController.getName()); - logger.warn(msg, e); - updateAutomationControllerEntryForGC(); - throw new CloudRuntimeException(msg, e); - } - } else { + if (!vmsDestroyed || !validateControllerVMsDestroyed()) { String msg = String.format("Failed to destroy one or more VMs as part of automation controller : %s cleanup",automationController.getName()); logger.warn(msg); - updateAutomationControllerEntryForGC(); throw new CloudRuntimeException(msg); } - stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded); - final String accessType = "internal"; + try { + checkForRulesToDelete(); + } catch (ManagementServerException | CloudRuntimeException e) { + logger.warn(String.format("Failed to remove one or more network rules of automation controller %s; continuing controller cleanup", + automationController.getName()), e); + } + if (!AutomationController.State.Destroyed.equals(automationController.getState()) + && !stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded)) { + throw new CloudRuntimeException(String.format("Failed to mark automation controller %s as Destroyed", + automationController.getName())); + } boolean deleted = automationControllerDao.remove(automationController.getId()); if (!deleted) { - logMessage(Level.WARN, String.format("Failed to delete automation controller : %s", automationController.getName()), null); - updateAutomationControllerEntryForGC(); - return false; + throw new CloudRuntimeException(String.format("Failed to delete automation controller : %s. The delete operation can be retried.", + automationController.getName())); } if (logger.isInfoEnabled()) { logger.info(String.format("Automation Controller : %s is successfully deleted", automationController.getName())); diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerResourceModifierActionWorker.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerResourceModifierActionWorker.java index d27a7083bd70..758fd422c33e 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerResourceModifierActionWorker.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerResourceModifierActionWorker.java @@ -44,8 +44,6 @@ import com.cloud.network.firewall.FirewallService; import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.network.rules.FirewallRule; -import com.cloud.network.rules.FirewallRule.TrafficType; -import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -193,7 +191,7 @@ protected DeployDestination plan() throws InsufficientServerCapacityException { if (logger.isDebugEnabled()) { logger.debug(String.format("Checking deployment destination for automation controller : %s in zone : %s", automationController.getName(), zone.getName())); } - final long dest = 2; + final long dest = 1; return plan(dest, zone, offering); } @@ -232,33 +230,6 @@ protected IpAddress getSourceNatIp(Network network) { return null; } - protected void removeFirewallIngressRule(final IpAddress publicIp) { - List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() != null && firewallRule.getSourcePortEnd() != null) { - if (firewallRule.getSourcePortStart() == CLUSTER_USER_PORTAL_PORT && - firewallRule.getSourcePortEnd() == CLUSTER_API_PORT && firewallRule.getTrafficType() == TrafficType.Ingress) { - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - } - if (firewallRule.getSourcePortStart() == CLUSTER_SAMBA_PORT && - firewallRule.getSourcePortEnd() == CLUSTER_SAMBA_PORT && firewallRule.getTrafficType() == TrafficType.Ingress) { - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - } - } - } - } - - protected void removeFirewallEgressRule(final Network network) { - List firewallRules = firewallRulesDao.listByNetworkAndPurposeAndNotRevoked(network.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() != null && firewallRule.getSourcePortEnd() != null) { - if (firewallRule.getSourcePortStart() == CLUSTER_USER_PORTAL_PORT && firewallRule.getSourcePortEnd() == CLUSTER_ADMIN_PORTAL_PORT && firewallRule.getTrafficType() == TrafficType.Egress) { - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - } - } - } - } - protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List removedVMIds) throws ResourceUnavailableException { if (!CollectionUtils.isEmpty(removedVMIds)) { for (Long vmId : removedVMIds) { @@ -309,7 +280,7 @@ protected boolean provisionFirewallRules(final IpAddress publicIp, final Account return sccuess; } - protected boolean provisionEgressFirewallRules(final Network network, final Account account, Integer startPort, Integer endPort) throws NoSuchFieldException, + protected boolean provisionEgressFirewallRules(final Network network, final Account account, String protocol, Integer startPort, Integer endPort) throws NoSuchFieldException, IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { List sourceCidrList = new ArrayList(); sourceCidrList.add("0.0.0.0/0"); @@ -327,7 +298,7 @@ protected boolean provisionEgressFirewallRules(final Network network, final Acco Field protocolField = rule.getClass().getDeclaredField("protocol"); protocolField.setAccessible(true); - protocolField.set(rule, "TCP"); + protocolField.set(rule, protocol); Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); startPortField.setAccessible(true); diff --git a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerStartWorker.java b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerStartWorker.java index 895bda3a7cc5..128f6c3ac9ac 100644 --- a/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerStartWorker.java +++ b/plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerStartWorker.java @@ -29,14 +29,11 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Objects; import java.util.List; import java.util.Properties; -import java.net.InetAddress; import java.util.concurrent.TimeUnit; import com.cloud.api.query.vo.UserAccountJoinVO; -import com.cloud.automation.version.AutomationControllerVersion; import com.cloud.dc.DataCenter; import com.cloud.deploy.DeployDestination; import com.cloud.exception.ConcurrentOperationException; @@ -64,7 +61,6 @@ import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.Level; import com.cloud.uservm.UserVm; @@ -78,7 +74,6 @@ public class AutomationControllerStartWorker extends AutomationControllerResourceModifierActionWorker { - private AutomationControllerVersion automationControllerVersion; private static final long GiB_TO_BYTES = 1024 * 1024 * 1024; private static final int AUTOMATION_CONTROLLER_HTTP_PORT = 80; private static final int HTTP_CONNECT_TIMEOUT_MS = 5000; @@ -86,22 +81,21 @@ public class AutomationControllerStartWorker extends AutomationControllerResourc private static final long AUTOMATION_CONTROLLER_CREATE_READY_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(30); private static final long AUTOMATION_CONTROLLER_READY_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(15); private static final long AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS = TimeUnit.SECONDS.toMillis(5); + private static final int AUTOMATION_CONTROLLER_READY_SUCCESS_THRESHOLD = 3; public AutomationControllerStartWorker(final AutomationController automationController, final AutomationControllerManagerImpl automationManager) { super(automationController, automationManager); } - @Override - protected String readResourceFile(String resource) throws IOException { - return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), StringUtils.getPreferredCharset()); - } - private void startAutomationControllerVMs() { List automationVms = getAutomationControllerVMs(); for (final UserVm vm : automationVms) { if (vm == null) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Control VMs in automation controller : %s", automationController.getName()), automationController.getId(), AutomationController.Event.OperationFailed); } + if (VirtualMachine.State.Running.equals(vm.getState())) { + continue; + } try { startAutomationVM(vm); } catch (ManagementServerException ex) { @@ -216,6 +210,9 @@ private String[] getServiceUserKeys(Account owner) { } String username = owner.getAccountName(); UserAccount user = accountService.getActiveUserAccount(username, owner.getDomainId()); + if (user == null) { + throw new CloudRuntimeException(String.format("Unable to find an active API user for automation controller account %s", username)); + } String[] keys = null; String apiKey = user.getApiKey(); String secretKey = user.getSecretKey(); @@ -224,6 +221,9 @@ private String[] getServiceUserKeys(Account owner) { } else { keys = new String[]{apiKey, secretKey}; } + if (keys == null || keys.length < 2 || keys[0] == null || keys[1] == null) { + throw new CloudRuntimeException(String.format("Unable to prepare API keys for automation controller account %s", username)); + } return keys; } @@ -233,21 +233,26 @@ private String[] getServerProperties() { final String HTTPS_ENABLE = "https.enable"; final String HTTPS_PORT = "https.port"; final File confFile = PropertiesUtil.findConfigFile("server.properties"); - try { - InputStream is = new FileInputStream(confFile); + if (confFile == null) { + throw new CloudRuntimeException("Unable to locate server.properties while preparing Genie cloud-init data"); + } + try (InputStream is = new FileInputStream(confFile)) { String port = null; String protocol = null; final Properties properties = ServerProperties.getServerProperties(is); - if (properties.getProperty(HTTPS_ENABLE).equals("true")){ + if (Boolean.parseBoolean(properties.getProperty(HTTPS_ENABLE, "false"))){ port = properties.getProperty(HTTPS_PORT); protocol = "https://"; } else { port = properties.getProperty(HTTP_PORT); protocol = "http://"; } + if (port == null || port.trim().isEmpty()) { + throw new CloudRuntimeException("Management server API port is not configured in server.properties"); + } serverInfo = new String[]{port, protocol}; } catch (final IOException e) { - logger.warn("Failed to read configuration from server.properties file", e); + throw new CloudRuntimeException("Failed to read configuration from server.properties file", e); } return serverInfo; } @@ -257,7 +262,19 @@ private String getAutomationControllerConfig(final DataCenter zone) throws IOExc String[] info = getServerProperties(); String automationControllerConfig = readResourceFile("/conf/genie"); NetworkVO ntwk = networkDao.findByIdIncludingRemoved(automationController.getNetworkId()); - final String managementIp = ApiServiceConfiguration.ManagementServerAddresses.value(); + if (ntwk == null) { + throw new CloudRuntimeException(String.format("Automation controller network %d was not found", + automationController.getNetworkId())); + } + if (zone == null) { + throw new CloudRuntimeException(String.format("Automation controller zone %d was not found", + automationController.getZoneId())); + } + final String configuredManagementAddresses = ApiServiceConfiguration.ManagementServerAddresses.value(); + if (configuredManagementAddresses == null || configuredManagementAddresses.trim().isEmpty()) { + throw new CloudRuntimeException("Management server address is not configured"); + } + final String managementIp = configuredManagementAddresses.split(",")[0].trim(); final String automationControllerId = "{{ automation_controller_id }}"; final String automationControllerName = "{{ automation_controller_instance_name }}"; final String acPublicIp = "{{ ac_public_ip }}"; @@ -277,11 +294,17 @@ private String getAutomationControllerConfig(final DataCenter zone) throws IOExc automationControllerConfig = automationControllerConfig.replace(automationControllerName, automationController.getName()+"-genie"); if (ntwk.getGuestType() == Network.GuestType.Isolated) { List ipAddresses = ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true); - if (ipAddresses != null && ipAddresses.size() == 1) { - automationControllerConfig = automationControllerConfig.replace(acPublicIp, ipAddresses.get(0).getAddress().addr()); + IPAddressVO sourceNatIp = findSourceNatIp(ipAddresses); + if (sourceNatIp == null) { + throw new CloudRuntimeException(String.format("Source NAT IP was not found for automation controller network %s", ntwk.getName())); } + automationControllerConfig = automationControllerConfig.replace(acPublicIp, sourceNatIp.getAddress().addr()); } List domain = userAccountJoinDao.searchByAccountId(owner.getId()); + if (domain == null || domain.isEmpty()) { + throw new CloudRuntimeException(String.format("Unable to resolve domain information for automation controller account %s", + owner.getAccountName())); + } automationControllerConfig = automationControllerConfig.replace(zoneUuid, zone.getUuid()); automationControllerConfig = automationControllerConfig.replace(zoneName, zone.getName()); automationControllerConfig = automationControllerConfig.replace(networkId, Long.toString(automationController.getNetworkId())); @@ -305,60 +328,29 @@ private UserVm provisionAutomationControllerVm(final Network network) throws InsufficientCapacityException, ManagementServerException, ResourceUnavailableException { UserVm genieControlVms = null; genieControlVms = createAutomationControllerVM(network); - addAutomationControllerVm(automationController.getId(), genieControlVms.getId()); - startAutomationVM(genieControlVms); - genieControlVms = userVmDao.findById(genieControlVms.getId()); if (genieControlVms == null) { throw new ManagementServerException(String.format("Failed to provision VM for automation controller : %s" , automationController.getName())); } + addAutomationControllerVm(automationController.getId(), genieControlVms.getId()); if (logger.isInfoEnabled()) { - logger.info(String.format("Provisioned Genie Automation Control VM : %s in to the automation controller : %s", genieControlVms.getDisplayName(), automationController.getName())); + logger.info(String.format("Created Genie Automation Control VM : %s in the automation controller : %s", genieControlVms.getDisplayName(), automationController.getName())); } return genieControlVms; } - private boolean setupAutomationControllerNetworkRules(Network network, UserVm genieVm, IpAddress publicIp) throws ManagementServerException { -// boolean egress = false; -// boolean firewall = false; -// boolean firewall2 = false; -// boolean portForwarding = false; - // Firewall Egress Network + private void setupAutomationControllerNetworkRules(Network network) throws ManagementServerException { try { - provisionEgressFirewallRules(network, owner, AUTOMATION_CONTROLLER_PORT, AUTOMATION_CONTROLLER_PORT); -// if (logger.isInfoEnabled()) { -// logger.info(String.format("Provisioned egress firewall rule to open up port %d to %d on %s for Automation controller : %s", publicIp.getAddress(), automationController.getName())); -// } + boolean tcpEgressReady = provisionEgressFirewallRules(network, owner, "TCP", null, null); + boolean udpEgressReady = provisionEgressFirewallRules(network, owner, "UDP", null, null); + if (!tcpEgressReady || !udpEgressReady) { + throw new ManagementServerException(String.format("Unable to apply bootstrap egress rules for automation controller : %s", + automationController.getName())); + } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to provision egress firewall rules for Web access for the Automation controller : %s", automationController.getName()), e); - } -// // Firewall rule for Web access on GenieVM -// if (egress) { -// try { -// firewall = provisionFirewallRules(publicIp, owner, AUTOMATION_CONTROLLER_PORT, AUTOMATION_CONTROLLER_PORT); -// if (logger.isInfoEnabled()) { -// logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Automation controller : %s", AUTOMATION_CONTROLLER_PORT, publicIp.getAddress().addr(), automationController.getName())); -// } -//// firewall2 = provisionFirewallRules(publicIp, owner, CLUSTER_SAMBA_PORT, CLUSTER_SAMBA_PORT); -//// if (logger.isInfoEnabled()) { -//// logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Automation controller : %s", publicIp.getAddress().addr(), automationController.getName())); -//// } -// } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { -// throw new ManagementServerException(String.format("Failed to provision firewall rules for Web access for the Automation controller : %s", automationController.getName()), e); -// } -// if (firewall) { -// // Port forwarding rule fo Web access on WorksVM -// try { -// portForwarding = provisionPortForwardingRules(publicIp, network, owner, genieVm, AUTOMATION_CONTROLLER_PORT); -// } catch (ResourceUnavailableException | NetworkRuleConflictException e) { -// throw new ManagementServerException(String.format("Failed to activate Web port forwarding rules for the Automation controller : %s", automationController.getName()), e); -// } -// if (portForwarding) { -// return true; -// } -// } -// } - return false; + throw new ManagementServerException(String.format("Failed to provision bootstrap egress rules for the Automation controller : %s", + automationController.getName()), e); + } } public boolean startAutomationControllerOnCreate() { @@ -366,7 +358,10 @@ public boolean startAutomationControllerOnCreate() { if (logger.isInfoEnabled()) { logger.info(String.format("Starting Automation Controller : %s", automationController.getName())); } - stateTransitTo(automationController.getId(), AutomationController.Event.StartRequested); + if (!stateTransitTo(automationController.getId(), AutomationController.Event.StartRequested)) { + throw new CloudRuntimeException(String.format("Cannot create automation controller %s from state %s", + automationController.getName(), automationController.getState())); + } DeployDestination dest = null; try { dest = plan(); @@ -384,72 +379,27 @@ public boolean startAutomationControllerOnCreate() { if (publicIpAddress == null) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Automation Controller : %s as no public IP found for the Automation Controller" , automationController.getName()), automationController.getId(), AutomationController.Event.CreateFailed); } - List automationControllerVMs = new ArrayList<>(); UserVm genieVM = null; try { genieVM = provisionAutomationControllerVm(network); - } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the Automation Controller VM failed in the automation controller : %s, %s", automationController.getName(), e), automationController.getId(), AutomationController.Event.CreateFailed, e); } - if (genieVM.getState().equals(VirtualMachine.State.Running)) { - try { - setupAutomationControllerNetworkRules(network, genieVM, publicIpAddress); - } catch (ManagementServerException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Automation Controller : %s, unable to setup network rules", automationController.getName()), automationController.getId(), AutomationController.Event.CreateFailed, e); - } + try { + setupAutomationControllerNetworkRules(network); + startAutomationVM(genieVM); + genieVM = userVmDao.findById(genieVM.getId()); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Automation Controller : %s after preparing bootstrap network rules", + automationController.getName()), automationController.getId(), AutomationController.Event.CreateFailed, e); + } + if (genieVM != null && genieVM.getState().equals(VirtualMachine.State.Running)) { if (logger.isInfoEnabled()) { logger.info(String.format("automation controller : %s automation controller VMs successfully provisioned", automationController.getName())); } String publicIpAddressStr = String.valueOf(publicIpAddress.getAddress()); - try { - pingCheck(publicIpAddressStr, 450000); - } catch (Exception e) { - throw new RuntimeException(e); - } - try { - if (waitForAutomationControllerReady(publicIpAddressStr, AUTOMATION_CONTROLLER_HTTP_PORT, - AUTOMATION_CONTROLLER_CREATE_READY_TIMEOUT_MS, AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS)) { - if (logger.isInfoEnabled()) { - logger.info(String.format("Starting automation controller : %s", automationController.getName())); - } - stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded); - if (logger.isInfoEnabled()) { - logger.info(String.format("Automation Controller : %s successfully started", automationController.getName())); - } - return true; - }else { - if (logger.isInfoEnabled()) { - logger.info(String.format("Starting automation controller : %s", automationController.getName())); - } - stateTransitTo(automationController.getId(), AutomationController.Event.OperationFailed); - if (logger.isInfoEnabled()) { - logger.info(String.format("Automation Controller : %s unsuccessfully started", automationController.getName())); - } - return false; - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - return false; - } - - public boolean startStoppedAutomationController() throws CloudRuntimeException { - init(); - IpAddress publicIpAddress = null; - publicIpAddress = getAutomationControllerServerIp(); - String publicIpAddressStr = String.valueOf(publicIpAddress.getAddress()); - stateTransitTo(automationController.getId(), AutomationController.Event.StartRequested); - startAutomationControllerVMs(); - try { - pingCheck(publicIpAddressStr, 450000); - } catch (Exception e) { - throw new RuntimeException(e); - } - try { if (waitForAutomationControllerReady(publicIpAddressStr, AUTOMATION_CONTROLLER_HTTP_PORT, - AUTOMATION_CONTROLLER_READY_TIMEOUT_MS, AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS)) { + AUTOMATION_CONTROLLER_CREATE_READY_TIMEOUT_MS, AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS)) { if (logger.isInfoEnabled()) { logger.info(String.format("Starting automation controller : %s", automationController.getName())); } @@ -458,7 +408,7 @@ public boolean startStoppedAutomationController() throws CloudRuntimeException { logger.info(String.format("Automation Controller : %s successfully started", automationController.getName())); } return true; - }else { + } else { if (logger.isInfoEnabled()) { logger.info(String.format("Starting automation controller : %s", automationController.getName())); } @@ -468,22 +418,59 @@ public boolean startStoppedAutomationController() throws CloudRuntimeException { } return false; } - } catch (Exception e) { - throw new RuntimeException(e); } + stateTransitTo(automationController.getId(), AutomationController.Event.OperationFailed); + return false; } - public boolean pingCheck(String url, int timeout) throws Exception{ - InetAddress target = InetAddress.getByName(url); - return target.isReachable(timeout); + public boolean startStoppedAutomationController() throws CloudRuntimeException { + init(); + IpAddress publicIpAddress = getAutomationControllerServerIp(); + if (publicIpAddress == null) { + throw new CloudRuntimeException(String.format("No source NAT IP found for automation controller : %s", automationController.getName())); + } + String publicIpAddressStr = String.valueOf(publicIpAddress.getAddress()); + AutomationController.Event startEvent = AutomationController.State.Alert.equals(automationController.getState()) + ? AutomationController.Event.RecoveryRequested : AutomationController.Event.StartRequested; + if (!stateTransitTo(automationController.getId(), startEvent)) { + throw new CloudRuntimeException(String.format("Cannot start automation controller %s from state %s", + automationController.getName(), automationController.getState())); + } + startAutomationControllerVMs(); + if (waitForAutomationControllerReady(publicIpAddressStr, AUTOMATION_CONTROLLER_HTTP_PORT, + AUTOMATION_CONTROLLER_READY_TIMEOUT_MS, AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS)) { + if (logger.isInfoEnabled()) { + logger.info(String.format("Starting automation controller : %s", automationController.getName())); + } + stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded); + if (logger.isInfoEnabled()) { + logger.info(String.format("Automation Controller : %s successfully started", automationController.getName())); + } + return true; + } else { + if (logger.isInfoEnabled()) { + logger.info(String.format("Starting automation controller : %s", automationController.getName())); + } + stateTransitTo(automationController.getId(), AutomationController.Event.OperationFailed); + if (logger.isInfoEnabled()) { + logger.info(String.format("Automation Controller : %s unsuccessfully started", automationController.getName())); + } + return false; + } } private boolean waitForAutomationControllerReady(String address, int port, long timeoutMs, long retryIntervalMs) { long deadline = System.currentTimeMillis() + timeoutMs; + int consecutiveSuccesses = 0; while (System.currentTimeMillis() < deadline) { int responseCode = getHttpResponseCode(address, port); if (responseCode >= 200 && responseCode < 400) { - return true; + consecutiveSuccesses++; + if (consecutiveSuccesses >= AUTOMATION_CONTROLLER_READY_SUCCESS_THRESHOLD) { + return true; + } + } else { + consecutiveSuccesses = 0; } if (logger.isDebugEnabled()) { @@ -505,6 +492,18 @@ private boolean waitForAutomationControllerReady(String address, int port, long return false; } + private IPAddressVO findSourceNatIp(List ipAddresses) { + if (ipAddresses == null) { + return null; + } + for (IPAddressVO ipAddress : ipAddresses) { + if (ipAddress != null && ipAddress.isSourceNat()) { + return ipAddress; + } + } + return null; + } + private static int getHttpResponseCode(String address, int port) { HttpURLConnection connection = null; try { diff --git a/plugins/integrations/automation-service/src/main/resources/conf/genie.yml b/plugins/integrations/automation-service/src/main/resources/conf/genie.yml index a4107dca41c3..33333f12c01f 100644 --- a/plugins/integrations/automation-service/src/main/resources/conf/genie.yml +++ b/plugins/integrations/automation-service/src/main/resources/conf/genie.yml @@ -27,12 +27,31 @@ write_files: runcmd: - | + install -d -m 0755 /var/lib/genie + rm -f /var/lib/genie/bootstrap-complete /var/lib/genie/bootstrap-failed echo "[genie] downloading deploy_automation_controller.yml to /root" | tee -a /var/log/genie-bootstrap.log - wget --tries=10 --waitretry=5 https://raw.githubusercontent.com/ablecloud-team/ablestack-genie/master/genie-shell/automation_controller_template/deploy_automation_controller.yml -O /root/deploy_automation_controller.yml + wget --tries=10 --waitretry=5 --retry-connrefused --timeout=30 https://raw.githubusercontent.com/ablecloud-team/ablestack-genie/master/genie-shell/automation_controller_template/deploy_automation_controller.yml -O /root/deploy_automation_controller.yml.tmp status=$? echo "[genie] wget exit code: ${status}" | tee -a /var/log/genie-bootstrap.log + if [ ${status} -ne 0 ] || [ ! -s /root/deploy_automation_controller.yml.tmp ]; then + touch /var/lib/genie/bootstrap-failed + exit 1 + fi + bash -o pipefail -c 'ansible-playbook --syntax-check /root/deploy_automation_controller.yml.tmp 2>&1 | tee -a /var/log/genie-bootstrap.log' + status=$? + if [ ${status} -ne 0 ]; then + touch /var/lib/genie/bootstrap-failed + exit ${status} + fi + mv -f /root/deploy_automation_controller.yml.tmp /root/deploy_automation_controller.yml ls -l /root/deploy_automation_controller.yml | tee -a /var/log/genie-bootstrap.log - exit ${status} - | echo "[genie] running ansible-playbook /root/deploy_automation_controller.yml" | tee -a /var/log/genie-bootstrap.log - ansible-playbook /root/deploy_automation_controller.yml 2>&1 | tee -a /var/log/genie-bootstrap.log + bash -o pipefail -c 'ansible-playbook /root/deploy_automation_controller.yml 2>&1 | tee -a /var/log/genie-bootstrap.log' + status=$? + echo "[genie] ansible-playbook exit code: ${status}" | tee -a /var/log/genie-bootstrap.log + if [ ${status} -ne 0 ]; then + touch /var/lib/genie/bootstrap-failed + exit ${status} + fi + touch /var/lib/genie/bootstrap-complete diff --git a/plugins/integrations/automation-service/src/test/java/com/cloud/automation/controller/AutomationControllerStateTest.java b/plugins/integrations/automation-service/src/test/java/com/cloud/automation/controller/AutomationControllerStateTest.java new file mode 100644 index 000000000000..77d17731c4fa --- /dev/null +++ b/plugins/integrations/automation-service/src/test/java/com/cloud/automation/controller/AutomationControllerStateTest.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.automation.controller; + +import com.cloud.utils.fsm.NoTransitionException; +import org.junit.Assert; +import org.junit.Test; + +public class AutomationControllerStateTest { + + @Test + public void deleteCanBeRetriedFromTransientStates() throws NoTransitionException { + AutomationController.State[] states = { + AutomationController.State.Created, + AutomationController.State.Starting, + AutomationController.State.Stopping, + AutomationController.State.Scaling, + AutomationController.State.Upgrading, + AutomationController.State.Recovering, + AutomationController.State.Destroying + }; + + for (AutomationController.State state : states) { + Assert.assertEquals(AutomationController.State.Destroying, + AutomationController.State.getStateMachine().getNextState( + state, AutomationController.Event.DestroyRequested)); + } + } + + @Test + public void newControllerDoesNotReuseTemplateIdAsPrimaryKey() { + AutomationControllerVO controller = new AutomationControllerVO("genie", "controller", 7L, 1L, + 2L, 3L, "genie-network", 4L, 5L, AutomationController.State.Created, null); + + Assert.assertEquals(0L, controller.getId()); + Assert.assertEquals(7L, controller.getAutomationTemplateId()); + } +}