From 961fdd25306dac76eb3f6798ab5756e5fd23f70d Mon Sep 17 00:00:00 2001 From: davidjumani Date: Thu, 29 Jul 2021 16:24:35 +0530 Subject: [PATCH 1/5] kubernetes: Deploy kubernetes-provider when creating a cluster --- .../java/com/cloud/user/AccountService.java | 2 + .../cluster/KubernetesClusterManagerImpl.java | 49 +++++++- .../cluster/KubernetesClusterService.java | 1 + .../KubernetesClusterActionWorker.java | 118 ++++++++++++++++++ .../KubernetesClusterStartWorker.java | 7 ++ .../KubernetesClusterUpgradeWorker.java | 12 +- .../resources/conf/k8s-control-node-add.yml | 4 + .../main/resources/conf/k8s-control-node.yml | 4 + .../src/main/resources/conf/k8s-node.yml | 4 + .../resources/script/deploy-cloudstack-secret | 69 ++++++++++ .../src/main/resources/script/deploy-provider | 36 ++++++ .../resources/script/upgrade-kubernetes.sh | 4 + .../management/MockAccountManager.java | 6 + .../util/create-kubernetes-binaries-iso.sh | 8 ++ .../com/cloud/user/AccountManagerImpl.java | 4 + .../cloud/user/MockAccountManagerImpl.java | 5 + 16 files changed, 331 insertions(+), 2 deletions(-) create mode 100755 plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret create mode 100755 plugins/integrations/kubernetes-service/src/main/resources/script/deploy-provider diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 4e3733bb5a49..98b1618a8da8 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -121,4 +121,6 @@ UserAccount createUserAccount(String userName, String password, String firstName UserAccount getUserAccountById(Long userId); public Map getKeys(GetUserKeysCmd cmd); + + public Map getKeys(Long userId); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index a384a07d7b34..87d783e8da41 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.UUID; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -53,6 +54,7 @@ import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; @@ -134,7 +136,11 @@ import com.cloud.user.AccountManager; import com.cloud.user.AccountService; import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.UserVO; import com.cloud.user.dao.SSHKeyPairDao; +import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; @@ -198,6 +204,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected AccountManager accountManager; @Inject + protected UserDao userDao; + @Inject protected VMInstanceDao vmInstanceDao; @Inject protected UserVmJoinDao userVmJoinDao; @@ -644,7 +652,15 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes return response; } + private void validateEndpointUrl() { + String csUrl = ApiServiceConfiguration.ApiServletPath.value(); + if (csUrl == null || csUrl.contains("localhost")) { + throw new InvalidParameterValueException("Global setting endpointe.url has to be set to the Management Server's API end point"); + } + } + private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException { + validateEndpointUrl(); final String name = cmd.getName(); final Long zoneId = cmd.getZoneId(); final Long kubernetesVersionId = cmd.getKubernetesVersionId(); @@ -927,6 +943,8 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { // Validate parameters + validateEndpointUrl(); + final Long kubernetesClusterId = cmd.getId(); final Long upgradeVersionId = cmd.getKubernetesVersionId(); if (kubernetesClusterId == null || kubernetesClusterId < 1L) { @@ -1093,6 +1111,9 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate startWorker = ComponentContext.inject(startWorker); if (onCreate) { // Start for Kubernetes cluster in 'Created' state + Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); + String[] keys = getServiceUserKeys(owner); + startWorker.setKeys(keys); return startWorker.startKubernetesClusterOnCreate(); } else { // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started @@ -1100,6 +1121,29 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate } } + private String[] getServiceUserKeys(Account owner) { + if (owner == null) { + owner = CallContext.current().getCallingAccount(); + } + String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME; + UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId()); + String[] keys = null; + if (kubeadmin == null) { + User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(), + KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN)); + keys = accountService.createApiKeyAndSecretKey(kube.getId()); + } else { + String apiKey = kubeadmin.getApiKey(); + String secretKey = kubeadmin.getSecretKey(); + if (Strings.isNullOrEmpty(apiKey) || Strings.isNullOrEmpty(secretKey)) { + keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId()); + } else { + keys = new String[]{apiKey, secretKey}; + } + } + return keys; + } + @Override public boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRuntimeException { if (!KubernetesServiceEnabled.value()) { @@ -1240,9 +1284,12 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } validateKubernetesClusterUpgradeParameters(cmd); + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); + String[] keys = getServiceUserKeys(owner); KubernetesClusterUpgradeWorker upgradeWorker = new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()), - kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this); + kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this, keys); upgradeWorker = ComponentContext.inject(upgradeWorker); return upgradeWorker.upgradeCluster(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index db5ab91b3d11..07939ddb101a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -34,6 +34,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0"; static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2; static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048; + static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin"; static final ConfigKey KubernetesServiceEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.service.enabled", diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 5f663dff7849..e5a1b836fb35 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -17,7 +17,9 @@ package com.cloud.kubernetes.cluster.actionworkers; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -28,6 +30,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.commons.collections.CollectionUtils; @@ -50,6 +53,7 @@ import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; import com.cloud.network.NetworkModel; import com.cloud.network.dao.NetworkDao; import com.cloud.service.dao.ServiceOfferingDao; @@ -70,6 +74,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmService; import com.cloud.vm.dao.UserVmDao; import com.google.common.base.Strings; @@ -127,12 +132,21 @@ public class KubernetesClusterActionWorker { protected String publicIpAddress; protected int sshPort; + + protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret"; + protected final String deployProviderScriptFilename = "deploy-provider"; + protected File deploySecretsScriptFile; + protected File deployProviderScriptFile; + protected KubernetesClusterManagerImpl manager; + protected String[] keys; + protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { this.kubernetesCluster = kubernetesCluster; this.kubernetesClusterDao = clusterManager.kubernetesClusterDao; this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao; this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao; this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao; + this.manager = clusterManager; } protected void init() { @@ -380,4 +394,108 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve return false; } } + + protected boolean createCloudStackSecret(String[] keys) { + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + + try { + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo /opt/bin/deploy-cloudstack-secret -u '%s' -k '%s' -s '%s'", + ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]), + 10000, 10000, 60000); + return result.first(); + } catch (Exception e) { + String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName()); + LOGGER.warn(msg, e); + } + return false; + } + + protected File retrieveScriptFile(String filename) { + File file = null; + try { + String data = readResourceFile("/script/" + filename); + file = File.createTempFile(filename, ".sh"); + BufferedWriter writer = new BufferedWriter(new FileWriter(file)); + writer.write(data); + writer.close(); + } catch (IOException e) { + logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster %s, unable to prepare upgrade script %s", kubernetesCluster.getName(), filename), e); + } + return file; + } + + protected void retrieveScriptFiles() { + deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename); + deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename); + } + + protected void copyScripts(String nodeAddress, final int sshPort) { + try { + SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + "~/", deploySecretsScriptFile.getAbsolutePath(), "0755"); + SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + "~/", deployProviderScriptFile.getAbsolutePath(), "0755"); + String cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", deploySecretsScriptFile.getName(), deploySecretsScriptFilename); + SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + cmdStr, 10000, 10000, 10 * 60 * 1000); + cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", deployProviderScriptFile.getName(), deployProviderScriptFilename); + SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + cmdStr, 10000, 10000, 10 * 60 * 1000); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + protected boolean deployProvider() { + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it + if (network.getGuestType() != GuestType.Isolated) { + return true; + } + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + + try { + String command = "sudo /opt/bin/deploy-provider"; + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, command, 10000, 10000, 60000); + + // Maybe the file isn't present. Try and copy it + if (!result.first()) { + logMessage(Level.INFO, "Provider files missing. Adding them now", null); + retrieveScriptFiles(); + copyScripts(publicIpAddress, sshPort); + + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + + // If at first you don't succeed ... + result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + pkFile, null, command, 10000, 10000, 60000); + if (!result.first()) { + throw new CloudRuntimeException(result.second()); + } + } + return true; + } catch (Exception e) { + String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage()); + logAndThrow(Level.ERROR, msg); + return false; + } finally { + // Deploying the provider might fail but it can be deployed manually too, so no need to go to an alert state + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + } + + public void setKeys(String[] keys) { + this.keys = keys; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 9a30fdd82357..83bc13c154fc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -571,6 +571,13 @@ public boolean startKubernetesClusterOnCreate() { if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); } + retrieveScriptFiles(); + copyScripts(publicIpAddress, sshPort); + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); + } + deployProvider(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 86c5c8ed70bc..27646d5f123b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -50,9 +50,11 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster, final KubernetesSupportedVersion upgradeVersion, - final KubernetesClusterManagerImpl clusterManager) { + final KubernetesClusterManagerImpl clusterManager, + final String[] keys) { super(kubernetesCluster, clusterManager); this.upgradeVersion = upgradeVersion; + this.keys = keys; } private void retrieveUpgradeScriptFile() { @@ -110,6 +112,14 @@ private void upgradeKubernetesClusterNodes() { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } try { + int port = (sshPort == CLUSTER_NODES_DEFAULT_START_SSH_PORT) ? sshPort + i : sshPort; + retrieveScriptFiles(); + copyScripts(publicIpAddress, port); + if (!createCloudStackSecret(keys)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", + kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); + } + deployProvider(); result = runInstallScriptOnVM(vm, i); } catch (Exception e) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index a8650ac957b3..601df21454d4 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -125,6 +125,10 @@ write-files: done <<< "$output" setup_complete=true fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index c2cecc4a0994..44a78a346e1f 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -147,6 +147,10 @@ write-files: fi mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index f65cc9c82963..03ed7013a4ca 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -125,6 +125,10 @@ write-files: done <<< "$output" setup_complete=true fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret new file mode 100755 index 000000000000..a7d6f97e1993 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret @@ -0,0 +1,69 @@ + +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function usage() { + cat << USAGE +Usage: ./deploy-cloudstack-secret [OPTIONS]... +To deploy the keys needed for the cloudstack kubernetes provider. +Arguments: + -u, --url string ID of the cluster + -k, --key string API Key + -s, --secret string Secret Key +Other arguments: + -h, --help Display this help message and exit +Examples: + ./deploy-cloudstack-secret -u http://localhost:8080 -k abcd -s efgh +USAGE + exit 0 +} +API_URL="" +API_KEY="" +SECRET_KEY="" +while [ -n "$1" ]; do + case "$1" in + -h | --help) + usage + ;; + -u | --url) + API_URL=$2 + shift 2 + ;; + -k | --key) + API_KEY=$2 + shift 2 + ;; + -s | --secret) + SECRET_KEY=$2 + shift 2 + ;; + -*|*) + echo "ERROR: no such option $1. -h or --help for help" + exit 1 + ;; + esac +done +cat > /tmp/cloud-config < getKeys(GetUserKeysCmd cmd){ return null; } + + @Override + public Map getKeys(Long userId) { + return null; + } + @Override public void checkAccess(User user, ControlledEntity entity) throws PermissionDeniedException { diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index 67062bea4c41..241b45e8b33a 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -77,6 +77,11 @@ echo "Downloading dashboard config ${DASHBORAD_CONFIG_URL}" dashboard_conf_file="${working_dir}/dashboard.yaml" curl -sSL ${DASHBORAD_CONFIG_URL} -o ${dashboard_conf_file} +PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml" +echo "Downloading kubernetes cluster provider ${PROVIDER_URL}" +provider_conf_file="${working_dir}/provider.yaml" +curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} + echo "Fetching k8s docker images..." docker -v if [ $? -ne 0 ]; then @@ -102,6 +107,9 @@ do output=`printf "%s\n" ${output} ${images}` done +provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' '` +output=`printf "%s\n" ${output} ${provider_image}` + while read -r line; do echo "Downloading docker image $line ---" sudo docker pull "$line" diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index b6f4e5e36000..88cd217698a3 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -2431,7 +2431,11 @@ public Pair findUserByApiKey(String apiKey) { @Override public Map getKeys(GetUserKeysCmd cmd) { final long userId = cmd.getID(); + return getKeys(userId); + } + @Override + public Map getKeys(Long userId) { User user = getActiveUser(userId); if (user == null) { throw new InvalidParameterValueException("Unable to find user by id"); diff --git a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java index ea6287d2bcc1..7916007c4065 100644 --- a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java +++ b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java @@ -425,6 +425,11 @@ public Map getKeys(GetUserKeysCmd cmd) { return null; } + @Override + public Map getKeys(Long userId) { + return null; + } + @Override public void checkAccess(User user, ControlledEntity entity) throws PermissionDeniedException { From 98015bf6d6af3ad373d9269e873a240ffb2382d8 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Fri, 30 Jul 2021 13:52:05 +0530 Subject: [PATCH 2/5] Fix tests --- test/integration/smoke/test_kubernetes_clusters.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index d78f3a0afb3b..d4be6b8d3c67 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -75,8 +75,12 @@ def setUpClass(cls): cls.kubernetes_version_ids = [] if cls.hypervisorNotSupported == False: - cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, - name="cloud.kubernetes.service.enabled")[0].value + cls.endpoint_url = Configurations.list(cls.apiclient, name="endpointe.url")[0].value + if "localhost" in cls.endpoint_url: + endpoint_url = "http://%s:%d/client/api " %(cls.mgtSvrDetails["mgtSvrIp"], cls.mgtSvrDetails["port"]) + cls.debug("Setting endpointe.url to %s" %(endpoint_url)) + Configurations.update(cls.apiclient, "endpointe.url", endpoint_url) + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value if cls.initial_configuration_cks_enabled not in ["true", True]: cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") Configurations.update(cls.apiclient, From 1db057be797d71d48a892ae7baddf0d4b37e5633 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Fri, 30 Jul 2021 15:18:34 +0530 Subject: [PATCH 3/5] Addressing comments --- .../actionworkers/KubernetesClusterActionWorker.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index e5a1b836fb35..9023eafba5e8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -423,7 +423,8 @@ protected File retrieveScriptFile(String filename) { writer.write(data); writer.close(); } catch (IOException e) { - logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster %s, unable to prepare upgrade script %s", kubernetesCluster.getName(), filename), e); + logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to to fetch script %s", + kubernetesCluster.getName(), filename), e); } return file; } @@ -454,6 +455,8 @@ protected boolean deployProvider() { Network network = networkDao.findById(kubernetesCluster.getNetworkId()); // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it if (network.getGuestType() != GuestType.Isolated) { + logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isoalted network", + kubernetesCluster.getName()), null); return true; } File pkFile = getManagementServerSshPublicKeyFile(); @@ -489,9 +492,6 @@ protected boolean deployProvider() { String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage()); logAndThrow(Level.ERROR, msg); return false; - } finally { - // Deploying the provider might fail but it can be deployed manually too, so no need to go to an alert state - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } } From 6575a566fc240bd61ef38c054fc78a862dc3f170 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Mon, 2 Aug 2021 11:20:14 +0530 Subject: [PATCH 4/5] Update plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java Co-authored-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterActionWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 9023eafba5e8..2e19011e3b21 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -455,7 +455,7 @@ protected boolean deployProvider() { Network network = networkDao.findById(kubernetesCluster.getNetworkId()); // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it if (network.getGuestType() != GuestType.Isolated) { - logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isoalted network", + logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network", kubernetesCluster.getName()), null); return true; } From e4ed9c57e689f39bf001e6b0348ad87bc87459c4 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Thu, 5 Aug 2021 10:30:54 +0530 Subject: [PATCH 5/5] Addressing comments --- .../cluster/KubernetesClusterManagerImpl.java | 4 +++- .../KubernetesClusterActionWorker.java | 13 +++++++------ .../actionworkers/KubernetesClusterStartWorker.java | 6 ------ .../KubernetesClusterUpgradeWorker.java | 6 ------ .../main/resources/script/deploy-cloudstack-secret | 1 - 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 87d783e8da41..7e52d98bcc4b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -655,7 +655,9 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes private void validateEndpointUrl() { String csUrl = ApiServiceConfiguration.ApiServletPath.value(); if (csUrl == null || csUrl.contains("localhost")) { - throw new InvalidParameterValueException("Global setting endpointe.url has to be set to the Management Server's API end point"); + String error = String.format("Global setting %s has to be set to the Management Server's API end point", + ApiServiceConfiguration.ApiServletPath.key()); + throw new InvalidParameterValueException(error); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 2e19011e3b21..5426e9cd242e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -135,6 +135,7 @@ public class KubernetesClusterActionWorker { protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret"; protected final String deployProviderScriptFilename = "deploy-provider"; + protected final String scriptPath = "/opt/bin/"; protected File deploySecretsScriptFile; protected File deployProviderScriptFile; protected KubernetesClusterManagerImpl manager; @@ -402,10 +403,10 @@ protected boolean createCloudStackSecret(String[] keys) { sshPort = publicIpSshPort.second(); try { + final String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'", + scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo /opt/bin/deploy-cloudstack-secret -u '%s' -k '%s' -s '%s'", - ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]), - 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); return result.first(); } catch (Exception e) { String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName()); @@ -440,10 +441,10 @@ protected void copyScripts(String nodeAddress, final int sshPort) { "~/", deploySecretsScriptFile.getAbsolutePath(), "0755"); SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, "~/", deployProviderScriptFile.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", deploySecretsScriptFile.getName(), deploySecretsScriptFilename); + String cmdStr = String.format("sudo mv ~/%s %s/%s", deploySecretsScriptFile.getName(), scriptPath, deploySecretsScriptFilename); SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); - cmdStr = String.format("sudo mv ~/%s /opt/bin/%s", deployProviderScriptFile.getName(), deployProviderScriptFilename); + cmdStr = String.format("sudo mv ~/%s %s/%s", deployProviderScriptFile.getName(), scriptPath, deployProviderScriptFilename); SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { @@ -465,7 +466,7 @@ protected boolean deployProvider() { sshPort = publicIpSshPort.second(); try { - String command = "sudo /opt/bin/deploy-provider"; + String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename); Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, command, 10000, 10000, 60000); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 83bc13c154fc..072094e7c031 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -571,12 +571,6 @@ public boolean startKubernetesClusterOnCreate() { if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); } - retrieveScriptFiles(); - copyScripts(publicIpAddress, sshPort); - if (!createCloudStackSecret(keys)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", - kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); - } deployProvider(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 27646d5f123b..e8b61d4a26da 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -113,12 +113,6 @@ private void upgradeKubernetesClusterNodes() { } try { int port = (sshPort == CLUSTER_NODES_DEFAULT_START_SSH_PORT) ? sshPort + i : sshPort; - retrieveScriptFiles(); - copyScripts(publicIpAddress, port); - if (!createCloudStackSecret(keys)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", - kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); - } deployProvider(); result = runInstallScriptOnVM(vm, i); } catch (Exception e) { diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret index a7d6f97e1993..9356f8a03f14 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-cloudstack-secret @@ -1,4 +1,3 @@ - #!/bin/bash -e # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file