Merge remote-tracking branch 'apache/4.19' into 4.20

Author: shwstppr

api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java

@@ -196,6 +196,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement
     @Param(description = "true network requires restart")
     private Boolean restartRequired;
 
+    @SerializedName(ApiConstants.SPECIFY_VLAN)
+    @Param(description = "true if network supports specifying vlan, false otherwise")
+    private Boolean specifyVlan;
+
     @SerializedName(ApiConstants.SPECIFY_IP_RANGES)
     @Param(description = "true if network supports specifying ip ranges, false otherwise")
     private Boolean specifyIpRanges;
@@ -516,6 +520,10 @@ public void setRestartRequired(Boolean restartRequired) {
         this.restartRequired = restartRequired;
     }
 
+    public void setSpecifyVlan(Boolean specifyVlan) {
+        this.specifyVlan = specifyVlan;
+    }
+
     public void setSpecifyIpRanges(Boolean specifyIpRanges) {
         this.specifyIpRanges = specifyIpRanges;
     }

engine/schema/src/main/java/com/cloud/user/UserAccountVO.java

@@ -33,11 +33,11 @@
 import javax.persistence.Transient;
 
 import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.utils.db.Encrypt;
 import com.cloud.utils.db.GenericDao;
-import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
-import org.apache.commons.lang3.StringUtils;
 
 @Entity
 @Table(name = "user")
@@ -131,12 +131,6 @@ public enum Setup2FAstatus {
     public UserAccountVO() {
     }
 
-    @Override
-    public String toString() {
-        return String.format("UserAccount %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields
-                (this, "id", "uuid", "username", "accountName"));
-    }
-
     @Override
     public long getId() {
         return id;
@@ -379,4 +373,10 @@ public Map<String, String> getDetails() {
     public void setDetails(Map<String, String> details) {
         this.details = details;
     }
+
+    @Override
+    public String toString() {
+        return String.format("UserAccount %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields
+                (this, "id", "uuid", "username", "accountName"));
+    }
 }

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_update_api_permission.sql

@@ -0,0 +1,52 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION` (
+    IN role VARCHAR(255),
+    IN rule VARCHAR(255),
+    IN permission VARCHAR(255)
+)
+BEGIN
+     DECLARE role_id BIGINT(20) UNSIGNED
+;    DECLARE max_sort_order BIGINT(20) UNSIGNED
+
+;   SELECT `r`.`id` INTO role_id
+    FROM `cloud`.`roles` `r`
+    WHERE `r`.`name` = role
+        AND `r`.`is_default` = 1
+
+;   SELECT MAX(`rp`.`sort_order`) INTO max_sort_order
+    FROM `cloud`.`role_permissions` `rp`
+    WHERE `rp`.`role_id` = role_id
+
+;   IF NOT EXISTS (
+        SELECT * FROM `cloud`.`role_permissions` `rp`
+        WHERE `rp`.`role_id` = role_id
+            AND `rp`.`rule` = rule
+    ) THEN
+        UPDATE `cloud`.`role_permissions` `rp`
+        SET `rp`.`sort_order` = max_sort_order + 1
+        WHERE `rp`.`sort_order` = max_sort_order
+            AND `rp`.`role_id` = role_id
+
+;       INSERT INTO `cloud`.`role_permissions`
+            (uuid, role_id, rule, permission, sort_order)
+        VALUES (uuid(), role_id, rule, permission, max_sort_order)
+;   END IF
+;END;

engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql

@@ -21,3 +21,25 @@
 
 -- Add last_id to the volumes table
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) unsigned DEFAULT NULL');
+
+-- Grant access to 2FA APIs for the "Read-Only User - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'listUserTwoFactorAuthenticatorProviders', 'ALLOW');
+
+-- Grant access to 2FA APIs for the "Support User - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support User - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support User - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support User - Default', 'listUserTwoFactorAuthenticatorProviders', 'ALLOW');
+
+-- Grant access to 2FA APIs for the "Read-Only Admin - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+
+-- Grant access to 2FA APIs for the "Support Admin - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java

@@ -124,18 +124,24 @@ public List<StoragePool> allocateToPool(DiskProfile dskCh, VirtualMachineProfile
     protected List<StoragePool> reorderPoolsByCapacity(DeploymentPlan plan, List<StoragePool> pools) {
         Long zoneId = plan.getDataCenterId();
         Long clusterId = plan.getClusterId();
-        short capacityType;
 
         if (CollectionUtils.isEmpty(pools)) {
             return null;
         }
 
-        if (pools.get(0).getPoolType().isShared()) {
+        short capacityType = Capacity.CAPACITY_TYPE_LOCAL_STORAGE;
+        String storageType = "local";
+        StoragePool storagePool = pools.get(0);
+        if (storagePool.isShared()) {
             capacityType = Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED;
-        } else {
-            capacityType = Capacity.CAPACITY_TYPE_LOCAL_STORAGE;
+            storageType = "shared";
         }
 
+        logger.debug(String.format(
+                "Filtering storage pools by capacity type [%s] as the first storage pool of the list, with name [%s] and ID [%s], is a [%s] storage.",
+                capacityType, storagePool.getName(), storagePool.getUuid(), storageType
+        ));
+
         List<Long> poolIdsByCapacity = capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType);
 
         logger.debug(String.format("List of pools in descending order of available capacity [%s].", poolIdsByCapacity));
@@ -221,6 +227,8 @@ public List<StoragePool> reorderPools(List<StoragePool> pools, VirtualMachinePro
     }
 
     List<StoragePool> reorderStoragePoolsBasedOnAlgorithm(List<StoragePool> pools, DeploymentPlan plan, Account account) {
+        logger.debug(String.format("Using allocation algorithm [%s] to reorder pools.", allocationAlgorithm));
+
         if (allocationAlgorithm.equals("random") || allocationAlgorithm.equals("userconcentratedpod_random") || (account == null)) {
             reorderRandomPools(pools);
         } else if (StringUtils.equalsAny(allocationAlgorithm, "userdispersing", "firstfitleastconsumed")) {

packaging/el8/cloud.spec

@@ -71,7 +71,7 @@ Requires: (openssh-clients or openssh)
 Requires: (nfs-utils or nfs-client)
 Requires: iproute
 Requires: wget
-Requires: mysql
+Requires: (mysql or mariadb)
 Requires: sudo
 Requires: /sbin/service
 Requires: /sbin/chkconfig

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java

@@ -203,7 +203,7 @@ private boolean removeKubernetesClusterNode(final String ipAddress, final int po
             retryCounter++;
             try {
                 Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(),
-                        pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName),
+                        pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName),
                         10000, 10000, 60000);
                 if (!result.first()) {
                     logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster);

plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java

@@ -527,6 +527,10 @@ public void validateUserPasswordAndUpdateIfNeeded(String newPassword, UserVO use
 
     @Override
     public void checkApiAccess(Account account, String command) throws PermissionDeniedException {
+    }
 
+    @Override
+    public UserAccount clearUserTwoFactorAuthenticationInSetupStateOnLogin(UserAccount user) {
+        return null;
     }
 }

scripts/util/create-kubernetes-binaries-iso.sh

@@ -53,7 +53,7 @@ echo "Downloading Kubernetes tools ${RELEASE}..."
 k8s_dir="${working_dir}/k8s"
 mkdir -p "${k8s_dir}"
 cd "${k8s_dir}"
-curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
+curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
 kubeadm_file_permissions=`stat --format '%a' kubeadm`
 chmod +x kubeadm
 

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -2565,6 +2565,7 @@ public NetworkResponse createNetworkResponse(ResponseView view, Network network)
             response.setIsSystem(networkOffering.isSystemOnly());
             response.setNetworkOfferingAvailability(networkOffering.getAvailability().toString());
             response.setIsPersistent(networkOffering.isPersistent());
+            response.setSpecifyVlan(networkOffering.isSpecifyVlan());
             if (Network.GuestType.Isolated.equals(network.getGuestType()) && network.getVpcId() == null) {
                 response.setEgressDefaultPolicy(networkOffering.isEgressDefaultPolicy());
             }