Merge branch '4.19' into 4.20

Author: DaanHoogland

api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java

@@ -28,12 +28,11 @@
 import org.apache.cloudstack.api.EntityReference;
 
 import com.cloud.network.Network;
-import com.cloud.projects.ProjectAccount;
 import com.cloud.serializer.Param;
 import com.google.gson.annotations.SerializedName;
 
 @SuppressWarnings("unused")
-@EntityReference(value = {Network.class, ProjectAccount.class})
+@EntityReference(value = {Network.class})
 public class NetworkResponse extends BaseResponseWithAssociatedNetwork implements ControlledEntityResponse, SetResourceIconResponse {
 
     @SerializedName(ApiConstants.ID)

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java

@@ -22,23 +22,29 @@
 
 import javax.inject.Inject;
 
-import com.cloud.dc.dao.DataCenterDao;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.DeleteStoragePoolCommand;
+import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ResourceManager;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.StoragePoolHostVO;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.template.TemplateManager;
 import com.cloud.utils.Pair;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -59,6 +65,10 @@ public class BasePrimaryDataStoreLifeCycleImpl {
     protected DataCenterDao zoneDao;
     @Inject
     protected StoragePoolHostDao storagePoolHostDao;
+    @Inject
+    private PrimaryDataStoreDao primaryDataStoreDao;
+    @Inject
+    private TemplateManager templateMgr;
 
     private List<HostVO> getPoolHostsList(ClusterScope clusterScope, HypervisorType hypervisorType) {
         List<HostVO> hosts;
@@ -81,7 +91,7 @@ public void changeStoragePoolScopeToZone(DataStore store, ClusterScope clusterSc
                 try {
                     storageMgr.connectHostToSharedPool(host, store.getId());
                 } catch (Exception e) {
-                    logger.warn("Unable to establish a connection between " + host + " and " + store, e);
+                    logger.warn("Unable to establish a connection between {} and {}", host, store, e);
                 }
             }
         }
@@ -99,7 +109,7 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
 
                 if (answer != null) {
                     if (!answer.getResult()) {
-                        logger.debug("Failed to delete storage pool: " + answer.getResult());
+                        logger.debug("Failed to delete storage pool: {}", answer.getResult());
                     } else if (HypervisorType.KVM != hypervisorType) {
                         break;
                     }
@@ -108,4 +118,42 @@ public void changeStoragePoolScopeToCluster(DataStore store, ClusterScope cluste
         }
         dataStoreHelper.switchToCluster(store, clusterScope);
     }
+
+    private void evictTemplates(StoragePoolVO storagePoolVO) {
+        List<VMTemplateStoragePoolVO> unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO);
+        for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
+            if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
+                templateMgr.evictTemplateFromStoragePool(templatePoolVO);
+            }
+        }
+    }
+
+    private void deleteAgentStoragePools(StoragePool storagePool) {
+        List<StoragePoolHostVO> poolHostVOs = storagePoolHostDao.listByPoolId(storagePool.getId());
+        for (StoragePoolHostVO poolHostVO : poolHostVOs) {
+            DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool);
+            final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand);
+            if (answer != null && answer.getResult()) {
+                logger.info("Successfully deleted storage pool: {} from host: {}", storagePool.getId(), poolHostVO.getHostId());
+            } else {
+                if (answer != null) {
+                    logger.error("Failed to delete storage pool: {} from host: {} , result: {}", storagePool.getId(), poolHostVO.getHostId(), answer.getResult());
+                } else {
+                    logger.error("Failed to delete storage pool: {} from host: {}", storagePool.getId(), poolHostVO.getHostId());
+                }
+            }
+        }
+    }
+
+    protected boolean cleanupDatastore(DataStore store) {
+        StoragePool storagePool = (StoragePool)store;
+        StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId());
+        if (storagePoolVO == null) {
+            return false;
+        }
+
+        evictTemplates(storagePoolVO);
+        deleteAgentStoragePools(storagePool);
+        return true;
+    }
 }

plugins/storage/volume/linstor/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2025-03-13]
+
+### Fixed
+
+- Implemented missing delete datastore, to correctly cleanup on datastore removal
+
 ## [2025-02-21]
 
 ### Fixed

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java

@@ -286,7 +286,10 @@ public boolean cancelMaintain(DataStore store) {
 
     @Override
     public boolean deleteDataStore(DataStore store) {
-        return dataStoreHelper.deletePrimaryDataStore(store);
+        if (cleanupDatastore(store)) {
+            return dataStoreHelper.deletePrimaryDataStore(store);
+        }
+        return false;
     }
 
     /* (non-Javadoc)

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java

@@ -18,7 +18,6 @@
  */
 package org.apache.cloudstack.storage.datastore.lifecycle;
 
-import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLDecoder;
@@ -30,6 +29,7 @@
 
 import javax.inject.Inject;
 
+import com.cloud.utils.StringUtils;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
@@ -48,8 +48,6 @@
 import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.agent.AgentManager;
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.DeleteStoragePoolCommand;
 import com.cloud.agent.api.StoragePoolInfo;
 import com.cloud.capacity.CapacityManager;
 import com.cloud.dc.ClusterVO;
@@ -63,9 +61,6 @@
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.StoragePoolAutomation;
-import com.cloud.storage.StoragePoolHostVO;
-import com.cloud.storage.VMTemplateStoragePoolVO;
-import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.template.TemplateManager;
 import com.cloud.utils.UriUtils;
@@ -111,7 +106,7 @@ private org.apache.cloudstack.storage.datastore.api.StoragePool findStoragePool(
             List<org.apache.cloudstack.storage.datastore.api.StoragePool> storagePools = client.listStoragePools();
             for (org.apache.cloudstack.storage.datastore.api.StoragePool pool : storagePools) {
                 if (pool.getName().equals(storagePoolName)) {
-                    logger.info("Found PowerFlex storage pool: " + storagePoolName);
+                    logger.info("Found PowerFlex storage pool: {}", storagePoolName);
                     final org.apache.cloudstack.storage.datastore.api.StoragePoolStatistics poolStatistics = client.getStoragePoolStatistics(pool.getId());
                     pool.setStatistics(poolStatistics);
 
@@ -164,7 +159,7 @@ public DataStore initialize(Map<String, Object> dsInfos) {
             throw new CloudRuntimeException("Cluster Id must also be specified when the Pod Id is specified for Cluster-wide primary storage.");
         }
 
-        URI uri = null;
+        URI uri;
         try {
             uri = new URI(UriUtils.encodeURIComponent(url));
             if (uri.getScheme() == null || !uri.getScheme().equalsIgnoreCase("powerflex")) {
@@ -174,20 +169,16 @@ public DataStore initialize(Map<String, Object> dsInfos) {
             throw new InvalidParameterValueException(url + " is not a valid uri");
         }
 
-        String storagePoolName = null;
-        try {
-            storagePoolName = URLDecoder.decode(uri.getPath(), "UTF-8");
-        } catch (UnsupportedEncodingException e) {
-            logger.error("[ignored] we are on a platform not supporting \"UTF-8\"!?!", e);
-        }
+        String storagePoolName;
+        storagePoolName = URLDecoder.decode(uri.getPath(), StringUtils.getPreferredCharset());
         if (storagePoolName == null) { // if decoding fails, use getPath() anyway
             storagePoolName = uri.getPath();
         }
         storagePoolName = storagePoolName.replaceFirst("/", "");
 
         final String storageHost = uri.getHost();
         final int port = uri.getPort();
-        String gatewayApiURL = null;
+        String gatewayApiURL;
         if (port == -1) {
             gatewayApiURL = String.format("https://%s/api", storageHost);
         } else {
@@ -321,37 +312,11 @@ public void disableStoragePool(DataStore dataStore) {
 
     @Override
     public boolean deleteDataStore(DataStore dataStore) {
-        StoragePool storagePool = (StoragePool)dataStore;
-        StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId());
-        if (storagePoolVO == null) {
-            return false;
+        if (cleanupDatastore(dataStore)) {
+            ScaleIOGatewayClientConnectionPool.getInstance().removeClient(dataStore);
+            return dataStoreHelper.deletePrimaryDataStore(dataStore);
         }
-
-        List<VMTemplateStoragePoolVO> unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO);
-        for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
-            if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
-                templateMgr.evictTemplateFromStoragePool(templatePoolVO);
-            }
-        }
-
-        List<StoragePoolHostVO> poolHostVOs = storagePoolHostDao.listByPoolId(dataStore.getId());
-        for (StoragePoolHostVO poolHostVO : poolHostVOs) {
-            DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool);
-            final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand);
-            if (answer != null && answer.getResult()) {
-                logger.info("Successfully deleted storage pool: {} from host: {}", storagePool, poolHostVO.getHostId());
-            } else {
-                if (answer != null) {
-                    logger.error("Failed to delete storage pool: {} from host: {} , result: {}", storagePool, poolHostVO.getHostId(), answer.getResult());
-                } else {
-                    logger.error("Failed to delete storage pool: {} from host: {}", storagePool, poolHostVO.getHostId());
-                }
-            }
-        }
-
-        ScaleIOGatewayClientConnectionPool.getInstance().removeClient(dataStore);
-
-        return dataStoreHelper.deletePrimaryDataStore(dataStore);
+        return false;
     }
 
     @Override

scripts/storage/secondary/setup-sysvm-tmplt

@@ -99,7 +99,7 @@ if [[ -f $destdir/template.properties ]]; then
   failed 2 "Data already exists at destination $destdir"
 fi
 
-destfiles=$(find $destdir -name \*.$ext)
+destfiles=$(sudo find $destdir -name \*.$ext)
 if [[ "$destfiles" != "" ]]; then
   failed 2 "Data already exists at destination $destdir"
 fi
@@ -108,12 +108,12 @@ tmpfolder=/tmp/cloud/templates/
 mkdir -p $tmpfolder
 tmplfile=$tmpfolder/$localfile
 
-sudo touch $tmplfile
+touch $tmplfile
 if [[ $? -ne 0 ]]; then
   failed 2 "Failed to create temporary file in directory $tmpfolder -- is it read-only or full?\n"
 fi
 
-destcap=$(df -P $destdir | awk '{print $4}' | tail -1 )
+destcap=$(sudo df -P $destdir | awk '{print $4}' | tail -1 )
 [ $destcap -lt $DISKSPACE ] && echo "Insufficient free disk space for target folder $destdir: avail=${destcap}k req=${DISKSPACE}k" && failed 4
 
 localcap=$(df -P $tmpfolder | awk '{print $4}' | tail -1 )
@@ -146,9 +146,9 @@ fi
 
 
 tmpltfile=$destdir/$localfile
-tmpltsize=$(ls -l $tmpltfile | awk -F" " '{print $5}')
+tmpltsize=$(sudo ls -l $tmpltfile | awk -F" " '{print $5}')
 if [[ "$ext" == "qcow2" ]]; then
-  vrtmpltsize=$($qemuimgcmd info $tmpltfile | grep -i 'virtual size' | sed -ne 's/.*(\([0-9]*\).*/\1/p' | xargs)
+  vrtmpltsize=$(sudo $qemuimgcmd info $tmpltfile | grep -i 'virtual size' | sed -ne 's/.*(\([0-9]*\).*/\1/p' | xargs)
 else
   vrtmpltsize=$tmpltsize
 fi

server/conf/cloudstack-sudoers.in

@@ -18,7 +18,7 @@
 # The CloudStack management server needs sudo permissions
 # without a password.
 
-Cmnd_Alias CLOUDSTACK = /bin/mkdir, /bin/mount, /bin/umount, /bin/cp, /bin/chmod, /usr/bin/keytool, /bin/keytool, /bin/touch
+Cmnd_Alias CLOUDSTACK = /bin/mkdir, /bin/mount, /bin/umount, /bin/cp, /bin/chmod, /usr/bin/keytool, /bin/keytool, /bin/touch, /bin/find, /bin/df, /bin/ls, /bin/qemu-img
 
 Defaults:@MSUSER@ !requiretty
 

server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java

@@ -446,7 +446,9 @@ public DomainRouterVO startVirtualRouter(final DomainRouterVO router, final User
         final int retryIndex = 5;
         final ExcludeList[] avoids = new ExcludeList[5];
         avoids[0] = new ExcludeList();
-        avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn());
+        if (routerToBeAvoid.getPodIdToDeployIn() != null) {
+            avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn());
+        }
         avoids[1] = new ExcludeList();
         avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId());
         avoids[2] = new ExcludeList();