api: improve listVM API handling and response speed

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java

@@ -44,6 +44,7 @@
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.api.response.VpcResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.query.QueryService;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.log4j.Logger;
 
@@ -96,10 +97,12 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
     @Parameter(name = ApiConstants.DETAILS,
                type = CommandType.LIST,
                collectionType = CommandType.STRING,
-               description = "comma separated list of vm details requested, "
-                   + "value can be a list of [all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]."
-                   + " If no parameter is passed in, the details will be defaulted to all")
-    private List<String> viewDetails;
+               description = "comma separated list of host details requested, "
+                   + "value can be a list of [all, group, nics, stats, secgrp, tmpl, servoff, diskoff, iso, volume, min, affgrp]."
+                   + " If no parameter is passed in, the details will be defaulted to all details excluding/including stats "
+                   + " for the listVirtualMachines API as determined by list.vm.default.details.stats setting (default: false)."
+                   + " However, all details including stats are returned for the listVirtualMachinesMetrics API")
+    protected List<String> viewDetails;
 
     @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "list vms by template")
     private Long templateId;
@@ -243,6 +246,9 @@ public EnumSet<VMDetails> getDetails() throws InvalidParameterValueException {
         EnumSet<VMDetails> dv;
         if (viewDetails == null || viewDetails.size() <= 0) {
             dv = EnumSet.of(VMDetails.all);
+            if (QueryService.AllowStatsInDefaultDetailsForListVMs.value()) {
+                dv.add(VMDetails.stats);
+            }
         } else {
             try {
                 ArrayList<VMDetails> dc = new ArrayList<VMDetails>();

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -125,6 +125,9 @@ public interface QueryService {
     static final ConfigKey<Boolean> SharePublicTemplatesWithOtherDomains = new ConfigKey<>("Advanced", Boolean.class, "share.public.templates.with.other.domains", "true",
             "If false, templates of this domain will not show up in the list templates of other domains.", true, ConfigKey.Scope.Domain);
 
+    ConfigKey<Boolean> AllowStatsInDefaultDetailsForListVMs = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "false",
+            "Determines whether VM stats should be returned when details are not specified in listVirtualMachines API request", true, ConfigKey.Scope.Global);
+
     ListResponse<UserResponse> searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException;
 
     ListResponse<UserResponse> searchForUsers(Long domainId, boolean recursive) throws PermissionDeniedException;

engine/schema/src/main/resources/META-INF/db/schema-41900to41910-cleanup.sql

@@ -18,3 +18,7 @@
 --;
 -- Schema upgrade cleanup from 4.19.0.0 to 4.19.1.0
 --;
+
+-- List VMs response optimisation https://github.com/apache/cloudstack/pull/8985
+UPDATE cloud.configuration set value='false' where name='vm.stats.increment.metrics';
+DELETE from cloud.configuration where name='vm.stats.increment.metrics.in.memory';

plugins/metrics/src/main/java/org/apache/cloudstack/api/ListVMsMetricsCmd.java

@@ -17,10 +17,12 @@
 
 package org.apache.cloudstack.api;
 
+import java.util.EnumSet;
 import java.util.List;
 
 import javax.inject.Inject;
 
+import com.cloud.exception.InvalidParameterValueException;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.command.user.vm.ListVMsCmd;
@@ -56,6 +58,16 @@ public String getCommandName() {
         return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
     }
 
+    @Override
+    public EnumSet<ApiConstants.VMDetails> getDetails() throws InvalidParameterValueException {
+        final EnumSet<ApiConstants.VMDetails> dv = super.getDetails();
+        // Include stats detail when 'all' details are asked
+        if (dv.contains(ApiConstants.VMDetails.all)) {
+            dv.add(ApiConstants.VMDetails.stats);
+        }
+        return dv;
+    }
+
     @Override
     public void execute() {
         ListResponse<UserVmResponse> userVms = _queryService.searchForUserVMs(this);

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -5686,6 +5686,6 @@ public String getConfigComponentName() {
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {AllowUserViewDestroyedVM, UserVMDeniedDetails, UserVMReadOnlyDetails, SortKeyAscending,
-                AllowUserViewAllDomainAccounts, SharePublicTemplatesWithOtherDomains};
+                AllowUserViewAllDomainAccounts, SharePublicTemplatesWithOtherDomains, AllowStatsInDefaultDetailsForListVMs};
     }
 }

server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java

@@ -251,7 +251,7 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us
             userVmResponse.setOsDisplayName(guestOS.getDisplayName());
         }
 
-        if (details.contains(VMDetails.all) || details.contains(VMDetails.stats)) {
+        if (details.contains(VMDetails.stats)) {
             // stats calculation
             VmStats vmStats = ApiDBUtils.getVmStatistics(userVm.getId(), accumulateStats);
             if (vmStats != null) {
@@ -268,7 +268,6 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us
                 userVmResponse.setMemoryKBs(totalMemory);
                 userVmResponse.setMemoryIntFreeKBs(correctedFreeMemory);
                 userVmResponse.setMemoryTargetKBs((long)vmStats.getTargetMemoryKBs());
-
             }
         }
 

server/src/main/java/com/cloud/server/StatsCollector.java

@@ -278,12 +278,10 @@ public String toString() {
     private static final ConfigKey<String> statsOutputUri = new ConfigKey<>("Advanced", String.class, "stats.output.uri", "",
             "URI to send StatsCollector statistics to. The collector is defined on the URI scheme. Example: graphite://graphite-hostaddress:port or influxdb://influxdb-hostaddress/dbname. Note that the port is optional, if not added the default port for the respective collector (graphite or influxdb) will be used. Additionally, the database name '/dbname' is  also optional; default db name is 'cloudstack'. You must create and configure the database if using influxdb.",
             true);
-    protected static ConfigKey<Boolean> vmStatsIncrementMetrics = new ConfigKey<>("Advanced", Boolean.class, "vm.stats.increment.metrics", "true",
+    protected static ConfigKey<Boolean> vmStatsIncrementMetrics = new ConfigKey<>("Advanced", Boolean.class, "vm.stats.increment.metrics", "false",
             "When set to 'true', VM metrics(NetworkReadKBs, NetworkWriteKBs, DiskWriteKBs, DiskReadKBs, DiskReadIOs and DiskWriteIOs) that are collected from the hypervisor are summed before being returned."
                     + "On the other hand, when set to 'false', the VM metrics API will just display the latest metrics collected.", true);
-    private static final ConfigKey<Boolean> VM_STATS_INCREMENT_METRICS_IN_MEMORY = new ConfigKey<>("Advanced", Boolean.class, "vm.stats.increment.metrics.in.memory", "true",
-            "When set to 'true', VM metrics(NetworkReadKBs, NetworkWriteKBs, DiskWriteKBs, DiskReadKBs, DiskReadIOs and DiskWriteIOs) that are collected from the hypervisor are summed and stored in memory. "
-            + "On the other hand, when set to 'false', the VM metrics API will just display the latest metrics collected.", true);
+
     protected static ConfigKey<Integer> vmStatsMaxRetentionTime = new ConfigKey<>("Advanced", Integer.class, "vm.stats.max.retention.time", "720",
             "The maximum time (in minutes) for keeping VM stats records in the database. The VM stats cleanup process will be disabled if this is set to 0 or less than 0.", true);
 
@@ -2131,7 +2129,6 @@ public String getConfigComponentName() {
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri,
             vmStatsIncrementMetrics, vmStatsMaxRetentionTime, vmStatsCollectUserVMOnly, vmDiskStatsRetentionEnabled, vmDiskStatsMaxRetentionTime,
-                VM_STATS_INCREMENT_METRICS_IN_MEMORY,
                 MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL,
                 DATABASE_SERVER_STATUS_COLLECTION_INTERVAL,
                 DATABASE_SERVER_LOAD_HISTORY_RETENTION_NUMBER};

ui/src/config/section/compute.js

@@ -31,9 +31,9 @@ export default {
       permission: ['listVirtualMachinesMetrics'],
       resourceType: 'UserVm',
       params: () => {
-        var params = { details: 'servoff,tmpl,iso,nics,backoff' }
+        var params = { details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp' }
         if (store.getters.metrics) {
-          params = { details: 'servoff,tmpl,iso,nics,backoff,stats' }
+          params = { details: 'all' }
         }
         params.isvnf = false
         return params

ui/src/views/AutogenView.vue

@@ -904,6 +904,7 @@ export default {
       if (['listVirtualMachinesMetrics'].includes(this.apiName) && this.dataView) {
         delete params.details
         delete params.isvnf
+        params.details = 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp'
       }
 
       this.loading = true