Skip to content

Commit 671d8ad

Browse files
abh1sarvishesh92
abh1sar
and
vishesh92
authored
Track volume usage data at a vm granularity as well (#11531)
Co-authored-by: Vishesh <8760112+vishesh92@users.noreply.github.com>
1 parent 15439ed commit 671d8ad

18 files changed

Lines changed: 243 additions & 156 deletions

File tree

‎engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,14 @@ public static void publishUsageEvent(String usageType, long accountId, long zone
9494

9595
}
9696

97+
public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
98+
Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) {
99+
if (displayResource) {
100+
saveUsageEvent(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName);
101+
}
102+
publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID);
103+
}
104+
97105
public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
98106
Long size, Long virtualSize, String entityType, String entityUUID, Map<String, String> details) {
99107
saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, virtualSize, details);
@@ -202,6 +210,10 @@ public static void saveUsageEvent(String usageType, long accountId, long zoneId,
202210
s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, vmId, securityGroupId));
203211
}
204212

213+
public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
214+
s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName));
215+
}
216+
205217
private static void publishUsageEvent(String usageEventType, Long accountId, Long zoneId, String resourceType, String resourceUUID) {
206218
String configKey = "publish.usage.events";
207219
String value = s_configDao.getValue(configKey);

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -903,7 +903,7 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri
903903
// Save usage event and update resource count for user vm volumes
904904
if (vm.getType() == VirtualMachine.Type.User) {
905905
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size,
906-
Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
906+
Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
907907
_resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
908908
}
909909
DiskProfile diskProfile = toDiskProfile(vol, offering);
@@ -981,7 +981,7 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering
981981
}
982982

983983
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size,
984-
Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
984+
Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
985985

986986
_resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
987987
}

‎engine/schema/src/main/java/com/cloud/event/UsageEventVO.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ public enum DynamicParameters {
7575
@Column(name = "virtual_size")
7676
private Long virtualSize;
7777

78+
@Column(name = "vm_id")
79+
private Long vmId;
80+
7881
public UsageEventVO() {
7982
}
8083

@@ -143,6 +146,18 @@ public UsageEventVO(String usageType, long accountId, long zoneId, long vmId, lo
143146
this.offeringId = securityGroupId;
144147
}
145148

149+
public UsageEventVO(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
150+
this.type = usageType;
151+
this.accountId = accountId;
152+
this.zoneId = zoneId;
153+
this.resourceId = resourceId;
154+
this.offeringId = offeringId;
155+
this.templateId = templateId;
156+
this.size = size;
157+
this.vmId = vmId;
158+
this.resourceName = resourceName;
159+
}
160+
146161
@Override
147162
public long getId() {
148163
return id;
@@ -248,4 +263,11 @@ public void setVirtualSize(Long virtualSize) {
248263
this.virtualSize = virtualSize;
249264
}
250265

266+
public Long getVmId() {
267+
return vmId;
268+
}
269+
270+
public void setVmId(Long vmId) {
271+
this.vmId = vmId;
272+
}
251273
}

‎engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
4545
private final SearchBuilder<UsageEventVO> latestEventsSearch;
4646
private final SearchBuilder<UsageEventVO> IpeventsSearch;
4747
private static final String COPY_EVENTS =
48-
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
49-
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
48+
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
49+
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
5050
private static final String COPY_ALL_EVENTS =
51-
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
52-
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
51+
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
52+
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
5353
private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "
5454
+ "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? ";
5555
private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "

‎engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ public class UsageVolumeVO implements InternalIdentity {
5959
@Column(name = "size")
6060
private long size;
6161

62+
@Column(name = "vm_id")
63+
private Long vmId;
64+
6265
@Column(name = "created")
6366
@Temporal(value = TemporalType.TIMESTAMP)
6467
private Date created = null;
@@ -70,13 +73,14 @@ public class UsageVolumeVO implements InternalIdentity {
7073
protected UsageVolumeVO() {
7174
}
7275

73-
public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, long size, Date created, Date deleted) {
76+
public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, Long vmId, long size, Date created, Date deleted) {
7477
this.volumeId = id;
7578
this.zoneId = zoneId;
7679
this.accountId = accountId;
7780
this.domainId = domainId;
7881
this.diskOfferingId = diskOfferingId;
7982
this.templateId = templateId;
83+
this.vmId = vmId;
8084
this.size = size;
8185
this.created = created;
8286
this.deleted = deleted;
@@ -126,4 +130,12 @@ public void setDeleted(Date deleted) {
126130
public long getVolumeId() {
127131
return volumeId;
128132
}
133+
134+
public Long getVmId() {
135+
return vmId;
136+
}
137+
138+
public void setVmId(Long vmId) {
139+
this.vmId = vmId;
140+
}
129141
}

‎engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ public UsageStorageDaoImpl() {
5757
IdSearch.and("accountId", IdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
5858
IdSearch.and("id", IdSearch.entity().getEntityId(), SearchCriteria.Op.EQ);
5959
IdSearch.and("type", IdSearch.entity().getStorageType(), SearchCriteria.Op.EQ);
60+
IdSearch.and("deleted", IdSearch.entity().getDeleted(), SearchCriteria.Op.NULL);
6061
IdSearch.done();
6162

6263
IdZoneSearch = createSearchBuilder();
@@ -74,6 +75,7 @@ public List<UsageStorageVO> listById(long accountId, long id, int type) {
7475
sc.setParameters("accountId", accountId);
7576
sc.setParameters("id", id);
7677
sc.setParameters("type", type);
78+
sc.setParameters("deleted", null);
7779
return listBy(sc, null);
7880
}
7981

‎engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,7 @@
2323
import com.cloud.utils.db.GenericDao;
2424

2525
public interface UsageVolumeDao extends GenericDao<UsageVolumeVO, Long> {
26-
public void removeBy(long userId, long id);
27-
28-
public void update(UsageVolumeVO usage);
29-
3026
public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate, boolean limit, int page);
27+
28+
List<UsageVolumeVO> listByVolumeId(long volumeId, long accountId);
3129
}

‎engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java‎

Lines changed: 32 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -18,81 +18,46 @@
1818

1919
import java.sql.PreparedStatement;
2020
import java.sql.ResultSet;
21-
import java.sql.SQLException;
2221
import java.util.ArrayList;
2322
import java.util.Date;
2423
import java.util.List;
2524
import java.util.TimeZone;
2625

2726

28-
import com.cloud.exception.CloudException;
27+
import javax.annotation.PostConstruct;
28+
2929
import org.springframework.stereotype.Component;
3030

3131
import com.cloud.usage.UsageVolumeVO;
3232
import com.cloud.utils.DateUtil;
3333
import com.cloud.utils.db.GenericDaoBase;
34+
import com.cloud.utils.db.SearchBuilder;
35+
import com.cloud.utils.db.SearchCriteria;
3436
import com.cloud.utils.db.TransactionLegacy;
3537

3638
@Component
3739
public class UsageVolumeDaoImpl extends GenericDaoBase<UsageVolumeVO, Long> implements UsageVolumeDao {
3840

39-
protected static final String REMOVE_BY_USERID_VOLID = "DELETE FROM usage_volume WHERE account_id = ? AND volume_id = ?";
40-
protected static final String UPDATE_DELETED = "UPDATE usage_volume SET deleted = ? WHERE account_id = ? AND volume_id = ? and deleted IS NULL";
41-
protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
41+
protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
4242
+ "FROM usage_volume " + "WHERE account_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR "
4343
+ " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))";
44-
protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
44+
protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
4545
+ "FROM usage_volume " + "WHERE domain_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR "
4646
+ " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))";
47-
protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
47+
protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
4848
+ "FROM usage_volume " + "WHERE (deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?))";
49+
private SearchBuilder<UsageVolumeVO> volumeSearch;
4950

5051
public UsageVolumeDaoImpl() {
5152
}
5253

53-
@Override
54-
public void removeBy(long accountId, long volId) {
55-
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
56-
try {
57-
txn.start();
58-
try(PreparedStatement pstmt = txn.prepareStatement(REMOVE_BY_USERID_VOLID);) {
59-
if (pstmt != null) {
60-
pstmt.setLong(1, accountId);
61-
pstmt.setLong(2, volId);
62-
pstmt.executeUpdate();
63-
}
64-
}catch (SQLException e) {
65-
throw new CloudException("Error removing usageVolumeVO:"+e.getMessage(), e);
66-
}
67-
txn.commit();
68-
} catch (Exception e) {
69-
txn.rollback();
70-
logger.warn("Error removing usageVolumeVO:"+e.getMessage(), e);
71-
} finally {
72-
txn.close();
73-
}
74-
}
75-
76-
@Override
77-
public void update(UsageVolumeVO usage) {
78-
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
79-
PreparedStatement pstmt = null;
80-
try {
81-
txn.start();
82-
if (usage.getDeleted() != null) {
83-
pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED);
84-
pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getDeleted()));
85-
pstmt.setLong(2, usage.getAccountId());
86-
pstmt.setLong(3, usage.getVolumeId());
87-
pstmt.executeUpdate();
88-
}
89-
txn.commit();
90-
} catch (Exception e) {
91-
txn.rollback();
92-
logger.warn("Error updating UsageVolumeVO", e);
93-
} finally {
94-
txn.close();
95-
}
54+
@PostConstruct
55+
protected void init() {
56+
volumeSearch = createSearchBuilder();
57+
volumeSearch.and("accountId", volumeSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
58+
volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
59+
volumeSearch.and("deleted", volumeSearch.entity().getDeleted(), SearchCriteria.Op.NULL);
60+
volumeSearch.done();
9661
}
9762

9863
@Override
@@ -150,11 +115,15 @@ public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date s
150115
if (tId == 0) {
151116
tId = null;
152117
}
153-
long size = Long.valueOf(rs.getLong(7));
118+
Long vmId = Long.valueOf(rs.getLong(7));
119+
if (vmId == 0) {
120+
vmId = null;
121+
}
122+
long size = Long.valueOf(rs.getLong(8));
154123
Date createdDate = null;
155124
Date deletedDate = null;
156-
String createdTS = rs.getString(8);
157-
String deletedTS = rs.getString(9);
125+
String createdTS = rs.getString(9);
126+
String deletedTS = rs.getString(10);
158127

159128
if (createdTS != null) {
160129
createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS);
@@ -163,7 +132,7 @@ public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date s
163132
deletedDate = DateUtil.parseDateString(s_gmtTimeZone, deletedTS);
164133
}
165134

166-
usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, size, createdDate, deletedDate));
135+
usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, vmId, size, createdDate, deletedDate));
167136
}
168137
} catch (Exception e) {
169138
txn.rollback();
@@ -174,4 +143,13 @@ public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date s
174143

175144
return usageRecords;
176145
}
146+
147+
@Override
148+
public List<UsageVolumeVO> listByVolumeId(long volumeId, long accountId) {
149+
SearchCriteria<UsageVolumeVO> sc = volumeSearch.create();
150+
sc.setParameters("accountId", accountId);
151+
sc.setParameters("volumeId", volumeId);
152+
sc.setParameters("deleted", null);
153+
return listBy(sc);
154+
}
177155
}

‎engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHA
4141
-- Populate uuid for existing rows where uuid is NULL or empty
4242
UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid = '';
4343

44+
-- Add vm_id column to usage_event table for volume usage events
45+
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"');
46+
CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"');
47+
48+
-- Add vm_id column to cloud_usage.usage_volume table
49+
CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"');
50+
4451
-- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository.
4552
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone''');
4653

‎server/src/main/java/com/cloud/api/ApiResponseHelper.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4298,6 +4298,9 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map<String, Se
42984298
if (volume != null) {
42994299
builder.append("for ").append(volume.getName()).append(" (").append(volume.getUuid()).append(")");
43004300
}
4301+
if (vmInstance != null) {
4302+
builder.append(" attached to VM ").append(vmInstance.getHostName()).append(" (").append(vmInstance.getUuid()).append(")");
4303+
}
43014304
if (diskOff != null) {
43024305
builder.append(" with disk offering ").append(diskOff.getName()).append(" (").append(diskOff.getUuid()).append(")");
43034306
}

0 commit comments

Comments
 (0)