Skip to content

Commit 3d5c987

Browse files
solracsfsolracsf
committed
perf(files/cache): SQL aggregation for folder size calculation
Signed-off-by: Git'Fellow <12234510+solracsf@users.noreply.github.com>
1 parent f8dbe23 commit 3d5c987

2 files changed

Lines changed: 44 additions & 36 deletions

File tree

‎lib/private/Files/Cache/Cache.php‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1015,7 +1015,6 @@ public function calculateFolderSize($path, $entry = null) {
10151015
/**
10161016
* inner function because we can't add new params to the public function without breaking any child classes
10171017
*
1018-
* @param string $path
10191018
* @param array|null|ICacheEntry $entry (optional) meta data of the folder
10201019
* @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown
10211020
* @return int|float
@@ -1029,7 +1028,19 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
10291028
$id = $entry['fileid'];
10301029

10311030
$query = $this->getQueryBuilder();
1032-
$query->select('size', 'unencrypted_size')
1031+
$sizeCol = $query->getColumnName('size');
1032+
$unencryptedSizeCol = $query->getColumnName('unencrypted_size');
1033+
$query->selectAlias($query->func()->sum('size'), 'size_sum')
1034+
->selectAlias($query->func()->min('size'), 'size_min')
1035+
->selectAlias($query->func()->max('unencrypted_size'), 'unencrypted_max')
1036+
->selectAlias(
1037+
$query->createFunction("SUM(CASE WHEN $unencryptedSizeCol > 0 THEN $unencryptedSizeCol ELSE $sizeCol END)"),
1038+
'unencrypted_sum'
1039+
)
1040+
->selectAlias(
1041+
$query->createFunction("MIN(CASE WHEN $unencryptedSizeCol > 0 THEN $unencryptedSizeCol ELSE $sizeCol END)"),
1042+
'unencrypted_min'
1043+
)
10331044
->from('filecache')
10341045
->whereStorageId($this->getNumericStorageId())
10351046
->whereParent($id);
@@ -1038,34 +1049,24 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
10381049
}
10391050

10401051
$result = $query->executeQuery();
1041-
$rows = $result->fetchAll();
1052+
$agg = $result->fetch();
10421053
$result->closeCursor();
10431054

1044-
if ($rows) {
1045-
$sizes = array_map(function (array $row) {
1046-
return Util::numericToNumber($row['size']);
1047-
}, $rows);
1048-
$unencryptedOnlySizes = array_map(function (array $row) {
1049-
return Util::numericToNumber($row['unencrypted_size']);
1050-
}, $rows);
1051-
$unencryptedSizes = array_map(function (array $row) {
1052-
return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
1053-
}, $rows);
1054-
1055-
$sum = array_sum($sizes);
1056-
$min = min($sizes);
1057-
1058-
$unencryptedSum = array_sum($unencryptedSizes);
1059-
$unencryptedMin = min($unencryptedSizes);
1060-
$unencryptedMax = max($unencryptedOnlySizes);
1055+
// SUM() returns NULL on empty set
1056+
if ($agg && $agg['size_sum'] !== null) {
1057+
$sum = Util::numericToNumber($agg['size_sum']);
1058+
$min = Util::numericToNumber($agg['size_min']);
1059+
$unencryptedMax = Util::numericToNumber($agg['unencrypted_max'] ?? 0);
1060+
$unencryptedSum = Util::numericToNumber($agg['unencrypted_sum'] ?? 0);
1061+
$unencryptedMin = Util::numericToNumber($agg['unencrypted_min'] ?? 0);
10611062

10621063
$sum = 0 + $sum;
10631064
$min = 0 + $min;
1064-
if ($min === -1) {
1065-
$totalSize = $min;
1066-
} else {
1067-
$totalSize = $sum;
1068-
}
1065+
$unencryptedSum = 0 + $unencryptedSum;
1066+
$unencryptedMin = 0 + $unencryptedMin;
1067+
1068+
$totalSize = ($min === -1) ? $min : $sum;
1069+
10691070
if ($unencryptedMin === -1 || $min === -1) {
10701071
$unencryptedTotal = $unencryptedMin;
10711072
} else {
@@ -1077,15 +1078,16 @@ protected function calculateFolderSizeInner(string $path, $entry = null, bool $i
10771078
$unencryptedMax = 0;
10781079
}
10791080

1080-
// only set unencrypted size for a folder if any child entries have it set, or the folder is empty
1081+
// only set unencrypted size for a folder if any child entries have it set
1082+
// or if the folder is empty
10811083
$shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || ($entry['unencrypted_size'] ?? 0) > 0;
10821084
if ($entry['size'] !== $totalSize || (($entry['unencrypted_size'] ?? 0) !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) {
10831085
if ($shouldWriteUnEncryptedSize) {
1084-
// if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes
1086+
// if all children have an unencrypted size of 0
1087+
// just set the folder unencrypted size to 0 instead of summing the sizes
10851088
if ($unencryptedMax === 0) {
10861089
$unencryptedTotal = 0;
10871090
}
1088-
10891091
$this->update($id, [
10901092
'size' => $totalSize,
10911093
'unencrypted_size' => $unencryptedTotal,

‎lib/private/Files/Cache/Propagator.php‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -218,29 +218,32 @@ public function commitBatch(): void {
218218
$query = $this->connection->getQueryBuilder();
219219
$query->update('filecache')
220220
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
221-
->set('etag', $query->expr()->literal(uniqid()))
221+
->set('etag', $query->createParameter('etag'))
222222
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
223223
->andWhere($query->expr()->eq('fileid', $query->createParameter('fileid')));
224224

225225
$queryWithSize = $this->connection->getQueryBuilder();
226226
$queryWithSize->update('filecache')
227227
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
228-
->set('etag', $queryWithSize->expr()->literal(uniqid()))
228+
->set('etag', $queryWithSize->createParameter('etag'))
229229
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
230230
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
231231
->andWhere($queryWithSize->expr()->eq('fileid', $queryWithSize->createParameter('fileid')));
232232

233233
while ($row = $result->fetchAssociative()) {
234234
$item = $this->batch[$row['path']];
235+
$newEtag = uniqid();
235236
if ($item['size'] && $row['size'] > -1) {
236237
$queryWithSize->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT)
237238
->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT)
238239
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
239-
->executeStatement();
240+
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
241+
$queryWithSize->executeStatement();
240242
} else {
241243
$query->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT)
242244
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
243-
->executeStatement();
245+
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
246+
$query->executeStatement();
244247
}
245248
}
246249
}
@@ -249,28 +252,31 @@ public function commitBatch(): void {
249252
$query = $this->connection->getQueryBuilder();
250253
$query->update('filecache')
251254
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
252-
->set('etag', $query->expr()->literal(uniqid()))
255+
->set('etag', $query->createParameter('etag'))
253256
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
254257
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
255258

256259
$queryWithSize = $this->connection->getQueryBuilder();
257260
$queryWithSize->update('filecache')
258261
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
259-
->set('etag', $queryWithSize->expr()->literal(uniqid()))
262+
->set('etag', $queryWithSize->createParameter('etag'))
260263
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
261264
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
262265
->andWhere($queryWithSize->expr()->eq('path_hash', $queryWithSize->createParameter('hash')));
263266

264267
foreach ($this->batch as $item) {
268+
$newEtag = uniqid();
265269
if ($item['size']) {
266270
$queryWithSize->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR)
267271
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
268272
->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT)
269-
->executeStatement();
273+
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
274+
$queryWithSize->executeStatement();
270275
} else {
271276
$query->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR)
272277
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
273-
->executeStatement();
278+
->setParameter('etag', $newEtag, IQueryBuilder::PARAM_STR);
279+
$query->executeStatement();
274280
}
275281
}
276282
}

0 commit comments

Comments
 (0)