Reclaim freed SQLite filesystem space

When data is removed from a SQLite database, pages are marked as free
instead of being release back to the OS / filesystem. Consequently, when
an app uses a lot of IndexedDB space at some point in time, even if the
database contents are deleted afterwards, the database file size will
remain with the max size so far. This can cause filesystem space to get
exhausted in constrained environments where small volume sizes are used
for storage.It can also cause issues with the database operations when
full volume capacity was reached (e.g. multiple domains, local storage
usage) despite not using the full "real" quota and WAL is being used.

These changes attempt to mitigate this issue by running the incremental
auto vacuum whenever some amount of free space is available to be
released. This is currently done on database open. While not ideal, as
there is no guarantee that the database will be opened again to free
that space, there is also no guarantee that the database close method
( SQLiteIDBBackingStore::close() ) will be invoked in all cases. The
cleanup can also not be run in sqlite hooks, as it would not be safe
due potential conflict with ongoing transactions.

The changes also contain a fix for the incremental vacuum operation
itself, as it was not being correctly performed and was returning an
error when it would actually succeed.

Author: filipe-norte-red

Source/WebCore/Modules/indexeddb/server/SQLiteIDBBackingStore.cpp

@@ -988,6 +988,7 @@ IDBError SQLiteIDBBackingStore::getOrEstablishDatabaseInfo(IDBDatabaseInfo& info
         return IDBError { ExceptionCode::UnknownError, "Unable to open database file on disk"_s };
 
     m_sqliteDB->disableThreadingChecks();
+    m_sqliteDB->turnOnIncrementalAutoVacuum();
     m_sqliteDB->enableAutomaticWALTruncation();
 
     m_sqliteDB->setCollationFunction("IDBKEY"_s, [](int aLength, const void* a, int bLength, const void* b) {
@@ -1036,6 +1037,10 @@ IDBError SQLiteIDBBackingStore::getOrEstablishDatabaseInfo(IDBDatabaseInfo& info
 
     m_databaseInfo = WTFMove(databaseInfo);
     info = *m_databaseInfo;
+
+    // Check if we are be able to free up some space on the file system
+    incrementalVacuumIfNeeded();
+
     return IDBError { };
 }
 
@@ -2843,6 +2848,22 @@ void SQLiteIDBBackingStore::handleLowMemoryWarning()
         m_sqliteDB->releaseMemory();
 }
 
+void SQLiteIDBBackingStore::incrementalVacuumIfNeeded()
+{
+    ASSERT(m_sqliteDB);
+    ASSERT(m_sqliteDB->isOpen());
+
+    int64_t freeSpaceSize = m_sqliteDB->freeSpaceSize();
+    int64_t totalSize = m_sqliteDB->totalSize();
+
+    if (totalSize <= 10 * freeSpaceSize) {
+        int result = m_sqliteDB->runIncrementalVacuumCommand();
+        if (result != SQLITE_DONE)
+            LOG_ERROR("Failed to perform incremental vacuum on database for path '%s'", m_sqliteDB->utf8().data());
+    }
+}
+
+
 #undef TABLE_SCHEMA_PREFIX
 #undef V3_RECORDS_TABLE_SCHEMA_SUFFIX
 #undef V3_INDEX_RECORDS_TABLE_SCHEMA_SUFFIX

Source/WebCore/Modules/indexeddb/server/SQLiteIDBBackingStore.h

@@ -202,6 +202,8 @@ class SQLiteIDBBackingStore final : public IDBBackingStore {
     SQLiteStatementAutoResetScope cachedStatement(SQL, ASCIILiteral);
     SQLiteStatementAutoResetScope cachedStatementForGetAllObjectStoreRecords(const IDBGetAllRecordsData&);
 
+    void incrementalVacuumIfNeeded();
+
     std::unique_ptr<SQLiteStatement> m_cachedStatements[static_cast<int>(SQL::Invalid)];
 
     IDBDatabaseIdentifier m_identifier;

Source/WebCore/platform/sql/SQLiteDatabase.cpp

@@ -540,8 +540,17 @@ int SQLiteDatabase::runIncrementalVacuumCommand()
     Locker locker { m_authorizerLock };
     enableAuthorizer(false);
 
-    if (!executeCommand("PRAGMA incremental_vacuum"_s))
+    auto statement = prepareStatement("PRAGMA incremental_vacuum"_s);
+    if (!statement)
+        return statement.error();
+
+    auto ret = statement->step();
+    while (ret == SQLITE_ROW) {
+        ret = statement->step();
+    }
+    if (ret != SQLITE_DONE) {
         LOG(SQLDatabase, "Unable to run incremental vacuum - %s", lastErrorMsg());
+    }
 
     enableAuthorizer(true);
     return lastError();