Skip to content

Users: unified storage table #65806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Altahrim wants to merge 3 commits into from
+428 −33
Draft

Users: unified storage table #65806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1de3b01
feat(query): introduce from_unixtime function
Altahrim Oct 23, 2026
87adb8e
feat(users): create unified user storage table
Altahrim Oct 23, 2026
55fabdd
feat(users): add migration job to populate new table
Altahrim Oct 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions core/BackgroundJobs/UserMigrationJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
<?php

declare(strict_types=1);

/*
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OC\Core\BackgroundJobs;

use Exception;
use NCU\User\UserStatus;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IAppConfig;
use OCP\IDBConnection;
use OCP\IServerInfo;
use OCP\Snowflake\ISnowflakeGenerator;
use Override;
use Psr\Log\LoggerInterface;

class UserMigrationJob extends TimedJob {
private const BATCH_SIZE = 200;

public function __construct(
ITimeFactory $time,
private readonly IAppConfig $appConfig,
private readonly IJobList $jobList,
private readonly IServerInfo $serverInfo,
private readonly IDBConnection $connection,
private readonly ISnowflakeGenerator $snowflakeGenerator,
private readonly LoggerInterface $logger,
) {
parent::__construct($time);
$this->setTimeSensitivity(self::TIME_INSENSITIVE);
$this->setInterval(3600);
}

#[Override]
protected function run(mixed $argument): void {
$start = time();
$workerId = (int)$argument['workerId'];
$nbWorkers = (int)$argument['nbWorkers'];
$this->logger->debug('Start to move users from bucket {bucketId} ({nbBuckets} in total).', [
'bucketId' => $workerId,
'nbWorkers' => $nbWorkers,
]);

// Try to move users until none are moved or time elapsed
do {
$this->connection->beginTransaction();
$nbMoved = $this->moveUsers(0, /*FIXME*/ $workerId, $nbWorkers);
$this->connection->commit();
} while ($nbMoved > 0 && time() < ($start + 3300));
}

private function moveUsers(int $backendId, int $workerId, int $nbWorkers): int {
$qb = $this->connection->getQueryBuilder();
$results = $qb
->select(
'uid',
'displayname',
'password',
'disabled.configvalue AS enabled',
'email.configvalue AS email',
'lastseen.configvalue AS lastseen',
)
->from('users', 'U')
->leftJoin(
'U',
'preferences',
'email',
$qb->expr()->andX(
$qb->expr()->eq('email.userid', 'U.uid'),
$qb->expr()->eq('email.appid', $qb->expr()->literal('settings')),
$qb->expr()->eq('email.configkey', $qb->expr()->literal('email')),
),
)
->leftJoin(
'U',
'preferences',
'disabled',
$qb->expr()->andX(
$qb->expr()->eq('disabled.userid', 'U.uid'),
$qb->expr()->eq('disabled.appid', $qb->expr()->literal('core')),
$qb->expr()->eq('disabled.configkey', $qb->expr()->literal('enabled')),
),
)
->leftJoin(
'U',
'preferences',
'lastseen',
$qb->expr()->andX(
$qb->expr()->eq('lastseen.userid', 'U.uid'),
$qb->expr()->eq('lastseen.appid', $qb->expr()->literal('login')),
$qb->expr()->eq('lastseen.configkey', $qb->expr()->literal('lastLogin')),
),
)
->setMaxResults(self::BATCH_SIZE)
->executeQuery();

if ($results->rowCount() === 0) {
// User table is empty
// Remove the job
$this->jobList->removeById($this->getId());
return 0;
}

$movedIds = [];
foreach ($results->iterateAssociative() as $user) {
// Should we handle it?
$bucketId = hexdec(hash('xxh32', $user['uid'])) % $nbWorkers;
if ($bucketId !== $workerId) {
continue;
}
try {
$this->copyUser($backendId, $user);
$movedIds[] = $user['uid'];
} catch (Exception $e) {
$this->logger->warning('Fail to move user {user} to new user table: {error}', [
'error' => $e->getMessage(),
'user' => $user['uid'],
'exception' => $e,
]);
}
}

if (count($movedIds) === 0) {
// No matching user to move, but table isn't empty yet.
// Stop job for now
return 0;
}

// Remove moved users
$qb
->delete('users')
->where($qb->expr()->in('uid', $qb->createNamedParameter($movedIds, IQueryBuilder::PARAM_STR_ARRAY)))
->executeStatement();

return count($movedIds);
}

private function copyUser(int $backendId, array $user): void {
$qb = $this->connection->getQueryBuilder();
$lastSeen = $user['lastseen'] === null ? $qb->createNamedParameter(null) : $qb->func()->fromUnixtime($user['lastseen']);

$qb
->insert('user_accounts')
->setValue('account_id', $qb->createNamedParameter($this->snowflakeGenerator->nextId()))
->setValue('backend_id', $qb->createNamedParameter($backendId))
->setValue('account_identifier', $qb->createNamedParameter($user['uid']))
->setValue('account_primary_email', $qb->createNamedParameter($user['email'] ?? ''))
->setValue('account_display_name', $qb->createNamedParameter($user['displayname'] ?? $user['uid']))
->setValue('account_password', $qb->createNamedParameter($user['password']))
->setValue('legacy_uid', $qb->createNamedParameter($user['uid']))
->setValue(
'account_status',
$qb->createNamedParameter($user['enabled'] === 'false' ? UserStatus::DISABLED->value : UserStatus::ENABLED->value),

Check failure on line 161 in core/BackgroundJobs/UserMigrationJob.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

UndefinedClass 

core/BackgroundJobs/UserMigrationJob.php:161:92: UndefinedClass: Class, interface or enum named NCU\User\UserStatus does not exist (see https://psalm.dev/019)

Check failure on line 161 in core/BackgroundJobs/UserMigrationJob.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

UndefinedClass 

core/BackgroundJobs/UserMigrationJob.php:161:62: UndefinedClass: Class, interface or enum named NCU\User\UserStatus does not exist (see https://psalm.dev/019)
)
->setValue('account_last_seen', $lastSeen)
->executeStatement();
}
}
142 changes: 142 additions & 0 deletions core/Migrations/Version36000Date20261005112424.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OC\Core\Migrations;

use Closure;
use Doctrine\DBAL\Platforms\MariaDBPlatform;
use Doctrine\DBAL\Platforms\MySQLPlatform;
use Doctrine\DBAL\Platforms\PostgreSQLPlatform;
use Doctrine\DBAL\Platforms\SqlitePlatform;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Schema\ColumnType;
use OCP\DB\Types;
use OCP\Migration\Attributes\CreateTable;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;
use Override;

/**
* Unified user table
*/
#[CreateTable(
table: 'oc_user_backends',
columns: ['backend_id', 'backend_name', 'backend_class'],
description: 'List of registered user backends',
)]
#[CreateTable(
table: 'oc_user_accounts',
columns: [
'account_id',
'backend_id',
'account_identifier',
'account_primary_email',
'account_display_name',
'account_password',
'legacy_uid',
'account_status',
'account_last_seen',
],
description: 'List of users from all backends',
)]
class Version36000Date20261005112424 extends SimpleMigrationStep {
#[Override]
public function name(): string {
return 'Create unified user table';
}

#[Override]
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$changed = false;

// We need a case-insensitive collation
// Logins "user123" and "UsEr123" are identical
$platform = $schema->getDatabasePlatform();
$caseInsensitiveCollation = match (true) {
$platform instanceof MySQLPlatform => 'utf8mb4_unicode_ci',
$platform instanceof MariaDBPlatform => 'utf8mb4_unicode_ci',
$platform instanceof PostgreSQLPlatform => 'und-x-icu',
$platform instanceof SqlitePlatform => 'NOCASE',
default => null,
};

if (!$schema->hasTable('user_backends')) {
$userBackendsTable = $schema->createTable('user_backends');
$userBackendsTable->addColumn('backend_id', Types::SMALLINT, ['autoincrement' => true, 'unsigned' => true, 'notnull' => true]);
$userBackendsTable->addColumn('backend_name', Types::STRING, [

Check failure on line 74 in core/Migrations/Version36000Date20261005112424.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

InvalidArgument 

core/Migrations/Version36000Date20261005112424.php:74:65: InvalidArgument: Argument 3 of OCP\DB\Schema\ITable::addColumn expects array{autoincrement?: bool, comment?: string, default?: null|scalar, fixed?: bool, length?: int|null, notnull?: bool, precision?: int, scale?: int, type?: 'bigint'|'binary'|'blob'|'boolean'|'date'|'date_immutable'|'datetime'|'datetime_immutable'|'datetimetz'|'datetimetz_immutable'|'decimal'|'float'|'guid'|'integer'|'json'|'smallint'|'string'|'text'|'time'|'time_immutable'|OCP\DB\Schema\ColumnType, unsigned?: bool}, but array{customSchemaOptions: array{collation: 'NOCASE'|'und-x-icu'|'utf8mb4_unicode_ci'|null}, length: 64} with additional array shape fields (customSchemaOptions) was provided (see https://psalm.dev/004)
'length' => 64,
'customSchemaOptions' => [
'collation' => $caseInsensitiveCollation,
],
]);
$userBackendsTable->addColumn('backend_class', Types::STRING, [

Check failure on line 80 in core/Migrations/Version36000Date20261005112424.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

InvalidArgument 

core/Migrations/Version36000Date20261005112424.php:80:66: InvalidArgument: Argument 3 of OCP\DB\Schema\ITable::addColumn expects array{autoincrement?: bool, comment?: string, default?: null|scalar, fixed?: bool, length?: int|null, notnull?: bool, precision?: int, scale?: int, type?: 'bigint'|'binary'|'blob'|'boolean'|'date'|'date_immutable'|'datetime'|'datetime_immutable'|'datetimetz'|'datetimetz_immutable'|'decimal'|'float'|'guid'|'integer'|'json'|'smallint'|'string'|'text'|'time'|'time_immutable'|OCP\DB\Schema\ColumnType, unsigned?: bool}, but array{customSchemaOptions: array{collation: 'NOCASE'|'und-x-icu'|'utf8mb4_unicode_ci'|null}, length: 64} with additional array shape fields (customSchemaOptions) was provided (see https://psalm.dev/004)
'length' => 64,
'customSchemaOptions' => [
'collation' => $caseInsensitiveCollation,
],
]);
$userBackendsTable->setPrimaryKey(['backend_id']);
$userBackendsTable->addUniqueIndex(['backend_name']);
$changed = true;
}

if (!$schema->hasTable('user_accounts')) {
$userAccountsTable = $schema->createTable('user_accounts');
$userAccountsTable->addColumn('account_id', Types::BIGINT, ['unsigned' => true, 'notnull' => true]);
$userAccountsTable->addColumn('backend_id', Types::SMALLINT, ['unsigned' => true, 'notnull' => true, 'default' => 0]);
$userAccountsTable->addColumn('account_identifier', Types::STRING, [

Check failure on line 95 in core/Migrations/Version36000Date20261005112424.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

InvalidArgument 

core/Migrations/Version36000Date20261005112424.php:95:71: InvalidArgument: Argument 3 of OCP\DB\Schema\ITable::addColumn expects array{autoincrement?: bool, comment?: string, default?: null|scalar, fixed?: bool, length?: int|null, notnull?: bool, precision?: int, scale?: int, type?: 'bigint'|'binary'|'blob'|'boolean'|'date'|'date_immutable'|'datetime'|'datetime_immutable'|'datetimetz'|'datetimetz_immutable'|'decimal'|'float'|'guid'|'integer'|'json'|'smallint'|'string'|'text'|'time'|'time_immutable'|OCP\DB\Schema\ColumnType, unsigned?: bool}, but array{customSchemaOptions: array{collation: 'NOCASE'|'und-x-icu'|'utf8mb4_unicode_ci'|null}, length: 64, notnull: true} with additional array shape fields (customSchemaOptions) was provided (see https://psalm.dev/004)
'notnull' => true,
'length' => 64,
'customSchemaOptions' => [
'collation' => $caseInsensitiveCollation,
],
]);
$userAccountsTable->addColumn('account_primary_email', Types::STRING, [

Check failure on line 102 in core/Migrations/Version36000Date20261005112424.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

InvalidArgument 

core/Migrations/Version36000Date20261005112424.php:102:74: InvalidArgument: Argument 3 of OCP\DB\Schema\ITable::addColumn expects array{autoincrement?: bool, comment?: string, default?: null|scalar, fixed?: bool, length?: int|null, notnull?: bool, precision?: int, scale?: int, type?: 'bigint'|'binary'|'blob'|'boolean'|'date'|'date_immutable'|'datetime'|'datetime_immutable'|'datetimetz'|'datetimetz_immutable'|'decimal'|'float'|'guid'|'integer'|'json'|'smallint'|'string'|'text'|'time'|'time_immutable'|OCP\DB\Schema\ColumnType, unsigned?: bool}, but array{customSchemaOptions: array{collation: 'NOCASE'|'und-x-icu'|'utf8mb4_unicode_ci'|null}, length: 64, notnull: true} with additional array shape fields (customSchemaOptions) was provided (see https://psalm.dev/004)
'notnull' => true,
'length' => 64,
'customSchemaOptions' => [
'collation' => $caseInsensitiveCollation,
],
]);
$userAccountsTable->addColumn('account_display_name', Types::STRING, [

Check failure on line 109 in core/Migrations/Version36000Date20261005112424.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

InvalidArgument 

core/Migrations/Version36000Date20261005112424.php:109:73: InvalidArgument: Argument 3 of OCP\DB\Schema\ITable::addColumn expects array{autoincrement?: bool, comment?: string, default?: null|scalar, fixed?: bool, length?: int|null, notnull?: bool, precision?: int, scale?: int, type?: 'bigint'|'binary'|'blob'|'boolean'|'date'|'date_immutable'|'datetime'|'datetime_immutable'|'datetimetz'|'datetimetz_immutable'|'decimal'|'float'|'guid'|'integer'|'json'|'smallint'|'string'|'text'|'time'|'time_immutable'|OCP\DB\Schema\ColumnType, unsigned?: bool}, but array{customSchemaOptions: array{collation: 'NOCASE'|'und-x-icu'|'utf8mb4_unicode_ci'|null}, default: '', length: 64, notnull: true} with additional array shape fields (customSchemaOptions) was provided (see https://psalm.dev/004)
'default' => '',
'notnull' => true,
'length' => 64,
'customSchemaOptions' => [
'collation' => $caseInsensitiveCollation,
],
]);
$userAccountsTable->addColumn('account_password', Types::BINARY, [
'default' => '',
'notnull' => true,
'length' => 255,
]);
$userAccountsTable->addColumn('legacy_uid', Types::STRING, [

Check failure on line 122 in core/Migrations/Version36000Date20261005112424.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

InvalidArgument 

core/Migrations/Version36000Date20261005112424.php:122:63: InvalidArgument: Argument 3 of OCP\DB\Schema\ITable::addColumn expects array{autoincrement?: bool, comment?: string, default?: null|scalar, fixed?: bool, length?: int|null, notnull?: bool, precision?: int, scale?: int, type?: 'bigint'|'binary'|'blob'|'boolean'|'date'|'date_immutable'|'datetime'|'datetime_immutable'|'datetimetz'|'datetimetz_immutable'|'decimal'|'float'|'guid'|'integer'|'json'|'smallint'|'string'|'text'|'time'|'time_immutable'|OCP\DB\Schema\ColumnType, unsigned?: bool}, but array{customSchemaOptions: array{collation: 'NOCASE'|'und-x-icu'|'utf8mb4_unicode_ci'|null}, default: '', length: 64, notnull: true} with additional array shape fields (customSchemaOptions) was provided (see https://psalm.dev/004)
'default' => '',
'notnull' => true,
'length' => 64,
'customSchemaOptions' => [
'collation' => $caseInsensitiveCollation,
],
]);
$userAccountsTable->addColumn('account_status', ColumnType::TinyInt, ['unsigned' => true, 'notnull' => true, 'default' => 0]);
$userAccountsTable->addColumn('account_last_seen', ColumnType::TimestampImmutable, [
'notnull' => false,
]);

$userAccountsTable->setPrimaryKey(['account_id']);
$userAccountsTable->addUniqueIndex(['account_identifier']);
$changed = true;
}

return $changed ? $schema : null;
}
}
8 changes: 8 additions & 0 deletions lib/composer/composer/autoload_classmap.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@
'NCU\\Sharing\\Source\\IShareSourceType' => $baseDir . '/lib/unstable/Sharing/Source/IShareSourceType.php',
'NCU\\Sharing\\Source\\ShareSource' => $baseDir . '/lib/unstable/Sharing/Source/ShareSource.php',
'NCU\\Sharing\\Source\\ShareSourceMetadata' => $baseDir . '/lib/unstable/Sharing/Source/ShareSourceMetadata.php',
'NCU\\User\\IAccount' => $baseDir . '/lib/unstable/User/IAccount.php',
'NCU\\User\\IUserBackend' => $baseDir . '/lib/unstable/User/IUserBackend.php',
'NCU\\User\\UserStatus' => $baseDir . '/lib/unstable/User/UserStatus.php',
'NCU\\WorkflowEngine\\Events\\RegisterRuntimeOperationsEvent' => $baseDir . '/lib/unstable/WorkflowEngine/Events/RegisterRuntimeOperationsEvent.php',
'NCU\\WorkflowEngine\\RuntimeOperation' => $baseDir . '/lib/unstable/WorkflowEngine/RuntimeOperation.php',
'NCU\\WorkflowEngine\\RuntimeScope' => $baseDir . '/lib/unstable/WorkflowEngine/RuntimeScope.php',
Expand Down Expand Up @@ -1451,6 +1454,7 @@
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
'OC\\Core\\BackgroundJobs\\PreviewMigrationJob' => $baseDir . '/core/BackgroundJobs/PreviewMigrationJob.php',
'OC\\Core\\BackgroundJobs\\UserMigrationJob' => $baseDir . '/core/BackgroundJobs/UserMigrationJob.php',
'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php',
'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php',
'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php',
Expand Down Expand Up @@ -1758,6 +1762,7 @@
'OC\\Core\\Migrations\\Version34000Date20260518163022' => $baseDir . '/core/Migrations/Version34000Date20260518163022.php',
'OC\\Core\\Migrations\\Version34000Date20260521110333' => $baseDir . '/core/Migrations/Version34000Date20260521110333.php',
'OC\\Core\\Migrations\\Version35000Date20260527162338' => $baseDir . '/core/Migrations/Version35000Date20260527162338.php',
'OC\\Core\\Migrations\\Version36000Date20261005112424' => $baseDir . '/core/Migrations/Version36000Date20261005112424.php',
'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
'OC\\Core\\Service\\CronService' => $baseDir . '/core/Service/CronService.php',
Expand Down Expand Up @@ -2234,6 +2239,7 @@
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php',
'OC\\Repair\\AddMovePreviewJob' => $baseDir . '/lib/private/Repair/AddMovePreviewJob.php',
'OC\\Repair\\AddMoveUsersJob' => $baseDir . '/lib/private/Repair/AddMoveUsersJob.php',
'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php',
'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php',
'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php',
Expand Down Expand Up @@ -2457,11 +2463,13 @@
'OC\\Updater\\VersionCheck' => $baseDir . '/lib/private/Updater/VersionCheck.php',
'OC\\UserStatus\\ISettableProvider' => $baseDir . '/lib/private/UserStatus/ISettableProvider.php',
'OC\\UserStatus\\Manager' => $baseDir . '/lib/private/UserStatus/Manager.php',
'OC\\User\\Account' => $baseDir . '/lib/private/User/Account.php',
'OC\\User\\AvailabilityCoordinator' => $baseDir . '/lib/private/User/AvailabilityCoordinator.php',
'OC\\User\\Backend' => $baseDir . '/lib/private/User/Backend.php',
'OC\\User\\BackgroundJobs\\CleanupDeletedUsers' => $baseDir . '/lib/private/User/BackgroundJobs/CleanupDeletedUsers.php',
'OC\\User\\BackgroundJobs\\CleanupLoginTokens' => $baseDir . '/lib/private/User/BackgroundJobs/CleanupLoginTokens.php',
'OC\\User\\Database' => $baseDir . '/lib/private/User/Database.php',
'OC\\User\\Db\\AccountRepository' => $baseDir . '/lib/private/User/Db/AccountRepository.php',
'OC\\User\\DisabledUserException' => $baseDir . '/lib/private/User/DisabledUserException.php',
'OC\\User\\DisplayNameCache' => $baseDir . '/lib/private/User/DisplayNameCache.php',
'OC\\User\\LastInteractiveLogin' => $baseDir . '/lib/private/User/LastInteractiveLogin.php',
Expand Down
Loading
Loading