From e0628593c970383600d4f7ec7ec5f19e5c6dba1a Mon Sep 17 00:00:00 2001 From: turegjorup Date: Wed, 21 Oct 2026 12:25:56 +0200 Subject: [PATCH 1/6] fix: keep only sites inside the latest dir result Ignore dir results older than the server's latest, so replays cannot apply an old list. Attach sites to the listed dir they are in instead of a LIKE prefix, where _ was a wildcard, and delete sites and results outside every listed dir. --- phpstan.dist.neon | 1 - src/Handler/DirectoryHandler.php | 54 ++++--- src/Repository/DetectionResultRepository.php | 15 -- src/Repository/SiteRepository.php | 13 -- tests/Handler/DirectoryHandlerTest.php | 149 +++++++++++++++++++ 5 files changed, 183 insertions(+), 49 deletions(-) create mode 100644 tests/Handler/DirectoryHandlerTest.php diff --git a/phpstan.dist.neon b/phpstan.dist.neon index b723202c..f135648a 100644 --- a/phpstan.dist.neon +++ b/phpstan.dist.neon @@ -23,4 +23,3 @@ parameters: - src/Entity/ - src/Form/ - src/Service/ - - src/Handler/ diff --git a/src/Handler/DirectoryHandler.php b/src/Handler/DirectoryHandler.php index 5b7cca17..ea71f7c4 100644 --- a/src/Handler/DirectoryHandler.php +++ b/src/Handler/DirectoryHandler.php @@ -10,7 +10,9 @@ use App\Repository\SiteRepository; use App\Service\InstallationFactory; use App\Types\DetectionType; +use App\Utils\RootDirNormalizer; use Doctrine\Common\Collections\Collection; +use Doctrine\ORM\EntityManagerInterface; /** * Handler for DetectionResult off type "dir" (Installations). @@ -24,34 +26,38 @@ public function __construct( private SiteRepository $siteRepository, private InstallationFactory $installationFactory, private DetectionResultRepository $detectionResultRepository, + private EntityManagerInterface $entityManager, ) { } public function handleResult(DetectionResult $detectionResult): void { $server = $detectionResult->getServer(); - $oldInstallations = $server->getInstallations(); + + // Only the latest dir result describes the server. Older ones (e.g. from a replay) are history. + $latest = $this->detectionResultRepository->findOneBy(['server' => $server, 'type' => DetectionType::DIRECTORY], ['lastContact' => 'DESC']); + if ($latest !== $detectionResult) { + return; + } $installations = $this->installationFactory->getInstallations($detectionResult); - /** @var Installation $installation */ - foreach ($installations as $installation) { - $sites = $this->siteRepository->findByRootDirAndServer( - $installation->getRootDir(), - $server - ); - /* @var Site $site */ - foreach ($sites as $site) { + // Sites belong to the listed dir they are in. Sites outside every listed dir are gone. + foreach ($this->siteRepository->findBy(['server' => $server]) as $site) { + $installation = $this->findEnclosing($installations, $site->getRootDir()); + if (null === $installation) { + $this->entityManager->remove($site); + } else { $installation->addSite($site); } } - $detectionResult->getServer()->setInstallations($installations); + $server->setInstallations($installations); - // Delete results from installations no longer seen on the server. - foreach ($oldInstallations as $oldInstallation) { - if (!$installations->contains($oldInstallation)) { - $this->detectionResultRepository->deleteByInstallation($oldInstallation); + // Delete results from dirs no longer listed, so a replay cannot recreate them. + foreach ($this->detectionResultRepository->findBy(['server' => $server]) as $result) { + if (DetectionType::DIRECTORY !== $result->getType() && null === $this->findEnclosing($installations, $result->getRootDir())) { + $this->entityManager->remove($result); } } } @@ -61,15 +67,23 @@ public function supportsType(string $type): bool return DetectionType::DIRECTORY === $type; } - private function getRemovedInstallations(Collection $oldInstallations, Collection $newInstallations): array + /** + * Find the installation for the root dir itself or, failing that, its closest parent dir. + * + * @param Collection $installations + */ + private function findEnclosing(Collection $installations, string $rootDir): ?Installation { - $result = []; - foreach ($oldInstallations as $installation) { - if (!$newInstallations->contains($installation)) { - $result[] = $installation; + $rootDir = RootDirNormalizer::normalize($rootDir); + $enclosing = null; + foreach ($installations as $installation) { + $dir = $installation->getRootDir(); + if (($rootDir === $dir || str_starts_with($rootDir, $dir.'/')) + && strlen($dir) > strlen($enclosing?->getRootDir() ?? '')) { + $enclosing = $installation; } } - return $result; + return $enclosing; } } diff --git a/src/Repository/DetectionResultRepository.php b/src/Repository/DetectionResultRepository.php index 56364897..0f304f4a 100644 --- a/src/Repository/DetectionResultRepository.php +++ b/src/Repository/DetectionResultRepository.php @@ -5,7 +5,6 @@ namespace App\Repository; use App\Entity\DetectionResult; -use App\Entity\Installation; use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; use Doctrine\Persistence\ManagerRegistry; @@ -89,18 +88,4 @@ public function cleanup(DetectionResult $detectionResult, int $keep = 5, bool $f $em->flush(); } } - - /** - * Delete all detection results for a given installation. - */ - public function deleteByInstallation(Installation $installation): void - { - $qb = $this->getEntityManager()->createQueryBuilder(); - $qb->delete(DetectionResult::class, 'd') - ->where('d.server = :server') - ->andWhere('d.rootDir =:rootDir') - ->setParameter('server', $installation->getServer()) - ->setParameter('rootDir', $installation->getRootDir()); - $qb->getQuery()->execute(); - } } diff --git a/src/Repository/SiteRepository.php b/src/Repository/SiteRepository.php index d4df5296..f8ed961e 100644 --- a/src/Repository/SiteRepository.php +++ b/src/Repository/SiteRepository.php @@ -4,7 +4,6 @@ namespace App\Repository; -use App\Entity\Server; use App\Entity\Site; use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository; use Doctrine\ORM\AbstractQuery; @@ -25,18 +24,6 @@ public function __construct(ManagerRegistry $registry) parent::__construct($registry, Site::class); } - public function findByRootDirAndServer(string $rootDir, Server $server): mixed - { - return $this->createQueryBuilder('s') - ->andWhere('s.rootDir LIKE :rootDir') - ->setParameter('rootDir', $rootDir.'%') - ->andWhere('s.server = :server') - ->setParameter('server', $server->getId()->toBinary()) - ->getQuery() - ->getResult() - ; - } - /** * Get unique primary domains from existing sites. */ diff --git a/tests/Handler/DirectoryHandlerTest.php b/tests/Handler/DirectoryHandlerTest.php new file mode 100644 index 00000000..1c5e13b5 --- /dev/null +++ b/tests/Handler/DirectoryHandlerTest.php @@ -0,0 +1,149 @@ +entityManager = self::getContainer()->get(EntityManagerInterface::class); + $this->serverId = (string) $this->entityManager->getRepository(Server::class)->findOneBy([])->getId(); + } + + public function testRemovesInstallationsNoLongerInDirResult(): void + { + $this->process(DetectionType::DIRECTORY, '/data/www', [self::KEPT_DIR, self::GONE_DIR], new \DateTimeImmutable('-1 day')); + $this->process(DetectionType::DOCKER, self::KEPT_DIR, $this->dockerData('kept-probe.example.com')); + $goneDockerResultId = $this->process(DetectionType::DOCKER, self::GONE_DIR, $this->dockerData('gone-probe.example.com')); + + $this->assertNotNull($this->findInstallation(self::GONE_DIR)); + + $this->process(DetectionType::DIRECTORY, '/data/www', [self::KEPT_DIR]); + + $this->assertNull($this->findInstallation(self::GONE_DIR)); + $this->assertNull($this->entityManager->getRepository(Site::class)->findOneBy(['rootDir' => self::GONE_DIR])); + $this->assertNull($this->entityManager->getRepository(DetectionResult::class)->find($goneDockerResultId)); + + $this->assertNotNull($this->findInstallation(self::KEPT_DIR)); + $this->assertNotNull($this->entityManager->getRepository(Site::class)->findOneBy(['rootDir' => self::KEPT_DIR])); + } + + public function testIgnoresDirResultsOlderThanTheLatest(): void + { + $this->process(DetectionType::DIRECTORY, '/data/www', [self::KEPT_DIR, self::GONE_DIR]); + $this->process(DetectionType::DOCKER, self::GONE_DIR, $this->dockerData('gone-probe.example.com')); + + // E.g. replaying a dir result from before GONE_DIR was added. + $this->process(DetectionType::DIRECTORY, '/data', [self::KEPT_DIR], new \DateTimeImmutable('-1 day')); + + $this->assertNotNull($this->findInstallation(self::GONE_DIR)); + $this->assertNotNull($this->entityManager->getRepository(Site::class)->findOneBy(['rootDir' => self::GONE_DIR])); + $this->assertNotNull($this->entityManager->getRepository(DetectionResult::class)->findOneBy(['rootDir' => self::GONE_DIR])); + } + + public function testKeepsOnlySitesInsideAListedDir(): void + { + $subDir = self::KEPT_DIR.'/web'; + $lookAlikes = [ + '/data/www/kept.probe/htdocs', // matched "kept_probe" when "_" was a LIKE wildcard + self::KEPT_DIR.'_old', // matched as a prefix without a "/" boundary + ]; + + $this->process(DetectionType::DIRECTORY, '/data/www', [self::KEPT_DIR], new \DateTimeImmutable('-1 day')); + foreach ([$subDir, ...$lookAlikes] as $i => $rootDir) { + $this->process(DetectionType::DOCKER, $rootDir, $this->dockerData('probe'.$i.'.example.com')); + } + + // The state prod is in: look-alike sites moved onto the kept installation, their own installations gone. + $siteRepository = $this->entityManager->getRepository(Site::class); + $kept = $this->findInstallation(self::KEPT_DIR); + foreach ($lookAlikes as $rootDir) { + $site = $siteRepository->findOneBy(['rootDir' => $rootDir]); + $this->entityManager->remove($site->getInstallation()); + $kept->addSite($site); + } + $this->entityManager->flush(); + + $this->process(DetectionType::DIRECTORY, '/data', [self::KEPT_DIR]); + + $this->entityManager->clear(); + foreach ($lookAlikes as $rootDir) { + $this->assertNull($siteRepository->findOneBy(['rootDir' => $rootDir]), $rootDir); + $this->assertNull($this->entityManager->getRepository(DetectionResult::class)->findOneBy(['rootDir' => $rootDir]), $rootDir); + } + $this->assertSame(self::KEPT_DIR, $siteRepository->findOneBy(['rootDir' => $subDir])?->getInstallation()->getRootDir()); + $this->assertNotNull($this->entityManager->getRepository(DetectionResult::class)->findOneBy(['rootDir' => $subDir])); + } + + /** + * @return array + */ + private function dockerData(string $domain): array + { + return [ + 'domain' => $domain, + 'containers' => [[ + 'name' => 'phpfpm', + 'image' => 'itkdev/php8.4-fpm', + 'packages' => 'unknown', + 'audit' => 'unknown', + 'drupal' => 'unknown', + 'symfony' => 'unknown', + ]], + ]; + } + + /** + * Persist a detection result and run it through the real processing flow. + * + * @param array $data + */ + private function process(string $type, string $rootDir, array $data, ?\DateTimeImmutable $lastContact = null): string + { + $server = $this->entityManager->getRepository(Server::class)->find($this->serverId); + + $detectionResult = new DetectionResult(); + $detectionResult->setType($type); + $detectionResult->setRootDir($rootDir); + $detectionResult->setServer($server); + $detectionResult->setData(json_encode($data, JSON_THROW_ON_ERROR)); + $detectionResult->generateHash(); + $detectionResult->setLastContact($lastContact); + $this->entityManager->persist($detectionResult); + $this->entityManager->flush(); + + $id = $detectionResult->getId(); + self::getContainer()->get(ProcessDetectionResultHandler::class)(new ProcessDetectionResult($id)); + + return (string) $id; + } + + private function findInstallation(string $rootDir): ?Installation + { + $this->entityManager->clear(); + + return $this->entityManager->getRepository(Installation::class)->findOneBy(['rootDir' => $rootDir]); + } +} From ed1c3399dd434227c25b8417b3f2657d99759335 Mon Sep 17 00:00:00 2001 From: turegjorup Date: Wed, 21 Oct 2026 12:25:59 +0200 Subject: [PATCH 2/6] fix: re-apply unchanged dir results on every harvest An unchanged dir result only had its timestamp updated, so sites recreated after it was first processed, e.g. by a docker replay, were never removed. --- .../PersistDetectionResultHandler.php | 7 ++++ .../PersistDetectionResultHandlerTest.php | 34 ++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/MessageHandler/PersistDetectionResultHandler.php b/src/MessageHandler/PersistDetectionResultHandler.php index c745f155..24bc0008 100644 --- a/src/MessageHandler/PersistDetectionResultHandler.php +++ b/src/MessageHandler/PersistDetectionResultHandler.php @@ -8,6 +8,7 @@ use App\Message\ProcessDetectionResult; use App\Repository\DetectionResultRepository; use App\Repository\ServerRepository; +use App\Types\DetectionType; use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\Messenger\Attribute\AsMessageHandler; use Symfony\Component\Messenger\Exception\UnrecoverableMessageHandlingException; @@ -58,6 +59,12 @@ public function __invoke(PersistDetectionResult $message): void $existingResult->setLastContact($message->receivedAt); $this->entityManager->flush(); + + // Except dir results: they decide which installations exist, so re-apply them to + // remove anything recreated since they were first processed (e.g. by a replay). + if (DetectionType::DIRECTORY === $existingResult->getType() && null !== $id = $existingResult->getId()) { + $this->messageBus->dispatch(new ProcessDetectionResult($id)); + } } $this->entityManager->clear(); diff --git a/tests/MessageHandler/PersistDetectionResultHandlerTest.php b/tests/MessageHandler/PersistDetectionResultHandlerTest.php index 830d8e75..ff041e91 100644 --- a/tests/MessageHandler/PersistDetectionResultHandlerTest.php +++ b/tests/MessageHandler/PersistDetectionResultHandlerTest.php @@ -12,6 +12,7 @@ use App\Repository\DetectionResultRepository; use App\Repository\ServerRepository; use Doctrine\ORM\EntityManagerInterface; +use PHPUnit\Framework\MockObject\MockObject; use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase; use Symfony\Component\Messenger\Envelope; use Symfony\Component\Messenger\MessageBusInterface; @@ -19,10 +20,10 @@ class PersistDetectionResultHandlerTest extends KernelTestCase { - private ServerRepository $serverRepositoryMock; - private DetectionResultRepository $detectionResultRepositoryMock; - private EntityManagerInterface $entityManagerMock; - private MessageBusInterface $messageBusMock; + private ServerRepository&MockObject $serverRepositoryMock; + private DetectionResultRepository&MockObject $detectionResultRepositoryMock; + private EntityManagerInterface&MockObject $entityManagerMock; + private MessageBusInterface&MockObject $messageBusMock; private PersistDetectionResultHandler $handler; private DetectionResult $detectionResult; @@ -78,6 +79,8 @@ public function testInvokeNewDetectionResult(): void public function testInvokeExistingDetectionResult(): void { + $this->detectionResult->setType('docker'); + $apiKey = '12345678'; $receivedAt = new \DateTimeImmutable(); $detectionResultReceived = new PersistDetectionResult($this->detectionResult, $apiKey, $receivedAt); @@ -99,4 +102,27 @@ public function testInvokeExistingDetectionResult(): void // "lastContact" should only be updated for the existing result $this->assertEquals($receivedAt, $existingResult->getLastContact()); } + + public function testInvokeExistingDirDetectionResultIsReprocessed(): void + { + $receivedAt = new \DateTimeImmutable(); + $detectionResultReceived = new PersistDetectionResult($this->detectionResult, '12345678', $receivedAt); + + $existingResult = new DetectionResult(); + $existingResult->setId(new Ulid()); + $existingResult->setType('dir'); + + $this->detectionResultRepositoryMock->expects($this->once())->method('findOneBy') + ->willReturn($existingResult); + $this->entityManagerMock->expects($this->never())->method('persist'); + + // The dir list decides which installations exist, so it is re-applied on every harvest. + $this->messageBusMock->expects($this->once())->method('dispatch') + ->with(new ProcessDetectionResult($existingResult->getId())) + ->willReturn(new Envelope(new ProcessDetectionResult(new Ulid()))); + + $this->handler->__invoke($detectionResultReceived); + + $this->assertEquals($receivedAt, $existingResult->getLastContact()); + } } From a34976854083347ed9a78105016b8dbea6297d44 Mon Sep 17 00:00:00 2001 From: turegjorup Date: Wed, 21 Oct 2026 12:26:01 +0200 Subject: [PATCH 3/6] feat: purge detection results older than 30 days daily Run app:data:purge on the default schedule at 03:00 with up to 30 minutes jitter. --date now takes relative dates such as "30 days ago". The worker consumes scheduler_default. --- composer.json | 2 +- docker-compose.server.override.yml | 2 +- src/Command/PurgeCommand.php | 10 ++-- tests/Command/PurgeCommandTest.php | 78 ++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 tests/Command/PurgeCommandTest.php diff --git a/composer.json b/composer.json index 212f9c70..4bd02797 100644 --- a/composer.json +++ b/composer.json @@ -139,7 +139,7 @@ "bin/console hautelook:fixtures:load --no-interaction" ], "queues": [ - "bin/console messenger:consume async scheduler_drupal_releases --failure-limit=1 -vvv" + "bin/console messenger:consume async scheduler_drupal_releases scheduler_default --failure-limit=1 -vvv" ], "tests": [ "bin/console --env=test doctrine:database:drop --if-exists --force --quiet", diff --git a/docker-compose.server.override.yml b/docker-compose.server.override.yml index 8eb7ce73..da42b572 100644 --- a/docker-compose.server.override.yml +++ b/docker-compose.server.override.yml @@ -24,7 +24,7 @@ services: restart: unless-stopped stop_grace_period: 20s environment: - - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_drupal_releases + - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_drupal_releases scheduler_default - APP_SUPERVISOR_WORKERS=1 - APP_SUPERVISOR_USER=deploy - PHP_MEMORY_LIMIT=256M diff --git a/src/Command/PurgeCommand.php b/src/Command/PurgeCommand.php index d37cbad1..32981dc2 100644 --- a/src/Command/PurgeCommand.php +++ b/src/Command/PurgeCommand.php @@ -11,11 +11,13 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; +use Symfony\Component\Scheduler\Attribute\AsPeriodicTask; #[AsCommand( name: 'app:data:purge', description: 'Purge old data', )] +#[AsPeriodicTask(frequency: '1 day', from: '03:00', jitter: 1800, arguments: '--date="30 days ago"')] class PurgeCommand extends Command { public function __construct( @@ -29,17 +31,17 @@ public function __construct( */ protected function configure(): void { - $this->addOption('date', null, InputOption::VALUE_REQUIRED, 'Remove results older that this date (Y-m-d)'); + $this->addOption('date', null, InputOption::VALUE_REQUIRED, 'Remove results not seen since this date, e.g. "2026-09-21" or "30 days ago"'); } protected function execute(InputInterface $input, OutputInterface $output): int { $io = new SymfonyStyle($input, $output); - $format = 'Y-m-d'; $date = (string) $input->getOption('date'); - $dateTime = \DateTime::createFromFormat($format, $date); - if (false === $dateTime || $dateTime->format($format) !== $date) { + try { + $dateTime = new \DateTime($date); + } catch (\DateMalformedStringException) { $output->writeln('Unknown date format: '.$date); return Command::FAILURE; diff --git a/tests/Command/PurgeCommandTest.php b/tests/Command/PurgeCommandTest.php new file mode 100644 index 00000000..50ba149a --- /dev/null +++ b/tests/Command/PurgeCommandTest.php @@ -0,0 +1,78 @@ +get(EntityManagerInterface::class); + $server = $entityManager->getRepository(Server::class)->findOneBy([]); + foreach (['-31 days' => 'old', '-29 days' => 'recent'] as $lastContact => $data) { + $result = new DetectionResult(); + $result->setType(DetectionType::DOCKER); + $result->setRootDir(self::ROOT_DIR); + $result->setServer($server); + $result->setData($data); + $result->generateHash(); + $result->setLastContact(new \DateTimeImmutable($lastContact)); + $entityManager->persist($result); + } + $entityManager->flush(); + + $tester = $this->console(['--date' => '30 days ago']); + + $tester->assertCommandIsSuccessful(); + $entityManager->clear(); + $remaining = $entityManager->getRepository(DetectionResult::class)->findBy(['rootDir' => self::ROOT_DIR]); + $this->assertSame(['recent'], array_map(static fn (DetectionResult $result): string => $result->getData(), $remaining)); + } + + public function testRejectsAnUnparsableDate(): void + { + self::bootKernel(); + + $this->console(['--date' => 'not a date'])->assertCommandFailed(); + } + + public function testRunsDailyOnTheDefaultSchedule(): void + { + self::bootKernel(); + + $tester = $this->console(['command' => 'debug:scheduler', 'schedule' => ['default']]); + + $tester->assertCommandIsSuccessful(); + $this->assertStringContainsString('every 1 day with 0-1800 second jitter', $tester->getDisplay()); + $this->assertStringContainsString('app:data:purge --date="30 days ago"', $tester->getDisplay()); + } + + /** + * @param array $input + */ + private function console(array $input): ApplicationTester + { + $application = new Application(self::$kernel); + $application->setAutoExit(false); + + $tester = new ApplicationTester($application); + $tester->run($input + ['command' => 'app:data:purge']); + + return $tester; + } +} From 1d8a04472584f6f921365ef5d002968b4a962ead Mon Sep 17 00:00:00 2001 From: turegjorup Date: Wed, 21 Oct 2026 12:33:21 +0200 Subject: [PATCH 4/6] refactor: run scheduled tasks on one stateful default schedule Attribute tasks such as the purge cannot be stateful on their own, and the worker restarts every 15 minutes, so the jittered purge would skip most days. Turn DrupalReleaseSchedule into the stateful default schedule that attribute tasks join, and consume only scheduler_default. --- composer.json | 2 +- docker-compose.server.override.yml | 2 +- ...eleaseSchedule.php => DefaultSchedule.php} | 12 +++-- tests/Scheduler/DefaultScheduleTest.php | 51 +++++++++++++++++++ tests/Scheduler/DrupalReleaseScheduleTest.php | 39 -------------- 5 files changed, 60 insertions(+), 46 deletions(-) rename src/Scheduler/{DrupalReleaseSchedule.php => DefaultSchedule.php} (62%) create mode 100644 tests/Scheduler/DefaultScheduleTest.php delete mode 100644 tests/Scheduler/DrupalReleaseScheduleTest.php diff --git a/composer.json b/composer.json index 4bd02797..01ef105c 100644 --- a/composer.json +++ b/composer.json @@ -139,7 +139,7 @@ "bin/console hautelook:fixtures:load --no-interaction" ], "queues": [ - "bin/console messenger:consume async scheduler_drupal_releases scheduler_default --failure-limit=1 -vvv" + "bin/console messenger:consume async scheduler_default --failure-limit=1 -vvv" ], "tests": [ "bin/console --env=test doctrine:database:drop --if-exists --force --quiet", diff --git a/docker-compose.server.override.yml b/docker-compose.server.override.yml index da42b572..384113c1 100644 --- a/docker-compose.server.override.yml +++ b/docker-compose.server.override.yml @@ -24,7 +24,7 @@ services: restart: unless-stopped stop_grace_period: 20s environment: - - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_drupal_releases scheduler_default + - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_default - APP_SUPERVISOR_WORKERS=1 - APP_SUPERVISOR_USER=deploy - PHP_MEMORY_LIMIT=256M diff --git a/src/Scheduler/DrupalReleaseSchedule.php b/src/Scheduler/DefaultSchedule.php similarity index 62% rename from src/Scheduler/DrupalReleaseSchedule.php rename to src/Scheduler/DefaultSchedule.php index 01b13ce4..3c7f2e62 100644 --- a/src/Scheduler/DrupalReleaseSchedule.php +++ b/src/Scheduler/DefaultSchedule.php @@ -12,13 +12,14 @@ use Symfony\Contracts\Cache\CacheInterface; /** - * Check drupal.org releases every 12 hours. + * The app's one schedule. Tasks declared with #[AsPeriodicTask] or + * #[AsCronTask] join it, e.g. app:data:purge. * - * Anchored and stateful because the worker restarts every 15 minutes - * (--time-limit=900), which would otherwise reset the clock on each boot. + * Stateful because the worker restarts every 15 minutes (--time-limit=900), + * which would otherwise reset the clock on each boot and skip runs. */ -#[AsSchedule('drupal_releases')] -final readonly class DrupalReleaseSchedule implements ScheduleProviderInterface +#[AsSchedule] +final readonly class DefaultSchedule implements ScheduleProviderInterface { public function __construct( private CacheInterface $cache, @@ -28,6 +29,7 @@ public function __construct( public function getSchedule(): Schedule { return new Schedule() + // Check drupal.org releases every 12 hours. ->add(RecurringMessage::every('12 hours', new CheckDrupalReleases(), from: '06:00')) ->stateful($this->cache); } diff --git a/tests/Scheduler/DefaultScheduleTest.php b/tests/Scheduler/DefaultScheduleTest.php new file mode 100644 index 00000000..3eaeb858 --- /dev/null +++ b/tests/Scheduler/DefaultScheduleTest.php @@ -0,0 +1,51 @@ +get('messenger.receiver_locator'); + + $this->assertTrue($receivers->has('scheduler_default')); + $this->assertFalse($receivers->has('scheduler_drupal_releases')); + } + + public function testIsStatefulAndHoldsTheDrupalCheckAndThePurge(): void + { + self::bootKernel(); + /** @var Schedule $schedule */ + $schedule = self::getContainer()->get('scheduler.provider.default'); + + // Stateful, so the worker's 15 minute restarts do not skip runs. + $this->assertNotNull($schedule->getState()); + + $messages = []; + foreach ($schedule->getRecurringMessages() as $message) { + $messages[$this->messageOf($message)::class] = $message; + } + $this->assertEqualsCanonicalizing([CheckDrupalReleases::class, RunCommandMessage::class], array_keys($messages)); + + $trigger = $messages[CheckDrupalReleases::class]->getTrigger(); + $this->assertEquals(new \DateTimeImmutable('18:00'), $trigger->getNextRunDate(new \DateTimeImmutable('07:00'))); + $this->assertEquals(new \DateTimeImmutable('tomorrow 06:00'), $trigger->getNextRunDate(new \DateTimeImmutable('19:00'))); + } + + private function messageOf(RecurringMessage $message): object + { + $context = new MessageContext('default', $message->getId(), $message->getTrigger(), new \DateTimeImmutable()); + + return iterator_to_array($message->getMessages($context))[0]; + } +} diff --git a/tests/Scheduler/DrupalReleaseScheduleTest.php b/tests/Scheduler/DrupalReleaseScheduleTest.php deleted file mode 100644 index 224cc2d6..00000000 --- a/tests/Scheduler/DrupalReleaseScheduleTest.php +++ /dev/null @@ -1,39 +0,0 @@ -assertTrue(self::getContainer()->get('messenger.receiver_locator')->has('scheduler_drupal_releases')); - } - - public function testRecursEvery12HoursFrom0600AndIsStateful(): void - { - self::bootKernel(); - $schedule = self::getContainer()->get(DrupalReleaseSchedule::class)->getSchedule(); - - $this->assertNotNull($schedule->getState()); - - $messages = $schedule->getRecurringMessages(); - $this->assertCount(1, $messages); - $message = $messages[0]; - $trigger = $message->getTrigger(); - - $context = new MessageContext('drupal_releases', $message->getId(), $trigger, new \DateTimeImmutable()); - $this->assertInstanceOf(CheckDrupalReleases::class, iterator_to_array($message->getMessages($context))[0]); - - $this->assertEquals(new \DateTimeImmutable('18:00'), $trigger->getNextRunDate(new \DateTimeImmutable('07:00'))); - $this->assertEquals(new \DateTimeImmutable('tomorrow 06:00'), $trigger->getNextRunDate(new \DateTimeImmutable('19:00'))); - } -} From d2911b1e956f19288ef9e8a1a7edea018d6869ba Mon Sep 17 00:00:00 2001 From: turegjorup Date: Wed, 21 Oct 2026 12:46:36 +0200 Subject: [PATCH 5/6] chore: drop --failure-limit=1 from the workers The worker resets services after each message, which replaces a closed EntityManager, and failed messages are retried and then sent to the failed transport. Stopping on every failure only restarted the worker. --- README.md | 2 +- Taskfile.yml | 2 +- composer.json | 2 +- docker-compose.server.override.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 06197c5e..3c9d0817 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ docker compose exec phpfpm composer queues or ```shell -docker compose exec phpfpm bin/console messenger:consume async --failure-limit=1 -vvv +docker compose exec phpfpm bin/console messenger:consume async -vvv ``` ### Assets diff --git a/Taskfile.yml b/Taskfile.yml index c17a696f..a4807ea8 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -59,7 +59,7 @@ tasks: cmds: - task: console vars: - CONSOLE_ARGS: messenger:consume async --failure-limit=1 -vvv + CONSOLE_ARGS: messenger:consume async -vvv composer: desc: "Run `composer` command. Example: task composer -- normalize" diff --git a/composer.json b/composer.json index 01ef105c..053673da 100644 --- a/composer.json +++ b/composer.json @@ -139,7 +139,7 @@ "bin/console hautelook:fixtures:load --no-interaction" ], "queues": [ - "bin/console messenger:consume async scheduler_default --failure-limit=1 -vvv" + "bin/console messenger:consume async scheduler_default -vvv" ], "tests": [ "bin/console --env=test doctrine:database:drop --if-exists --force --quiet", diff --git a/docker-compose.server.override.yml b/docker-compose.server.override.yml index 384113c1..994958e4 100644 --- a/docker-compose.server.override.yml +++ b/docker-compose.server.override.yml @@ -24,7 +24,7 @@ services: restart: unless-stopped stop_grace_period: 20s environment: - - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_default + - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 async scheduler_default - APP_SUPERVISOR_WORKERS=1 - APP_SUPERVISOR_USER=deploy - PHP_MEMORY_LIMIT=256M From 8d03f65d57bced827461f6ad9a97d557f1459acd Mon Sep 17 00:00:00 2001 From: turegjorup Date: Wed, 21 Oct 2026 12:47:10 +0200 Subject: [PATCH 6/6] docs: add changelog entry for #167 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fae2c595..4c5d1e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- [#167](https://github.com/itk-dev/devops_itksites/pull/167) + Let the latest dir result decide which sites exist, so replays no longer + revive removed sites or delete live ones, and purge detection results not + seen for 30 days every night - [#166](https://github.com/itk-dev/devops_itksites/pull/166) Update EasyAdmin to 5.7, which adds the UI Kit at `/admin/_ea_ui_kit` in debug mode