8669: Clean up removed sites and purge old detection results

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- [#167](https://github.com/itk-dev/devops_itksites/pull/167)
+  Let the latest dir result decide which sites exist, so replays no longer
+  revive removed sites or delete live ones, and purge detection results not
+  seen for 30 days every night
 - [#166](https://github.com/itk-dev/devops_itksites/pull/166)
   Update EasyAdmin to 5.7, which adds the UI Kit at `/admin/_ea_ui_kit` in
   debug mode

README.md

@@ -201,7 +201,7 @@ docker compose exec phpfpm composer queues
 or
 
 ```shell
-docker compose exec phpfpm bin/console messenger:consume async --failure-limit=1 -vvv
+docker compose exec phpfpm bin/console messenger:consume async -vvv
 ```
 
 ### Assets

Taskfile.yml

@@ -59,7 +59,7 @@ tasks:
         cmds:
             - task: console
               vars:
-                  CONSOLE_ARGS: messenger:consume async --failure-limit=1 -vvv
+                  CONSOLE_ARGS: messenger:consume async -vvv
 
     composer:
         desc: "Run `composer` command. Example: task composer -- normalize"

composer.json

@@ -139,7 +139,7 @@
             "bin/console hautelook:fixtures:load --no-interaction"
         ],
         "queues": [
-            "bin/console messenger:consume async scheduler_drupal_releases --failure-limit=1 -vvv"
+            "bin/console messenger:consume async scheduler_default -vvv"
         ],
         "tests": [
             "bin/console --env=test doctrine:database:drop --if-exists --force --quiet",

docker-compose.server.override.yml

@@ -24,7 +24,7 @@ services:
     restart: unless-stopped
     stop_grace_period: 20s
     environment:
-      - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 --failure-limit=1 async scheduler_drupal_releases
+      - APP_SUPERVISOR_COMMAND=/app/bin/console messenger:consume --env=prod --no-debug --time-limit=900 async scheduler_default
       - APP_SUPERVISOR_WORKERS=1
       - APP_SUPERVISOR_USER=deploy
       - PHP_MEMORY_LIMIT=256M

phpstan.dist.neon

@@ -23,4 +23,3 @@ parameters:
                 - src/Entity/
                 - src/Form/
                 - src/Service/
-                - src/Handler/

src/Command/PurgeCommand.php

@@ -11,11 +11,13 @@
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\Scheduler\Attribute\AsPeriodicTask;
 
 #[AsCommand(
     name: 'app:data:purge',
     description: 'Purge old data',
 )]
+#[AsPeriodicTask(frequency: '1 day', from: '03:00', jitter: 1800, arguments: '--date="30 days ago"')]
 class PurgeCommand extends Command
 {
     public function __construct(
@@ -29,17 +31,17 @@ public function __construct(
      */
     protected function configure(): void
     {
-        $this->addOption('date', null, InputOption::VALUE_REQUIRED, 'Remove results older that this date (Y-m-d)');
+        $this->addOption('date', null, InputOption::VALUE_REQUIRED, 'Remove results not seen since this date, e.g. "2026-09-21" or "30 days ago"');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $io = new SymfonyStyle($input, $output);
 
-        $format = 'Y-m-d';
         $date = (string) $input->getOption('date');
-        $dateTime = \DateTime::createFromFormat($format, $date);
-        if (false === $dateTime || $dateTime->format($format) !== $date) {
+        try {
+            $dateTime = new \DateTime($date);
+        } catch (\DateMalformedStringException) {
             $output->writeln('<error>Unknown date format:</error> '.$date);
 
             return Command::FAILURE;

src/Handler/DirectoryHandler.php

@@ -10,7 +10,9 @@
 use App\Repository\SiteRepository;
 use App\Service\InstallationFactory;
 use App\Types\DetectionType;
+use App\Utils\RootDirNormalizer;
 use Doctrine\Common\Collections\Collection;
+use Doctrine\ORM\EntityManagerInterface;
 
 /**
  * Handler for DetectionResult off type "dir" (Installations).
@@ -24,34 +26,38 @@ public function __construct(
         private SiteRepository $siteRepository,
         private InstallationFactory $installationFactory,
         private DetectionResultRepository $detectionResultRepository,
+        private EntityManagerInterface $entityManager,
     ) {
     }
 
     public function handleResult(DetectionResult $detectionResult): void
     {
         $server = $detectionResult->getServer();
-        $oldInstallations = $server->getInstallations();
+
+        // Only the latest dir result describes the server. Older ones (e.g. from a replay) are history.
+        $latest = $this->detectionResultRepository->findOneBy(['server' => $server, 'type' => DetectionType::DIRECTORY], ['lastContact' => 'DESC']);
+        if ($latest !== $detectionResult) {
+            return;
+        }
 
         $installations = $this->installationFactory->getInstallations($detectionResult);
 
-        /** @var Installation $installation */
-        foreach ($installations as $installation) {
-            $sites = $this->siteRepository->findByRootDirAndServer(
-                $installation->getRootDir(),
-                $server
-            );
-            /* @var Site $site */
-            foreach ($sites as $site) {
+        // Sites belong to the listed dir they are in. Sites outside every listed dir are gone.
+        foreach ($this->siteRepository->findBy(['server' => $server]) as $site) {
+            $installation = $this->findEnclosing($installations, $site->getRootDir());
+            if (null === $installation) {
+                $this->entityManager->remove($site);
+            } else {
                 $installation->addSite($site);
             }
         }
 
-        $detectionResult->getServer()->setInstallations($installations);
+        $server->setInstallations($installations);
 
-        // Delete results from installations no longer seen on the server.
-        foreach ($oldInstallations as $oldInstallation) {
-            if (!$installations->contains($oldInstallation)) {
-                $this->detectionResultRepository->deleteByInstallation($oldInstallation);
+        // Delete results from dirs no longer listed, so a replay cannot recreate them.
+        foreach ($this->detectionResultRepository->findBy(['server' => $server]) as $result) {
+            if (DetectionType::DIRECTORY !== $result->getType() && null === $this->findEnclosing($installations, $result->getRootDir())) {
+                $this->entityManager->remove($result);
             }
         }
     }
@@ -61,15 +67,23 @@ public function supportsType(string $type): bool
         return DetectionType::DIRECTORY === $type;
     }
 
-    private function getRemovedInstallations(Collection $oldInstallations, Collection $newInstallations): array
+    /**
+     * Find the installation for the root dir itself or, failing that, its closest parent dir.
+     *
+     * @param Collection<int, Installation> $installations
+     */
+    private function findEnclosing(Collection $installations, string $rootDir): ?Installation
     {
-        $result = [];
-        foreach ($oldInstallations as $installation) {
-            if (!$newInstallations->contains($installation)) {
-                $result[] = $installation;
+        $rootDir = RootDirNormalizer::normalize($rootDir);
+        $enclosing = null;
+        foreach ($installations as $installation) {
+            $dir = $installation->getRootDir();
+            if (($rootDir === $dir || str_starts_with($rootDir, $dir.'/'))
+                && strlen($dir) > strlen($enclosing?->getRootDir() ?? '')) {
+                $enclosing = $installation;
             }
         }
 
-        return $result;
+        return $enclosing;
     }
 }

src/MessageHandler/PersistDetectionResultHandler.php

@@ -8,6 +8,7 @@
 use App\Message\ProcessDetectionResult;
 use App\Repository\DetectionResultRepository;
 use App\Repository\ServerRepository;
+use App\Types\DetectionType;
 use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Component\Messenger\Attribute\AsMessageHandler;
 use Symfony\Component\Messenger\Exception\UnrecoverableMessageHandlingException;
@@ -58,6 +59,12 @@ public function __invoke(PersistDetectionResult $message): void
             $existingResult->setLastContact($message->receivedAt);
 
             $this->entityManager->flush();
+
+            // Except dir results: they decide which installations exist, so re-apply them to
+            // remove anything recreated since they were first processed (e.g. by a replay).
+            if (DetectionType::DIRECTORY === $existingResult->getType() && null !== $id = $existingResult->getId()) {
+                $this->messageBus->dispatch(new ProcessDetectionResult($id));
+            }
         }
 
         $this->entityManager->clear();

src/Repository/DetectionResultRepository.php

@@ -5,7 +5,6 @@
 namespace App\Repository;
 
 use App\Entity\DetectionResult;
-use App\Entity\Installation;
 use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
 use Doctrine\Persistence\ManagerRegistry;
 
@@ -89,18 +88,4 @@ public function cleanup(DetectionResult $detectionResult, int $keep = 5, bool $f
             $em->flush();
         }
     }
-
-    /**
-     * Delete all detection results for a given installation.
-     */
-    public function deleteByInstallation(Installation $installation): void
-    {
-        $qb = $this->getEntityManager()->createQueryBuilder();
-        $qb->delete(DetectionResult::class, 'd')
-            ->where('d.server = :server')
-            ->andWhere('d.rootDir =:rootDir')
-            ->setParameter('server', $installation->getServer())
-            ->setParameter('rootDir', $installation->getRootDir());
-        $qb->getQuery()->execute();
-    }
 }

src/Repository/SiteRepository.php

@@ -4,7 +4,6 @@
 
 namespace App\Repository;
 
-use App\Entity\Server;
 use App\Entity\Site;
 use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
 use Doctrine\ORM\AbstractQuery;
@@ -25,18 +24,6 @@ public function __construct(ManagerRegistry $registry)
         parent::__construct($registry, Site::class);
     }
 
-    public function findByRootDirAndServer(string $rootDir, Server $server): mixed
-    {
-        return $this->createQueryBuilder('s')
-            ->andWhere('s.rootDir LIKE :rootDir')
-            ->setParameter('rootDir', $rootDir.'%')
-            ->andWhere('s.server = :server')
-            ->setParameter('server', $server->getId()->toBinary())
-            ->getQuery()
-            ->getResult()
-        ;
-    }
-
     /**
      * Get unique primary domains from existing sites.
      */

src/Scheduler/DrupalReleaseSchedule.php → src/Scheduler/DefaultSchedule.php

@@ -12,13 +12,14 @@
 use Symfony\Contracts\Cache\CacheInterface;
 
 /**
- * Check drupal.org releases every 12 hours.
+ * The app's one schedule. Tasks declared with #[AsPeriodicTask] or
+ * #[AsCronTask] join it, e.g. app:data:purge.
  *
- * Anchored and stateful because the worker restarts every 15 minutes
- * (--time-limit=900), which would otherwise reset the clock on each boot.
+ * Stateful because the worker restarts every 15 minutes (--time-limit=900),
+ * which would otherwise reset the clock on each boot and skip runs.
  */
-#[AsSchedule('drupal_releases')]
-final readonly class DrupalReleaseSchedule implements ScheduleProviderInterface
+#[AsSchedule]
+final readonly class DefaultSchedule implements ScheduleProviderInterface
 {
     public function __construct(
         private CacheInterface $cache,
@@ -28,6 +29,7 @@ public function __construct(
     public function getSchedule(): Schedule
     {
         return new Schedule()
+            // Check drupal.org releases every 12 hours.
             ->add(RecurringMessage::every('12 hours', new CheckDrupalReleases(), from: '06:00'))
             ->stateful($this->cache);
     }

tests/Command/PurgeCommandTest.php

@@ -0,0 +1,78 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Tests\Command;
+
+use App\Entity\DetectionResult;
+use App\Entity\Server;
+use App\Types\DetectionType;
+use Doctrine\ORM\EntityManagerInterface;
+use Hautelook\AliceBundle\PhpUnit\RefreshDatabaseTrait;
+use Symfony\Bundle\FrameworkBundle\Console\Application;
+use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
+use Symfony\Component\Console\Tester\ApplicationTester;
+
+class PurgeCommandTest extends KernelTestCase
+{
+    use RefreshDatabaseTrait;
+
+    private const string ROOT_DIR = '/data/www/purge_probe/htdocs';
+
+    public function testPurgesResultsOlderThanARelativeDate(): void
+    {
+        self::bootKernel();
+        $entityManager = self::getContainer()->get(EntityManagerInterface::class);
+        $server = $entityManager->getRepository(Server::class)->findOneBy([]);
+        foreach (['-31 days' => 'old', '-29 days' => 'recent'] as $lastContact => $data) {
+            $result = new DetectionResult();
+            $result->setType(DetectionType::DOCKER);
+            $result->setRootDir(self::ROOT_DIR);
+            $result->setServer($server);
+            $result->setData($data);
+            $result->generateHash();
+            $result->setLastContact(new \DateTimeImmutable($lastContact));
+            $entityManager->persist($result);
+        }
+        $entityManager->flush();
+
+        $tester = $this->console(['--date' => '30 days ago']);
+
+        $tester->assertCommandIsSuccessful();
+        $entityManager->clear();
+        $remaining = $entityManager->getRepository(DetectionResult::class)->findBy(['rootDir' => self::ROOT_DIR]);
+        $this->assertSame(['recent'], array_map(static fn (DetectionResult $result): string => $result->getData(), $remaining));
+    }
+
+    public function testRejectsAnUnparsableDate(): void
+    {
+        self::bootKernel();
+
+        $this->console(['--date' => 'not a date'])->assertCommandFailed();
+    }
+
+    public function testRunsDailyOnTheDefaultSchedule(): void
+    {
+        self::bootKernel();
+
+        $tester = $this->console(['command' => 'debug:scheduler', 'schedule' => ['default']]);
+
+        $tester->assertCommandIsSuccessful();
+        $this->assertStringContainsString('every 1 day with 0-1800 second jitter', $tester->getDisplay());
+        $this->assertStringContainsString('app:data:purge --date="30 days ago"', $tester->getDisplay());
+    }
+
+    /**
+     * @param array<string, mixed> $input
+     */
+    private function console(array $input): ApplicationTester
+    {
+        $application = new Application(self::$kernel);
+        $application->setAutoExit(false);
+
+        $tester = new ApplicationTester($application);
+        $tester->run($input + ['command' => 'app:data:purge']);
+
+        return $tester;
+    }
+}