Fix broken remove past data

Past data is now removed again.
A test ensures the functionality.
Data is no longer marked as deleted but is actually deleted.

Relates: #9543

Classes/Service/Cleanup/Database.php

@@ -24,7 +24,6 @@ namespace Wrm\Events\Service\Cleanup;
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Database\Query\QueryBuilder;
-use TYPO3\CMS\Core\DataHandling\DataHandler;
 
 class Database
 {
@@ -33,21 +32,14 @@ class Database
      */
     private $connectionPool;
 
-    /**
-     * @var DataHandler
-     */
-    private $dataHandler;
-
     private const DATE_TABLE = 'tx_events_domain_model_date';
     private const EVENT_TABLE = 'tx_events_domain_model_event';
     private const ORGANIZER_TABLE = 'tx_events_domain_model_organizer';
 
     public function __construct(
-        ConnectionPool $connectionPool,
+        ConnectionPool $connectionPool
-        DataHandler $dataHandler
     ) {
         $this->connectionPool = $connectionPool;
-        $this->dataHandler = $dataHandler;
     }
 
     public function truncateTables(): void
@@ -72,7 +64,7 @@ class Database
             ->execute();
     }
 
-    public function getPastDates(): array
+    public function deletePastDates(): void
     {
         $queryBuilder = $this->connectionPool
             ->getConnectionForTable(self::DATE_TABLE)
@@ -81,32 +73,11 @@ class Database
         $queryBuilder->getRestrictions()->removeAll();
 
         $midnightToday = new \DateTimeImmutable('midnight today');
-        $records = $queryBuilder->select('uid')
+        $queryBuilder->delete(self::DATE_TABLE)
-            ->from(self::DATE_TABLE)
             ->where($queryBuilder->expr()->lte(
                 'end',
-                $queryBuilder->createNamedParameter($midnightToday->format('Y-m-d H:i:s'))
+                $queryBuilder->createNamedParameter($midnightToday->format('U'))
             ))
-            ->execute()
-            ->fetchAll();
-
-        return array_map(function ($record) {
-            if (is_array($record) === false) {
-                return '';
-            }
-
-            return $record['uid'];
-        }, $records);
-    }
-
-    public function deleteDates(int ...$uids): void
-    {
-        $queryBuilder = $this->connectionPool
-            ->getQueryBuilderForTable(self::DATE_TABLE);
-
-        $queryBuilder->delete(self::DATE_TABLE)
-            ->where('uid in (:uids)')
-            ->setParameter(':uids', $uids, Connection::PARAM_INT_ARRAY)
             ->execute();
     }
 
@@ -125,14 +96,13 @@ class Database
             ->execute()
             ->fetchAll(\PDO::FETCH_COLUMN);
 
-        $dataStructure = [self::EVENT_TABLE => []];
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable(self::EVENT_TABLE);
-        foreach ($recordUids as $recordUid) {
+        $queryBuilder->delete(self::EVENT_TABLE);
-            $dataStructure[self::EVENT_TABLE][$recordUid] = ['delete' => 1];
+        $queryBuilder->where($queryBuilder->expr()->in(
-        }
+            'uid',
-
+            $queryBuilder->createNamedParameter($recordUids, Connection::PARAM_INT_ARRAY)
-        $dataHandler = clone $this->dataHandler;
+        ));
-        $dataHandler->start([], $dataStructure);
+        $queryBuilder->execute();
-        $dataHandler->process_cmdmap();
 
         $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_category_record_mm');
         $queryBuilder->delete('sys_category_record_mm')

Classes/Service/CleanupService.php

@@ -31,7 +31,7 @@ class CleanupService
 
     public function deletePastData(): void
     {
-        $this->database->deleteDates(...$this->database->getPastDates());
+        $this->database->deletePastDates();
         $this->database->deleteEventsWithoutDates();
         $this->files->deleteDangling();
     }

Tests/Functional/Cleanup/Fixtures/RemovePastTest.xml

@@ -0,0 +1,297 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <pages>
+        <pid>0</pid>
+        <uid>1</uid>
+        <title>Root page</title>
+        <slug>1</slug>
+    </pages>
+    <pages>
+        <pid>1</pid>
+        <uid>2</uid>
+        <title>Storage</title>
+        <doktype>254</doktype>
+    </pages>
+
+    <sys_category>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Category 1</title>
+    </sys_category>
+    <sys_category>
+        <uid>2</uid>
+        <pid>2</pid>
+        <title>Example Category 2</title>
+    </sys_category>
+
+    <sys_category_record_mm>
+        <uid_local>1</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+    </sys_category_record_mm>
+    <sys_category_record_mm>
+        <uid_local>2</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+    </sys_category_record_mm>
+
+    <sys_file_storage>
+        <uid>1</uid>
+        <pid>0</pid>
+        <tstamp>1423209858</tstamp>
+        <crdate>1370878372</crdate>
+        <cruser_id>0</cruser_id>
+        <deleted>0</deleted>
+        <name>fileadmin/ (auto-created)</name>
+        <description>This is the local fileadmin/ directory. This storage mount has been created automatically by TYPO3.</description>
+        <driver>Local</driver>
+        <configuration><![CDATA[<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
+            <T3FlexForms>
+                <data>
+                    <sheet index="sDEF">
+                        <language index="lDEF">
+                            <field index="basePath">
+                                <value index="vDEF">fileadmin/</value>
+                            </field>
+                            <field index="pathType">
+                                <value index="vDEF">relative</value>
+                            </field>
+                            <field index="caseSensitive">
+                                <value index="vDEF">1</value>
+                            </field>
+                        </language>
+                    </sheet>
+                </data>
+        </T3FlexForms>]]></configuration>
+        <is_browsable>1</is_browsable>
+        <is_public>1</is_public>
+        <is_writable>1</is_writable>
+        <is_online>1</is_online>
+        <processingfolder>_processed_</processingfolder>
+        <is_default>1</is_default>
+        <auto_extract_metadata>1</auto_extract_metadata>
+    </sys_file_storage>
+
+    <sys_file>
+        <uid>1</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <type>2</type>
+        <storage>1</storage>
+        <identifier>/user_uploads/example-for-event.gif</identifier>
+        <extension>gif</extension>
+        <mime_type>image/gif</mime_type>
+        <name>ext_icon.gif</name>
+        <sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
+        <size>637</size>
+        <creation_date>1370877201</creation_date>
+        <modification_date>1369407629</modification_date>
+        <last_indexed>0</last_indexed>
+        <missing>0</missing>
+        <metadata>0</metadata>
+        <identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
+        <folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
+    </sys_file>
+    <sys_file>
+        <uid>2</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <type>2</type>
+        <storage>1</storage>
+        <identifier>/user_uploads/example-for-partner.gif</identifier>
+        <extension>gif</extension>
+        <mime_type>image/gif</mime_type>
+        <name>ext_icon.gif</name>
+        <sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
+        <size>637</size>
+        <creation_date>1370877201</creation_date>
+        <modification_date>1369407629</modification_date>
+        <last_indexed>0</last_indexed>
+        <missing>0</missing>
+        <metadata>0</metadata>
+        <identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
+        <folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
+    </sys_file>
+    <sys_file>
+        <uid>3</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <type>2</type>
+        <storage>1</storage>
+        <identifier>/user_uploads/example-for-future-event.gif</identifier>
+        <extension>gif</extension>
+        <mime_type>image/gif</mime_type>
+        <name>ext_icon.gif</name>
+        <sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
+        <size>637</size>
+        <creation_date>1370877201</creation_date>
+        <modification_date>1369407629</modification_date>
+        <last_indexed>0</last_indexed>
+        <missing>0</missing>
+        <metadata>0</metadata>
+        <identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
+        <folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
+    </sys_file>
+
+    <sys_file_metadata>
+        <uid>1</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <crdate>1371467047</crdate>
+        <cruser_id>1</cruser_id>
+        <file>1</file>
+    </sys_file_metadata>
+    <sys_file_metadata>
+        <uid>2</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <crdate>1371467047</crdate>
+        <cruser_id>1</cruser_id>
+        <file>2</file>
+    </sys_file_metadata>
+    <sys_file_metadata>
+        <uid>3</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <crdate>1371467047</crdate>
+        <cruser_id>1</cruser_id>
+        <file>3</file>
+    </sys_file_metadata>
+
+    <sys_file_reference>
+        <uid>1</uid>
+        <pid>2</pid>
+        <tstamp>1373537480</tstamp>
+        <crdate>1371484347</crdate>
+        <cruser_id>1</cruser_id>
+        <deleted>0</deleted>
+        <hidden>0</hidden>
+        <sys_language_uid>0</sys_language_uid>
+        <uid_local>1</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+        <fieldname>images</fieldname>
+        <sorting_foreign>1</sorting_foreign>
+        <table_local>sys_file</table_local>
+    </sys_file_reference>
+    <sys_file_reference>
+        <uid>2</uid>
+        <pid>2</pid>
+        <tstamp>1373537480</tstamp>
+        <crdate>1371484347</crdate>
+        <cruser_id>1</cruser_id>
+        <deleted>0</deleted>
+        <hidden>0</hidden>
+        <sys_language_uid>0</sys_language_uid>
+        <uid_local>2</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_partner</tablenames>
+        <fieldname>images</fieldname>
+        <sorting_foreign>1</sorting_foreign>
+        <table_local>sys_file</table_local>
+    </sys_file_reference>
+    <sys_file_reference>
+        <uid>3</uid>
+        <pid>2</pid>
+        <tstamp>1373537480</tstamp>
+        <crdate>1371484347</crdate>
+        <cruser_id>1</cruser_id>
+        <deleted>0</deleted>
+        <hidden>0</hidden>
+        <sys_language_uid>0</sys_language_uid>
+        <uid_local>3</uid_local>
+        <uid_foreign>2</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+        <fieldname>images</fieldname>
+        <sorting_foreign>1</sorting_foreign>
+        <table_local>sys_file</table_local>
+    </sys_file_reference>
+
+    <tx_events_domain_model_region>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Region</title>
+    </tx_events_domain_model_region>
+
+    <tx_events_domain_model_partner>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Partner</title>
+        <link>https://example.com</link>
+        <images>1</images>
+    </tx_events_domain_model_partner>
+
+    <tx_events_domain_model_organizer>
+        <uid>1</uid>
+        <pid>2</pid>
+        <name>Example Organizer</name>
+        <street>Example Street 17</street>
+        <district></district>
+        <city>Example Town</city>
+        <zip>00101</zip>
+        <phone>+49 2161 56 36 27 37 48 94 28</phone>
+        <web>https://example.com</web>
+        <email>someone@example.com</email>
+    </tx_events_domain_model_organizer>
+
+    <tx_events_domain_model_event>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Event</title>
+        <subtitle>Some further info about event</subtitle>
+        <global_id>5540-34</global_id>
+        <slug>5540-34</slug>
+        <organizer>1</organizer>
+        <partner>1</partner>
+        <region>1</region>
+        <images>0</images>
+        <categories>2</categories>
+    </tx_events_domain_model_event>
+    <tx_events_domain_model_date>
+        <uid>1</uid>
+        <pid>2</pid>
+        <event>1</event>
+        <start>0</start>
+        <end>10</end>
+        <canceled>no</canceled>
+    </tx_events_domain_model_date>
+
+    <tx_events_domain_model_event>
+        <uid>2</uid>
+        <pid>2</pid>
+        <title>Example Future Event</title>
+        <subtitle>Some further info about event</subtitle>
+        <global_id>5540-33</global_id>
+        <slug>5540-33</slug>
+        <organizer>2</organizer>
+        <partner>1</partner>
+        <region>1</region>
+        <images>0</images>
+        <categories>1</categories>
+    </tx_events_domain_model_event>
+    <tx_events_domain_model_date>
+        <uid>3</uid>
+        <pid>2</pid>
+        <event>2</event>
+        <start>9999999998</start>
+        <end>9999999999</end>
+        <canceled>no</canceled>
+    </tx_events_domain_model_date>
+    <tx_events_domain_model_organizer>
+        <uid>2</uid>
+        <pid>2</pid>
+        <name>Example Organizer for future event</name>
+        <street>Example Street 17</street>
+        <district></district>
+        <city>Example Town</city>
+        <zip>00101</zip>
+        <phone>+49 2161 56 36 27 37 48 94 28</phone>
+        <web>https://example.com</web>
+        <email>someone@example.com</email>
+    </tx_events_domain_model_organizer>
+    <sys_category_record_mm>
+        <uid_local>2</uid_local>
+        <uid_foreign>2</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+    </sys_category_record_mm>
+</dataset>

Tests/Functional/Cleanup/RemoveAllTest.php

@@ -18,7 +18,7 @@ class RemoveAllTest extends FunctionalTestCase
     ];
 
     protected $pathsToProvideInTestInstance = [
-        'typo3conf/ext/events/Tests/Functional/Cleanup/Fixtures/fileadmin/' => 'fileadmin/',
+        'typo3conf/ext/events/Tests/Functional/Cleanup/Fixtures/RemoveAllTestFileadmin/' => 'fileadmin/',
     ];
 
     protected function setUp(): void

Tests/Functional/Cleanup/RemovePastTest.php

@@ -0,0 +1,100 @@
+<?php
+
+namespace Wrm\Events\Tests\Functional\Cleanup;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Tester\CommandTester;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase;
+use Wrm\Events\Command\RemovePastCommand;
+
+/**
+ * @testdox Cleanup RemovePast
+ */
+class RemovePastTest extends FunctionalTestCase
+{
+    protected $testExtensionsToLoad = [
+        'typo3conf/ext/events',
+    ];
+
+    protected $pathsToProvideInTestInstance = [
+        'typo3conf/ext/events/Tests/Functional/Cleanup/Fixtures/RemovePastTestFileadmin/' => 'fileadmin/',
+    ];
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+
+        $this->setUpBackendUserFromFixture(1);
+    }
+
+    /**
+     * @test
+     */
+    public function removesPastData(): void
+    {
+        $this->importDataSet('EXT:events/Tests/Functional/Cleanup/Fixtures/RemovePastTest.xml');
+
+        $subject = $this->getContainer()->get(RemovePastCommand::class);
+        self::assertInstanceOf(Command::class, $subject);
+
+        $tester = new CommandTester($subject);
+        $tester->execute([], ['capture_stderr_separately' => true]);
+
+        self::assertSame(0, $tester->getStatusCode());
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_partner'),
+            'Partners are not kept.'
+        );
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_region'),
+            'Regions are not kept.'
+        );
+        self::assertCount(
+            2,
+            $this->getAllRecords('tx_events_domain_model_organizer'),
+            'Organizers are not kept.'
+        );
+
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_event'),
+            'Events are still there.'
+        );
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_date'),
+            'Dates are still there.'
+        );
+
+        self::assertCount(
+            1,
+            $this->getAllRecords('sys_category_record_mm'),
+            'Relations to categories still exist.'
+        );
+
+        self::assertCount(
+            2,
+            $this->getAllRecords('sys_file'),
+            'Unexpected number of sys_file records.'
+        );
+        self::assertCount(
+            2,
+            $this->getAllRecords('sys_file_reference'),
+            'Unexpected number of sys_file_reference records.'
+        );
+        self::assertCount(
+            2,
+            $this->getAllRecords('sys_file_metadata'),
+            'Unexpected number of sys_file_metadata records.'
+        );
+
+        $files = GeneralUtility::getFilesInDir('fileadmin/user_uploads');
+        self::assertIsArray($files, 'Failed to retrieve files from filesystem.');
+        self::assertCount(2, $files, 'Unexpectd number of files in filesystem.');
+        self::assertSame('example-for-future-event.gif', array_values($files)[0], 'Unexpected file in filesystem.');
+        self::assertSame('example-for-partner.gif', array_values($files)[1], 'Unexpected file in filesystem.');
+    }
+}