Delete files which were referenced by events objects

Adjust existing code base to not use hardcoded file path.
Instead use info from database and check all files which have references
to event objects.

Also add test to cover feature.

Classes/Service/Cleanup/Files.php

@@ -46,57 +46,88 @@ class Files
         $this->storageRepository = $storageRepository;
     }
 
-    public function deleteAll(): void
-    {
-        $this->delete($this->getFilesFromDb());
-    }
-
     public function deleteDangling(): void
     {
-        $this->delete($this->getFilesFromDb(function (QueryBuilder $queryBuilder) {
+        $this->markFileReferencesDeletedIfForeignRecordIsMissing();
-            $queryBuilder->leftJoin(
+        $this->deleteFilesWithoutProperReference();
+    }
+
+    private function markFileReferencesDeletedIfForeignRecordIsMissing(): void
+    {
+        $referencesQuery = $this->connectionPool
+            ->getQueryBuilderForTable('sys_file_reference');
+        $referencesQuery->getRestrictions()->removeAll();
+        $referencesQuery->select(
+            'uid',
+            'uid_foreign',
+            'tablenames'
+        );
+        $referencesQuery->from('sys_file_reference');
+        $referencesQuery->where(
+            $referencesQuery->expr()->like(
+                'tablenames',
+                $referencesQuery->createNamedParameter('tx_events_domain_model_%')
+            )
+        );
+        $referencesQuery->orderBy('tablenames');
+        $referencesQuery->addOrderBy('uid_foreign');
+
+        $references = $referencesQuery->execute();
+
+        $uidsPerTable = [];
+        $referenceUidsToMarkAsDeleted = [];
+
+        while ($reference = $references->fetch()) {
+            if (is_array($reference) === false) {
+                continue;
+            }
+
+            $uidsPerTable[(string)$reference['tablenames']][$reference['uid']] = $reference['uid_foreign'];
+        }
+
+        foreach ($uidsPerTable as $tableName => $records) {
+            $queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName);
+            $queryBuilder->getRestrictions()->removeAll();
+            $queryBuilder->select('uid');
+            $queryBuilder->from($tableName);
+            $queryBuilder->where($queryBuilder->expr()->in('uid', $records));
+            $referenceUidsToMarkAsDeleted = array_merge(
+                $referenceUidsToMarkAsDeleted,
+                array_keys(array_diff($records, $queryBuilder->execute()->fetchAll(\PDO::FETCH_COLUMN)))
+            );
+        }
+
+        if ($referenceUidsToMarkAsDeleted === []) {
+            return;
+        }
+
+        $updateQuery = $this->connectionPool->getQueryBuilderForTable('sys_file_reference');
+        $updateQuery->update('sys_file_reference');
+        $updateQuery->where($updateQuery->expr()->in('uid', $referenceUidsToMarkAsDeleted));
+        $updateQuery->set('deleted', '1');
+        $updateQuery->execute();
+    }
+
+    private function deleteFilesWithoutProperReference(): void
+    {
+        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
+        $queryBuilder->getRestrictions()->removeAll();
+        $queryBuilder->select('file.identifier', 'file.storage', 'file.uid')
+            ->from('sys_file', 'file')
+            ->leftJoin(
                 'file',
                 'sys_file_reference',
                 'reference',
-                $queryBuilder->expr()->eq('file.uid', $queryBuilder->quoteIdentifier('reference.uid_local'))
+                'reference.uid_local = file.uid'
-            );
+            )
-            $queryBuilder->andWhere(
+            ->where($queryBuilder->expr()->eq('reference.deleted', 1));
-                $queryBuilder->expr()->orX(
+        /** @var array{int: array{storage: int, identifier: string, uid: int}} $filesToDelete */
-                    $queryBuilder->expr()->isNull('reference.uid'),
+        $filesToDelete = $queryBuilder->execute()->fetchAll();
-                    $queryBuilder->expr()->eq('reference.deleted', 1),
-                    $queryBuilder->expr()->eq('reference.hidden', 1)
-                )
-            );
-        }));
-    }
 
-    private function getFilesFromDb(callable $whereGenerator = null): array
-    {
-        $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
-
-        $queryBuilder->getRestrictions()->removeAll();
-
-        $queryBuilder->select('file.identifier', 'file.storage', 'file.uid')
-            ->from('sys_file', 'file')
-            ->where($queryBuilder->expr()->like(
-                'file.identifier',
-                $queryBuilder->createNamedParameter('/staedte/%/events/%')
-            ));
-
-        if ($whereGenerator !== null) {
-            $whereGenerator($queryBuilder);
-        }
-
-        return $queryBuilder->execute()->fetchAll();
-    }
-
-    private function delete(array $filesToDelete): void
-    {
         $uidsToRemove = [];
-
         foreach ($filesToDelete as $fileToDelete) {
-            $this->deleteFromFal($fileToDelete['storage'], $fileToDelete['identifier']);
+            $this->deleteFromFal((int) $fileToDelete['storage'], (string) $fileToDelete['identifier']);
-            $uidsToRemove[] = $fileToDelete['uid'];
+            $uidsToRemove[] = (int) $fileToDelete['uid'];
         }
 
         $this->deleteFromDb(...$uidsToRemove);

Classes/Service/CleanupService.php

@@ -26,7 +26,7 @@ class CleanupService
     public function deleteAllData(): void
     {
         $this->database->truncateTables();
-        $this->files->deleteAll();
+        $this->files->deleteDangling();
     }
 
     public function deletePastData(): void

Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml

@@ -0,0 +1,224 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <pages>
+        <pid>0</pid>
+        <uid>1</uid>
+        <title>Root page</title>
+        <slug>1</slug>
+    </pages>
+    <pages>
+        <pid>1</pid>
+        <uid>2</uid>
+        <title>Storage</title>
+        <doktype>254</doktype>
+    </pages>
+
+    <sys_category>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Category 1</title>
+    </sys_category>
+    <sys_category>
+        <uid>2</uid>
+        <pid>2</pid>
+        <title>Example Category 2</title>
+    </sys_category>
+
+    <sys_category_record_mm>
+        <uid_local>1</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+    </sys_category_record_mm>
+    <sys_category_record_mm>
+        <uid_local>2</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+    </sys_category_record_mm>
+
+    <sys_file_storage>
+        <uid>1</uid>
+        <pid>0</pid>
+        <tstamp>1423209858</tstamp>
+        <crdate>1370878372</crdate>
+        <cruser_id>0</cruser_id>
+        <deleted>0</deleted>
+        <name>fileadmin/ (auto-created)</name>
+        <description>This is the local fileadmin/ directory. This storage mount has been created automatically by TYPO3.</description>
+        <driver>Local</driver>
+        <configuration><![CDATA[<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
+            <T3FlexForms>
+                <data>
+                    <sheet index="sDEF">
+                        <language index="lDEF">
+                            <field index="basePath">
+                                <value index="vDEF">fileadmin/</value>
+                            </field>
+                            <field index="pathType">
+                                <value index="vDEF">relative</value>
+                            </field>
+                            <field index="caseSensitive">
+                                <value index="vDEF">1</value>
+                            </field>
+                        </language>
+                    </sheet>
+                </data>
+        </T3FlexForms>]]></configuration>
+        <is_browsable>1</is_browsable>
+        <is_public>1</is_public>
+        <is_writable>1</is_writable>
+        <is_online>1</is_online>
+        <processingfolder>_processed_</processingfolder>
+        <is_default>1</is_default>
+        <auto_extract_metadata>1</auto_extract_metadata>
+    </sys_file_storage>
+
+    <sys_file>
+        <uid>1</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <type>2</type>
+        <storage>1</storage>
+        <identifier>/user_uploads/example-for-event.gif</identifier>
+        <extension>gif</extension>
+        <mime_type>image/gif</mime_type>
+        <name>ext_icon.gif</name>
+        <sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
+        <size>637</size>
+        <creation_date>1370877201</creation_date>
+        <modification_date>1369407629</modification_date>
+        <last_indexed>0</last_indexed>
+        <missing>0</missing>
+        <metadata>0</metadata>
+        <identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
+        <folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
+    </sys_file>
+    <sys_file>
+        <uid>2</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <type>2</type>
+        <storage>1</storage>
+        <identifier>/user_uploads/example-for-partner.gif</identifier>
+        <extension>gif</extension>
+        <mime_type>image/gif</mime_type>
+        <name>ext_icon.gif</name>
+        <sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
+        <size>637</size>
+        <creation_date>1370877201</creation_date>
+        <modification_date>1369407629</modification_date>
+        <last_indexed>0</last_indexed>
+        <missing>0</missing>
+        <metadata>0</metadata>
+        <identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
+        <folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
+    </sys_file>
+
+    <sys_file_metadata>
+        <uid>1</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <crdate>1371467047</crdate>
+        <cruser_id>1</cruser_id>
+        <file>1</file>
+    </sys_file_metadata>
+    <sys_file_metadata>
+        <uid>2</uid>
+        <pid>0</pid>
+        <tstamp>1371467047</tstamp>
+        <crdate>1371467047</crdate>
+        <cruser_id>1</cruser_id>
+        <file>2</file>
+    </sys_file_metadata>
+
+    <sys_file_reference>
+        <uid>1</uid>
+        <pid>2</pid>
+        <tstamp>1373537480</tstamp>
+        <crdate>1371484347</crdate>
+        <cruser_id>1</cruser_id>
+        <deleted>0</deleted>
+        <hidden>0</hidden>
+        <sys_language_uid>0</sys_language_uid>
+        <uid_local>1</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_event</tablenames>
+        <fieldname>images</fieldname>
+        <sorting_foreign>1</sorting_foreign>
+        <table_local>sys_file</table_local>
+    </sys_file_reference>
+    <sys_file_reference>
+        <uid>2</uid>
+        <pid>2</pid>
+        <tstamp>1373537480</tstamp>
+        <crdate>1371484347</crdate>
+        <cruser_id>1</cruser_id>
+        <deleted>0</deleted>
+        <hidden>0</hidden>
+        <sys_language_uid>0</sys_language_uid>
+        <uid_local>2</uid_local>
+        <uid_foreign>1</uid_foreign>
+        <tablenames>tx_events_domain_model_partner</tablenames>
+        <fieldname>images</fieldname>
+        <sorting_foreign>1</sorting_foreign>
+        <table_local>sys_file</table_local>
+    </sys_file_reference>
+
+    <tx_events_domain_model_region>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Region</title>
+    </tx_events_domain_model_region>
+
+    <tx_events_domain_model_partner>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Partner</title>
+        <link>https://example.com</link>
+        <images>1</images>
+    </tx_events_domain_model_partner>
+
+    <tx_events_domain_model_organizer>
+        <uid>1</uid>
+        <pid>2</pid>
+        <name>Example Organizer</name>
+        <street>Example Street 17</street>
+        <district></district>
+        <city>Example Town</city>
+        <zip>00101</zip>
+        <phone>+49 2161 56 36 27 37 48 94 28</phone>
+        <web>https://example.com</web>
+        <email>someone@example.com</email>
+    </tx_events_domain_model_organizer>
+
+    <tx_events_domain_model_event>
+        <uid>1</uid>
+        <pid>2</pid>
+        <title>Example Event</title>
+        <subtitle>Some further info about event</subtitle>
+        <global_id>5540-34</global_id>
+        <slug>5540-34</slug>
+        <organizer>1</organizer>
+        <partner>1</partner>
+        <region>1</region>
+        <images>0</images>
+        <categories>2</categories>
+    </tx_events_domain_model_event>
+
+    <tx_events_domain_model_date>
+        <uid>1</uid>
+        <pid>2</pid>
+        <event>1</event>
+        <start>4101372000</start>
+        <end>4101377400</end>
+        <canceled>no</canceled>
+    </tx_events_domain_model_date>
+
+    <tx_events_domain_model_date>
+        <uid>2</uid>
+        <pid>2</pid>
+        <event>1</event>
+        <start>4101372000</start>
+        <end>4101377400</end>
+        <canceled>no</canceled>
+    </tx_events_domain_model_date>
+</dataset>

Tests/Functional/Cleanup/RemoveAllTest.php

@@ -0,0 +1,104 @@
+<?php
+
+namespace Wrm\Events\Tests\Functional\Cleanup;
+
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Tester\CommandTester;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase;
+use Wrm\Events\Command\RemoveAllCommand;
+
+/**
+ * @testdox Cleanup RemoveAll
+ */
+class RemoveAllTest extends FunctionalTestCase
+{
+    protected $testExtensionsToLoad = [
+        'typo3conf/ext/events',
+    ];
+
+    protected $pathsToProvideInTestInstance = [
+        'typo3conf/ext/events/Tests/Functional/Cleanup/Fixtures/fileadmin/' => 'fileadmin/',
+    ];
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+
+        $this->setUpBackendUserFromFixture(1);
+    }
+
+    /**
+     * @test
+     */
+    public function removesAllData(): void
+    {
+        $this->importDataSet('EXT:events/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml');
+
+        $subject = $this->getContainer()->get(RemoveAllCommand::class);
+        self::assertInstanceOf(Command::class, $subject);
+
+        $tester = new CommandTester($subject);
+        $tester->execute([], ['capture_stderr_separately' => true]);
+
+        self::assertSame(0, $tester->getStatusCode());
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_partner'),
+            'Partners are not kept.'
+        );
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_region'),
+            'Regions are not kept.'
+        );
+        self::assertCount(
+            1,
+            $this->getAllRecords('tx_events_domain_model_partner'),
+            'Partners are not kept.'
+        );
+
+        self::assertCount(
+            0,
+            $this->getAllRecords('tx_events_domain_model_organizer'),
+            'Organizers are still there.'
+        );
+        self::assertCount(
+            0,
+            $this->getAllRecords('tx_events_domain_model_event'),
+            'Events are still there.'
+        );
+        self::assertCount(
+            0,
+            $this->getAllRecords('tx_events_domain_model_date'),
+            'Dates are still there.'
+        );
+
+        self::assertCount(
+            0,
+            $this->getAllRecords('sys_category_record_mm'),
+            'Relations to categories still exist.'
+        );
+
+        self::assertCount(
+            1,
+            $this->getAllRecords('sys_file'),
+            'Unexpected number of sys_file records.'
+        );
+        self::assertCount(
+            1,
+            $this->getAllRecords('sys_file_reference'),
+            'Unexpected number of sys_file_reference records.'
+        );
+        self::assertCount(
+            1,
+            $this->getAllRecords('sys_file_metadata'),
+            'Unexpected number of sys_file_metadata records.'
+        );
+
+        $files = GeneralUtility::getFilesInDir('fileadmin/user_uploads');
+        self::assertIsArray($files, 'Failed to retrieve files from filesystem.');
+        self::assertCount(1, $files, 'Unexpectd number of files in filesystem.');
+        self::assertSame('example-for-partner.gif', array_values($files)[0], 'Unexpectd file in filesystem.');
+    }
+}