From 6aa4681cb978895074278601be2dd4c60c90d2cf Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 16 Dec 2021 14:07:32 +0100 Subject: [PATCH] Delete files which were referenced by events objects Adjust existing code base to not use hardcoded file path. Instead use info from database and check all files which have references to event objects. Also add test to cover feature. --- Classes/Service/Cleanup/Files.php | 117 +++++---- Classes/Service/CleanupService.php | 2 +- .../Cleanup/Fixtures/RemoveAllTest.xml | 224 ++++++++++++++++++ .../user_uploads/example-for-event.gif | 0 .../user_uploads/example-for-partner.gif | 0 Tests/Functional/Cleanup/RemoveAllTest.php | 104 ++++++++ 6 files changed, 403 insertions(+), 44 deletions(-) create mode 100644 Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml create mode 100644 Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-event.gif create mode 100644 Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-partner.gif create mode 100644 Tests/Functional/Cleanup/RemoveAllTest.php diff --git a/Classes/Service/Cleanup/Files.php b/Classes/Service/Cleanup/Files.php index 9f62f4f..9fd30c5 100644 --- a/Classes/Service/Cleanup/Files.php +++ b/Classes/Service/Cleanup/Files.php @@ -46,57 +46,88 @@ class Files $this->storageRepository = $storageRepository; } - public function deleteAll(): void - { - $this->delete($this->getFilesFromDb()); - } - public function deleteDangling(): void { - $this->delete($this->getFilesFromDb(function (QueryBuilder $queryBuilder) { - $queryBuilder->leftJoin( + $this->markFileReferencesDeletedIfForeignRecordIsMissing(); + $this->deleteFilesWithoutProperReference(); + } + + private function markFileReferencesDeletedIfForeignRecordIsMissing(): void + { + $referencesQuery = $this->connectionPool + ->getQueryBuilderForTable('sys_file_reference'); + $referencesQuery->getRestrictions()->removeAll(); + $referencesQuery->select( + 'uid', + 'uid_foreign', + 'tablenames' + ); + $referencesQuery->from('sys_file_reference'); + $referencesQuery->where( + $referencesQuery->expr()->like( + 'tablenames', + $referencesQuery->createNamedParameter('tx_events_domain_model_%') + ) + ); + $referencesQuery->orderBy('tablenames'); + $referencesQuery->addOrderBy('uid_foreign'); + + $references = $referencesQuery->execute(); + + $uidsPerTable = []; + $referenceUidsToMarkAsDeleted = []; + + while ($reference = $references->fetch()) { + if (is_array($reference) === false) { + continue; + } + + $uidsPerTable[(string)$reference['tablenames']][$reference['uid']] = $reference['uid_foreign']; + } + + foreach ($uidsPerTable as $tableName => $records) { + $queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName); + $queryBuilder->getRestrictions()->removeAll(); + $queryBuilder->select('uid'); + $queryBuilder->from($tableName); + $queryBuilder->where($queryBuilder->expr()->in('uid', $records)); + $referenceUidsToMarkAsDeleted = array_merge( + $referenceUidsToMarkAsDeleted, + array_keys(array_diff($records, $queryBuilder->execute()->fetchAll(\PDO::FETCH_COLUMN))) + ); + } + + if ($referenceUidsToMarkAsDeleted === []) { + return; + } + + $updateQuery = $this->connectionPool->getQueryBuilderForTable('sys_file_reference'); + $updateQuery->update('sys_file_reference'); + $updateQuery->where($updateQuery->expr()->in('uid', $referenceUidsToMarkAsDeleted)); + $updateQuery->set('deleted', '1'); + $updateQuery->execute(); + } + + private function deleteFilesWithoutProperReference(): void + { + $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file'); + $queryBuilder->getRestrictions()->removeAll(); + $queryBuilder->select('file.identifier', 'file.storage', 'file.uid') + ->from('sys_file', 'file') + ->leftJoin( 'file', 'sys_file_reference', 'reference', - $queryBuilder->expr()->eq('file.uid', $queryBuilder->quoteIdentifier('reference.uid_local')) - ); - $queryBuilder->andWhere( - $queryBuilder->expr()->orX( - $queryBuilder->expr()->isNull('reference.uid'), - $queryBuilder->expr()->eq('reference.deleted', 1), - $queryBuilder->expr()->eq('reference.hidden', 1) - ) - ); - })); - } + 'reference.uid_local = file.uid' + ) + ->where($queryBuilder->expr()->eq('reference.deleted', 1)); + /** @var array{int: array{storage: int, identifier: string, uid: int}} $filesToDelete */ + $filesToDelete = $queryBuilder->execute()->fetchAll(); - private function getFilesFromDb(callable $whereGenerator = null): array - { - $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file'); - - $queryBuilder->getRestrictions()->removeAll(); - - $queryBuilder->select('file.identifier', 'file.storage', 'file.uid') - ->from('sys_file', 'file') - ->where($queryBuilder->expr()->like( - 'file.identifier', - $queryBuilder->createNamedParameter('/staedte/%/events/%') - )); - - if ($whereGenerator !== null) { - $whereGenerator($queryBuilder); - } - - return $queryBuilder->execute()->fetchAll(); - } - - private function delete(array $filesToDelete): void - { $uidsToRemove = []; - foreach ($filesToDelete as $fileToDelete) { - $this->deleteFromFal($fileToDelete['storage'], $fileToDelete['identifier']); - $uidsToRemove[] = $fileToDelete['uid']; + $this->deleteFromFal((int) $fileToDelete['storage'], (string) $fileToDelete['identifier']); + $uidsToRemove[] = (int) $fileToDelete['uid']; } $this->deleteFromDb(...$uidsToRemove); diff --git a/Classes/Service/CleanupService.php b/Classes/Service/CleanupService.php index 7b8f9c1..ce02d5d 100644 --- a/Classes/Service/CleanupService.php +++ b/Classes/Service/CleanupService.php @@ -26,7 +26,7 @@ class CleanupService public function deleteAllData(): void { $this->database->truncateTables(); - $this->files->deleteAll(); + $this->files->deleteDangling(); } public function deletePastData(): void diff --git a/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml b/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml new file mode 100644 index 0000000..88563ac --- /dev/null +++ b/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml @@ -0,0 +1,224 @@ + + + + 0 + 1 + Root page + 1 + + + 1 + 2 + Storage + 254 + + + + 1 + 2 + Example Category 1 + + + 2 + 2 + Example Category 2 + + + + 1 + 1 + tx_events_domain_model_event + + + 2 + 1 + tx_events_domain_model_event + + + + 1 + 0 + 1423209858 + 1370878372 + 0 + 0 + fileadmin/ (auto-created) + This is the local fileadmin/ directory. This storage mount has been created automatically by TYPO3. + Local + + + + + + + fileadmin/ + + + relative + + + 1 + + + + + ]]> + 1 + 1 + 1 + 1 + _processed_ + 1 + 1 + + + + 1 + 0 + 1371467047 + 2 + 1 + /user_uploads/example-for-event.gif + gif + image/gif + ext_icon.gif + 359ae0fb420fe8afe1a8b8bc5e46d75090a826b9 + 637 + 1370877201 + 1369407629 + 0 + 0 + 0 + 475768e491580fb8b74ed36c2b1aaf619ca5e11d + b4ab666a114d9905a50606d1837b74d952dfd90f + + + 2 + 0 + 1371467047 + 2 + 1 + /user_uploads/example-for-partner.gif + gif + image/gif + ext_icon.gif + 359ae0fb420fe8afe1a8b8bc5e46d75090a826b9 + 637 + 1370877201 + 1369407629 + 0 + 0 + 0 + 475768e491580fb8b74ed36c2b1aaf619ca5e11d + b4ab666a114d9905a50606d1837b74d952dfd90f + + + + 1 + 0 + 1371467047 + 1371467047 + 1 + 1 + + + 2 + 0 + 1371467047 + 1371467047 + 1 + 2 + + + + 1 + 2 + 1373537480 + 1371484347 + 1 + 0 + 0 + 0 + 1 + 1 + tx_events_domain_model_event + images + 1 + sys_file + + + 2 + 2 + 1373537480 + 1371484347 + 1 + 0 + 0 + 0 + 2 + 1 + tx_events_domain_model_partner + images + 1 + sys_file + + + + 1 + 2 + Example Region + + + + 1 + 2 + Example Partner + https://example.com + 1 + + + + 1 + 2 + Example Organizer + Example Street 17 + + Example Town + 00101 + +49 2161 56 36 27 37 48 94 28 + https://example.com + someone@example.com + + + + 1 + 2 + Example Event + Some further info about event + 5540-34 + 5540-34 + 1 + 1 + 1 + 0 + 2 + + + + 1 + 2 + 1 + 4101372000 + 4101377400 + no + + + + 2 + 2 + 1 + 4101372000 + 4101377400 + no + + diff --git a/Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-event.gif b/Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-event.gif new file mode 100644 index 0000000..e69de29 diff --git a/Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-partner.gif b/Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-partner.gif new file mode 100644 index 0000000..e69de29 diff --git a/Tests/Functional/Cleanup/RemoveAllTest.php b/Tests/Functional/Cleanup/RemoveAllTest.php new file mode 100644 index 0000000..73e7c08 --- /dev/null +++ b/Tests/Functional/Cleanup/RemoveAllTest.php @@ -0,0 +1,104 @@ + 'fileadmin/', + ]; + + protected function setUp(): void + { + parent::setUp(); + + $this->setUpBackendUserFromFixture(1); + } + + /** + * @test + */ + public function removesAllData(): void + { + $this->importDataSet('EXT:events/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml'); + + $subject = $this->getContainer()->get(RemoveAllCommand::class); + self::assertInstanceOf(Command::class, $subject); + + $tester = new CommandTester($subject); + $tester->execute([], ['capture_stderr_separately' => true]); + + self::assertSame(0, $tester->getStatusCode()); + self::assertCount( + 1, + $this->getAllRecords('tx_events_domain_model_partner'), + 'Partners are not kept.' + ); + self::assertCount( + 1, + $this->getAllRecords('tx_events_domain_model_region'), + 'Regions are not kept.' + ); + self::assertCount( + 1, + $this->getAllRecords('tx_events_domain_model_partner'), + 'Partners are not kept.' + ); + + self::assertCount( + 0, + $this->getAllRecords('tx_events_domain_model_organizer'), + 'Organizers are still there.' + ); + self::assertCount( + 0, + $this->getAllRecords('tx_events_domain_model_event'), + 'Events are still there.' + ); + self::assertCount( + 0, + $this->getAllRecords('tx_events_domain_model_date'), + 'Dates are still there.' + ); + + self::assertCount( + 0, + $this->getAllRecords('sys_category_record_mm'), + 'Relations to categories still exist.' + ); + + self::assertCount( + 1, + $this->getAllRecords('sys_file'), + 'Unexpected number of sys_file records.' + ); + self::assertCount( + 1, + $this->getAllRecords('sys_file_reference'), + 'Unexpected number of sys_file_reference records.' + ); + self::assertCount( + 1, + $this->getAllRecords('sys_file_metadata'), + 'Unexpected number of sys_file_metadata records.' + ); + + $files = GeneralUtility::getFilesInDir('fileadmin/user_uploads'); + self::assertIsArray($files, 'Failed to retrieve files from filesystem.'); + self::assertCount(1, $files, 'Unexpectd number of files in filesystem.'); + self::assertSame('example-for-partner.gif', array_values($files)[0], 'Unexpectd file in filesystem.'); + } +}