Delete files which were referenced by events objects

Adjust existing code base to not use hardcoded file path.
Instead use info from database and check all files which have references
to event objects.

Also add test to cover feature.
This commit is contained in:
Daniel Siepmann 2021-12-16 14:07:32 +01:00
parent f06f9c3303
commit 6aa4681cb9
6 changed files with 403 additions and 44 deletions

View file

@ -46,57 +46,88 @@ class Files
$this->storageRepository = $storageRepository; $this->storageRepository = $storageRepository;
} }
public function deleteAll(): void
{
$this->delete($this->getFilesFromDb());
}
public function deleteDangling(): void public function deleteDangling(): void
{ {
$this->delete($this->getFilesFromDb(function (QueryBuilder $queryBuilder) { $this->markFileReferencesDeletedIfForeignRecordIsMissing();
$queryBuilder->leftJoin( $this->deleteFilesWithoutProperReference();
}
private function markFileReferencesDeletedIfForeignRecordIsMissing(): void
{
$referencesQuery = $this->connectionPool
->getQueryBuilderForTable('sys_file_reference');
$referencesQuery->getRestrictions()->removeAll();
$referencesQuery->select(
'uid',
'uid_foreign',
'tablenames'
);
$referencesQuery->from('sys_file_reference');
$referencesQuery->where(
$referencesQuery->expr()->like(
'tablenames',
$referencesQuery->createNamedParameter('tx_events_domain_model_%')
)
);
$referencesQuery->orderBy('tablenames');
$referencesQuery->addOrderBy('uid_foreign');
$references = $referencesQuery->execute();
$uidsPerTable = [];
$referenceUidsToMarkAsDeleted = [];
while ($reference = $references->fetch()) {
if (is_array($reference) === false) {
continue;
}
$uidsPerTable[(string)$reference['tablenames']][$reference['uid']] = $reference['uid_foreign'];
}
foreach ($uidsPerTable as $tableName => $records) {
$queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName);
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder->select('uid');
$queryBuilder->from($tableName);
$queryBuilder->where($queryBuilder->expr()->in('uid', $records));
$referenceUidsToMarkAsDeleted = array_merge(
$referenceUidsToMarkAsDeleted,
array_keys(array_diff($records, $queryBuilder->execute()->fetchAll(\PDO::FETCH_COLUMN)))
);
}
if ($referenceUidsToMarkAsDeleted === []) {
return;
}
$updateQuery = $this->connectionPool->getQueryBuilderForTable('sys_file_reference');
$updateQuery->update('sys_file_reference');
$updateQuery->where($updateQuery->expr()->in('uid', $referenceUidsToMarkAsDeleted));
$updateQuery->set('deleted', '1');
$updateQuery->execute();
}
private function deleteFilesWithoutProperReference(): void
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder->select('file.identifier', 'file.storage', 'file.uid')
->from('sys_file', 'file')
->leftJoin(
'file', 'file',
'sys_file_reference', 'sys_file_reference',
'reference', 'reference',
$queryBuilder->expr()->eq('file.uid', $queryBuilder->quoteIdentifier('reference.uid_local')) 'reference.uid_local = file.uid'
); )
$queryBuilder->andWhere( ->where($queryBuilder->expr()->eq('reference.deleted', 1));
$queryBuilder->expr()->orX( /** @var array{int: array{storage: int, identifier: string, uid: int}} $filesToDelete */
$queryBuilder->expr()->isNull('reference.uid'), $filesToDelete = $queryBuilder->execute()->fetchAll();
$queryBuilder->expr()->eq('reference.deleted', 1),
$queryBuilder->expr()->eq('reference.hidden', 1)
)
);
}));
}
private function getFilesFromDb(callable $whereGenerator = null): array
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder->select('file.identifier', 'file.storage', 'file.uid')
->from('sys_file', 'file')
->where($queryBuilder->expr()->like(
'file.identifier',
$queryBuilder->createNamedParameter('/staedte/%/events/%')
));
if ($whereGenerator !== null) {
$whereGenerator($queryBuilder);
}
return $queryBuilder->execute()->fetchAll();
}
private function delete(array $filesToDelete): void
{
$uidsToRemove = []; $uidsToRemove = [];
foreach ($filesToDelete as $fileToDelete) { foreach ($filesToDelete as $fileToDelete) {
$this->deleteFromFal($fileToDelete['storage'], $fileToDelete['identifier']); $this->deleteFromFal((int) $fileToDelete['storage'], (string) $fileToDelete['identifier']);
$uidsToRemove[] = $fileToDelete['uid']; $uidsToRemove[] = (int) $fileToDelete['uid'];
} }
$this->deleteFromDb(...$uidsToRemove); $this->deleteFromDb(...$uidsToRemove);

View file

@ -26,7 +26,7 @@ class CleanupService
public function deleteAllData(): void public function deleteAllData(): void
{ {
$this->database->truncateTables(); $this->database->truncateTables();
$this->files->deleteAll(); $this->files->deleteDangling();
} }
public function deletePastData(): void public function deletePastData(): void

View file

@ -0,0 +1,224 @@
<?xml version="1.0" encoding="utf-8"?>
<dataset>
<pages>
<pid>0</pid>
<uid>1</uid>
<title>Root page</title>
<slug>1</slug>
</pages>
<pages>
<pid>1</pid>
<uid>2</uid>
<title>Storage</title>
<doktype>254</doktype>
</pages>
<sys_category>
<uid>1</uid>
<pid>2</pid>
<title>Example Category 1</title>
</sys_category>
<sys_category>
<uid>2</uid>
<pid>2</pid>
<title>Example Category 2</title>
</sys_category>
<sys_category_record_mm>
<uid_local>1</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_event</tablenames>
</sys_category_record_mm>
<sys_category_record_mm>
<uid_local>2</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_event</tablenames>
</sys_category_record_mm>
<sys_file_storage>
<uid>1</uid>
<pid>0</pid>
<tstamp>1423209858</tstamp>
<crdate>1370878372</crdate>
<cruser_id>0</cruser_id>
<deleted>0</deleted>
<name>fileadmin/ (auto-created)</name>
<description>This is the local fileadmin/ directory. This storage mount has been created automatically by TYPO3.</description>
<driver>Local</driver>
<configuration><![CDATA[<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
<T3FlexForms>
<data>
<sheet index="sDEF">
<language index="lDEF">
<field index="basePath">
<value index="vDEF">fileadmin/</value>
</field>
<field index="pathType">
<value index="vDEF">relative</value>
</field>
<field index="caseSensitive">
<value index="vDEF">1</value>
</field>
</language>
</sheet>
</data>
</T3FlexForms>]]></configuration>
<is_browsable>1</is_browsable>
<is_public>1</is_public>
<is_writable>1</is_writable>
<is_online>1</is_online>
<processingfolder>_processed_</processingfolder>
<is_default>1</is_default>
<auto_extract_metadata>1</auto_extract_metadata>
</sys_file_storage>
<sys_file>
<uid>1</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<type>2</type>
<storage>1</storage>
<identifier>/user_uploads/example-for-event.gif</identifier>
<extension>gif</extension>
<mime_type>image/gif</mime_type>
<name>ext_icon.gif</name>
<sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
<size>637</size>
<creation_date>1370877201</creation_date>
<modification_date>1369407629</modification_date>
<last_indexed>0</last_indexed>
<missing>0</missing>
<metadata>0</metadata>
<identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
<folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
</sys_file>
<sys_file>
<uid>2</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<type>2</type>
<storage>1</storage>
<identifier>/user_uploads/example-for-partner.gif</identifier>
<extension>gif</extension>
<mime_type>image/gif</mime_type>
<name>ext_icon.gif</name>
<sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
<size>637</size>
<creation_date>1370877201</creation_date>
<modification_date>1369407629</modification_date>
<last_indexed>0</last_indexed>
<missing>0</missing>
<metadata>0</metadata>
<identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
<folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
</sys_file>
<sys_file_metadata>
<uid>1</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<crdate>1371467047</crdate>
<cruser_id>1</cruser_id>
<file>1</file>
</sys_file_metadata>
<sys_file_metadata>
<uid>2</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<crdate>1371467047</crdate>
<cruser_id>1</cruser_id>
<file>2</file>
</sys_file_metadata>
<sys_file_reference>
<uid>1</uid>
<pid>2</pid>
<tstamp>1373537480</tstamp>
<crdate>1371484347</crdate>
<cruser_id>1</cruser_id>
<deleted>0</deleted>
<hidden>0</hidden>
<sys_language_uid>0</sys_language_uid>
<uid_local>1</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_event</tablenames>
<fieldname>images</fieldname>
<sorting_foreign>1</sorting_foreign>
<table_local>sys_file</table_local>
</sys_file_reference>
<sys_file_reference>
<uid>2</uid>
<pid>2</pid>
<tstamp>1373537480</tstamp>
<crdate>1371484347</crdate>
<cruser_id>1</cruser_id>
<deleted>0</deleted>
<hidden>0</hidden>
<sys_language_uid>0</sys_language_uid>
<uid_local>2</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_partner</tablenames>
<fieldname>images</fieldname>
<sorting_foreign>1</sorting_foreign>
<table_local>sys_file</table_local>
</sys_file_reference>
<tx_events_domain_model_region>
<uid>1</uid>
<pid>2</pid>
<title>Example Region</title>
</tx_events_domain_model_region>
<tx_events_domain_model_partner>
<uid>1</uid>
<pid>2</pid>
<title>Example Partner</title>
<link>https://example.com</link>
<images>1</images>
</tx_events_domain_model_partner>
<tx_events_domain_model_organizer>
<uid>1</uid>
<pid>2</pid>
<name>Example Organizer</name>
<street>Example Street 17</street>
<district></district>
<city>Example Town</city>
<zip>00101</zip>
<phone>+49 2161 56 36 27 37 48 94 28</phone>
<web>https://example.com</web>
<email>someone@example.com</email>
</tx_events_domain_model_organizer>
<tx_events_domain_model_event>
<uid>1</uid>
<pid>2</pid>
<title>Example Event</title>
<subtitle>Some further info about event</subtitle>
<global_id>5540-34</global_id>
<slug>5540-34</slug>
<organizer>1</organizer>
<partner>1</partner>
<region>1</region>
<images>0</images>
<categories>2</categories>
</tx_events_domain_model_event>
<tx_events_domain_model_date>
<uid>1</uid>
<pid>2</pid>
<event>1</event>
<start>4101372000</start>
<end>4101377400</end>
<canceled>no</canceled>
</tx_events_domain_model_date>
<tx_events_domain_model_date>
<uid>2</uid>
<pid>2</pid>
<event>1</event>
<start>4101372000</start>
<end>4101377400</end>
<canceled>no</canceled>
</tx_events_domain_model_date>
</dataset>

View file

@ -0,0 +1,104 @@
<?php
namespace Wrm\Events\Tests\Functional\Cleanup;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Tester\CommandTester;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase;
use Wrm\Events\Command\RemoveAllCommand;
/**
* @testdox Cleanup RemoveAll
*/
class RemoveAllTest extends FunctionalTestCase
{
protected $testExtensionsToLoad = [
'typo3conf/ext/events',
];
protected $pathsToProvideInTestInstance = [
'typo3conf/ext/events/Tests/Functional/Cleanup/Fixtures/fileadmin/' => 'fileadmin/',
];
protected function setUp(): void
{
parent::setUp();
$this->setUpBackendUserFromFixture(1);
}
/**
* @test
*/
public function removesAllData(): void
{
$this->importDataSet('EXT:events/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml');
$subject = $this->getContainer()->get(RemoveAllCommand::class);
self::assertInstanceOf(Command::class, $subject);
$tester = new CommandTester($subject);
$tester->execute([], ['capture_stderr_separately' => true]);
self::assertSame(0, $tester->getStatusCode());
self::assertCount(
1,
$this->getAllRecords('tx_events_domain_model_partner'),
'Partners are not kept.'
);
self::assertCount(
1,
$this->getAllRecords('tx_events_domain_model_region'),
'Regions are not kept.'
);
self::assertCount(
1,
$this->getAllRecords('tx_events_domain_model_partner'),
'Partners are not kept.'
);
self::assertCount(
0,
$this->getAllRecords('tx_events_domain_model_organizer'),
'Organizers are still there.'
);
self::assertCount(
0,
$this->getAllRecords('tx_events_domain_model_event'),
'Events are still there.'
);
self::assertCount(
0,
$this->getAllRecords('tx_events_domain_model_date'),
'Dates are still there.'
);
self::assertCount(
0,
$this->getAllRecords('sys_category_record_mm'),
'Relations to categories still exist.'
);
self::assertCount(
1,
$this->getAllRecords('sys_file'),
'Unexpected number of sys_file records.'
);
self::assertCount(
1,
$this->getAllRecords('sys_file_reference'),
'Unexpected number of sys_file_reference records.'
);
self::assertCount(
1,
$this->getAllRecords('sys_file_metadata'),
'Unexpected number of sys_file_metadata records.'
);
$files = GeneralUtility::getFilesInDir('fileadmin/user_uploads');
self::assertIsArray($files, 'Failed to retrieve files from filesystem.');
self::assertCount(1, $files, 'Unexpectd number of files in filesystem.');
self::assertSame('example-for-partner.gif', array_values($files)[0], 'Unexpectd file in filesystem.');
}
}