Work-Related/events
Work-Related/events
1
0
mirror of https://github.com/werkraum-media/events.git synced 2024-11-22 10:36:11 +01:00
Code Issues Activity

Delete files which were referenced by events objects

Browse source 
Adjust existing code base to not use hardcoded file path.
Instead use info from database and check all files which have references
to event objects.

Also add test to cover feature.
This commit is contained in:
Daniel Siepmann 2021-12-16 14:07:32 +01:00
parent f06f9c3303
commit 6aa4681cb9
6 changed files with 403 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

115
Classes/Service/Cleanup/Files.php
View file

@ -46,57 +46,88 @@ class Files
$this->storageRepository = $storageRepository; $this->storageRepository = $storageRepository;
} }
public function deleteAll(): void
{
$this->delete($this->getFilesFromDb());
}
public function deleteDangling(): void public function deleteDangling(): void
{ {
$this->delete($this->getFilesFromDb(function (QueryBuilder $queryBuilder) { $this->markFileReferencesDeletedIfForeignRecordIsMissing();
$queryBuilder->leftJoin( $this->deleteFilesWithoutProperReference();
}
private function markFileReferencesDeletedIfForeignRecordIsMissing(): void
{
$referencesQuery = $this->connectionPool
->getQueryBuilderForTable('sys_file_reference');
$referencesQuery->getRestrictions()->removeAll();
$referencesQuery->select(
'uid',
'uid_foreign',
'tablenames'
);
$referencesQuery->from('sys_file_reference');
$referencesQuery->where(
$referencesQuery->expr()->like(
'tablenames',
$referencesQuery->createNamedParameter('tx_events_domain_model_%')
)
);
$referencesQuery->orderBy('tablenames');
$referencesQuery->addOrderBy('uid_foreign');
$references = $referencesQuery->execute();
$uidsPerTable = [];
$referenceUidsToMarkAsDeleted = [];
while ($reference = $references->fetch()) {
if (is_array($reference) === false) {
continue;
}
$uidsPerTable[(string)$reference['tablenames']][$reference['uid']] = $reference['uid_foreign'];
}
foreach ($uidsPerTable as $tableName => $records) {
$queryBuilder = $this->connectionPool->getQueryBuilderForTable($tableName);
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder->select('uid');
$queryBuilder->from($tableName);
$queryBuilder->where($queryBuilder->expr()->in('uid', $records));
$referenceUidsToMarkAsDeleted = array_merge(
$referenceUidsToMarkAsDeleted,
array_keys(array_diff($records, $queryBuilder->execute()->fetchAll(\PDO::FETCH_COLUMN)))
);
}
if ($referenceUidsToMarkAsDeleted === []) {
return;
}
$updateQuery = $this->connectionPool->getQueryBuilderForTable('sys_file_reference');
$updateQuery->update('sys_file_reference');
$updateQuery->where($updateQuery->expr()->in('uid', $referenceUidsToMarkAsDeleted));
$updateQuery->set('deleted', '1');
$updateQuery->execute();
}
private function deleteFilesWithoutProperReference(): void
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder->select('file.identifier', 'file.storage', 'file.uid')
->from('sys_file', 'file')
->leftJoin(
'file', 'file',
'sys_file_reference', 'sys_file_reference',
'reference', 'reference',
$queryBuilder->expr()->eq('file.uid', $queryBuilder->quoteIdentifier('reference.uid_local')) 'reference.uid_local = file.uid'
);
$queryBuilder->andWhere(
$queryBuilder->expr()->orX(
$queryBuilder->expr()->isNull('reference.uid'),
$queryBuilder->expr()->eq('reference.deleted', 1),
$queryBuilder->expr()->eq('reference.hidden', 1)
) )
); ->where($queryBuilder->expr()->eq('reference.deleted', 1));
})); /** @var array{int: array{storage: int, identifier: string, uid: int}} $filesToDelete */
} $filesToDelete = $queryBuilder->execute()->fetchAll();
private function getFilesFromDb(callable $whereGenerator = null): array
{
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder->getRestrictions()->removeAll();
$queryBuilder->select('file.identifier', 'file.storage', 'file.uid')
->from('sys_file', 'file')
->where($queryBuilder->expr()->like(
'file.identifier',
$queryBuilder->createNamedParameter('/staedte/%/events/%')
));
if ($whereGenerator !== null) {
$whereGenerator($queryBuilder);
}
return $queryBuilder->execute()->fetchAll();
}
private function delete(array $filesToDelete): void
{
$uidsToRemove = []; $uidsToRemove = [];
foreach ($filesToDelete as $fileToDelete) { foreach ($filesToDelete as $fileToDelete) {
$this->deleteFromFal($fileToDelete['storage'], $fileToDelete['identifier']); $this->deleteFromFal((int) $fileToDelete['storage'], (string) $fileToDelete['identifier']);
$uidsToRemove[] = $fileToDelete['uid']; $uidsToRemove[] = (int) $fileToDelete['uid'];
} }
$this->deleteFromDb(...$uidsToRemove); $this->deleteFromDb(...$uidsToRemove);

2
Classes/Service/CleanupService.php
View file

@ -26,7 +26,7 @@ class CleanupService
public function deleteAllData(): void public function deleteAllData(): void
{ {
$this->database->truncateTables(); $this->database->truncateTables();
$this->files->deleteAll(); $this->files->deleteDangling();
} }
public function deletePastData(): void public function deletePastData(): void

224
Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml Normal file
View file

@ -0,0 +1,224 @@
<?xml version="1.0" encoding="utf-8"?>
<dataset>
<pages>
<pid>0</pid>
<uid>1</uid>
<title>Root page</title>
<slug>1</slug>
</pages>
<pages>
<pid>1</pid>
<uid>2</uid>
<title>Storage</title>
<doktype>254</doktype>
</pages>
<sys_category>
<uid>1</uid>
<pid>2</pid>
<title>Example Category 1</title>
</sys_category>
<sys_category>
<uid>2</uid>
<pid>2</pid>
<title>Example Category 2</title>
</sys_category>
<sys_category_record_mm>
<uid_local>1</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_event</tablenames>
</sys_category_record_mm>
<sys_category_record_mm>
<uid_local>2</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_event</tablenames>
</sys_category_record_mm>
<sys_file_storage>
<uid>1</uid>
<pid>0</pid>
<tstamp>1423209858</tstamp>
<crdate>1370878372</crdate>
<cruser_id>0</cruser_id>
<deleted>0</deleted>
<name>fileadmin/ (auto-created)</name>
<description>This is the local fileadmin/ directory. This storage mount has been created automatically by TYPO3.</description>
<driver>Local</driver>
<configuration><![CDATA[<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
<T3FlexForms>
<data>
<sheet index="sDEF">
<language index="lDEF">
<field index="basePath">
<value index="vDEF">fileadmin/</value>
</field>
<field index="pathType">
<value index="vDEF">relative</value>
</field>
<field index="caseSensitive">
<value index="vDEF">1</value>
</field>
</language>
</sheet>
</data>
</T3FlexForms>]]></configuration>
<is_browsable>1</is_browsable>
<is_public>1</is_public>
<is_writable>1</is_writable>
<is_online>1</is_online>
<processingfolder>_processed_</processingfolder>
<is_default>1</is_default>
<auto_extract_metadata>1</auto_extract_metadata>
</sys_file_storage>
<sys_file>
<uid>1</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<type>2</type>
<storage>1</storage>
<identifier>/user_uploads/example-for-event.gif</identifier>
<extension>gif</extension>
<mime_type>image/gif</mime_type>
<name>ext_icon.gif</name>
<sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
<size>637</size>
<creation_date>1370877201</creation_date>
<modification_date>1369407629</modification_date>
<last_indexed>0</last_indexed>
<missing>0</missing>
<metadata>0</metadata>
<identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
<folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
</sys_file>
<sys_file>
<uid>2</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<type>2</type>
<storage>1</storage>
<identifier>/user_uploads/example-for-partner.gif</identifier>
<extension>gif</extension>
<mime_type>image/gif</mime_type>
<name>ext_icon.gif</name>
<sha1>359ae0fb420fe8afe1a8b8bc5e46d75090a826b9</sha1>
<size>637</size>
<creation_date>1370877201</creation_date>
<modification_date>1369407629</modification_date>
<last_indexed>0</last_indexed>
<missing>0</missing>
<metadata>0</metadata>
<identifier_hash>475768e491580fb8b74ed36c2b1aaf619ca5e11d</identifier_hash>
<folder_hash>b4ab666a114d9905a50606d1837b74d952dfd90f</folder_hash>
</sys_file>
<sys_file_metadata>
<uid>1</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<crdate>1371467047</crdate>
<cruser_id>1</cruser_id>
<file>1</file>
</sys_file_metadata>
<sys_file_metadata>
<uid>2</uid>
<pid>0</pid>
<tstamp>1371467047</tstamp>
<crdate>1371467047</crdate>
<cruser_id>1</cruser_id>
<file>2</file>
</sys_file_metadata>
<sys_file_reference>
<uid>1</uid>
<pid>2</pid>
<tstamp>1373537480</tstamp>
<crdate>1371484347</crdate>
<cruser_id>1</cruser_id>
<deleted>0</deleted>
<hidden>0</hidden>
<sys_language_uid>0</sys_language_uid>
<uid_local>1</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_event</tablenames>
<fieldname>images</fieldname>
<sorting_foreign>1</sorting_foreign>
<table_local>sys_file</table_local>
</sys_file_reference>
<sys_file_reference>
<uid>2</uid>
<pid>2</pid>
<tstamp>1373537480</tstamp>
<crdate>1371484347</crdate>
<cruser_id>1</cruser_id>
<deleted>0</deleted>
<hidden>0</hidden>
<sys_language_uid>0</sys_language_uid>
<uid_local>2</uid_local>
<uid_foreign>1</uid_foreign>
<tablenames>tx_events_domain_model_partner</tablenames>
<fieldname>images</fieldname>
<sorting_foreign>1</sorting_foreign>
<table_local>sys_file</table_local>
</sys_file_reference>
<tx_events_domain_model_region>
<uid>1</uid>
<pid>2</pid>
<title>Example Region</title>
</tx_events_domain_model_region>
<tx_events_domain_model_partner>
<uid>1</uid>
<pid>2</pid>
<title>Example Partner</title>
<link>https://example.com</link>
<images>1</images>
</tx_events_domain_model_partner>
<tx_events_domain_model_organizer>
<uid>1</uid>
<pid>2</pid>
<name>Example Organizer</name>
<street>Example Street 17</street>
<district></district>
<city>Example Town</city>
<zip>00101</zip>
<phone>+49 2161 56 36 27 37 48 94 28</phone>
<web>https://example.com</web>
<email>someone@example.com</email>
</tx_events_domain_model_organizer>
<tx_events_domain_model_event>
<uid>1</uid>
<pid>2</pid>
<title>Example Event</title>
<subtitle>Some further info about event</subtitle>
<global_id>5540-34</global_id>
<slug>5540-34</slug>
<organizer>1</organizer>
<partner>1</partner>
<region>1</region>
<images>0</images>
<categories>2</categories>
</tx_events_domain_model_event>
<tx_events_domain_model_date>
<uid>1</uid>
<pid>2</pid>
<event>1</event>
<start>4101372000</start>
<end>4101377400</end>
<canceled>no</canceled>
</tx_events_domain_model_date>
<tx_events_domain_model_date>
<uid>2</uid>
<pid>2</pid>
<event>1</event>
<start>4101372000</start>
<end>4101377400</end>
<canceled>no</canceled>
</tx_events_domain_model_date>
</dataset>

0
Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-event.gif Normal file
View file

0
Tests/Functional/Cleanup/Fixtures/fileadmin/user_uploads/example-for-partner.gif Normal file
View file

104
Tests/Functional/Cleanup/RemoveAllTest.php Normal file
View file

@ -0,0 +1,104 @@
<?php
namespace Wrm\Events\Tests\Functional\Cleanup;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Tester\CommandTester;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase;
use Wrm\Events\Command\RemoveAllCommand;
/**
* @testdox Cleanup RemoveAll
*/
class RemoveAllTest extends FunctionalTestCase
{
protected $testExtensionsToLoad = [
'typo3conf/ext/events',
];
protected $pathsToProvideInTestInstance = [
'typo3conf/ext/events/Tests/Functional/Cleanup/Fixtures/fileadmin/' => 'fileadmin/',
];
protected function setUp(): void
{
parent::setUp();
$this->setUpBackendUserFromFixture(1);
}
/**
* @test
*/
public function removesAllData(): void
{
$this->importDataSet('EXT:events/Tests/Functional/Cleanup/Fixtures/RemoveAllTest.xml');
$subject = $this->getContainer()->get(RemoveAllCommand::class);
self::assertInstanceOf(Command::class, $subject);
$tester = new CommandTester($subject);
$tester->execute([], ['capture_stderr_separately' => true]);
self::assertSame(0, $tester->getStatusCode());
self::assertCount(
1,
$this->getAllRecords('tx_events_domain_model_partner'),
'Partners are not kept.'
);
self::assertCount(
1,
$this->getAllRecords('tx_events_domain_model_region'),
'Regions are not kept.'
);
self::assertCount(
1,
$this->getAllRecords('tx_events_domain_model_partner'),
'Partners are not kept.'
);
self::assertCount(
0,
$this->getAllRecords('tx_events_domain_model_organizer'),
'Organizers are still there.'
);
self::assertCount(
0,
$this->getAllRecords('tx_events_domain_model_event'),
'Events are still there.'
);
self::assertCount(
0,
$this->getAllRecords('tx_events_domain_model_date'),
'Dates are still there.'
);
self::assertCount(
0,
$this->getAllRecords('sys_category_record_mm'),
'Relations to categories still exist.'
);
self::assertCount(
1,
$this->getAllRecords('sys_file'),
'Unexpected number of sys_file records.'
);
self::assertCount(
1,
$this->getAllRecords('sys_file_reference'),
'Unexpected number of sys_file_reference records.'
);
self::assertCount(
1,
$this->getAllRecords('sys_file_metadata'),
'Unexpected number of sys_file_metadata records.'
);
$files = GeneralUtility::getFilesInDir('fileadmin/user_uploads');
self::assertIsArray($files, 'Failed to retrieve files from filesystem.');
self::assertCount(1, $files, 'Unexpectd number of files in filesystem.');
self::assertSame('example-for-partner.gif', array_values($files)[0], 'Unexpectd file in filesystem.');
}
}