Extend cleanup

Properly cleanup system.

Delete further records when deleting everything.
Also respect further records when purging old entries.

Respect:

* sys_category_record_mm
* sys_file_reference
* sys_file_metadata
This commit is contained in:
Daniel Siepmann 2021-09-07 09:49:03 +02:00
parent 10237d6f91
commit 1f769939b4
5 changed files with 144 additions and 98 deletions

View file

@ -6,12 +6,23 @@ use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Output\OutputInterface;
use TYPO3\CMS\Core\Core\Bootstrap; use TYPO3\CMS\Core\Core\Bootstrap;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use Wrm\Events\Service\CleanupService; use Wrm\Events\Service\CleanupService;
class RemoveAllCommand extends Command class RemoveAllCommand extends Command
{ {
/**
* @var CleanupService
*/
private $cleanupService;
public function __construct(
CleanupService $cleanupService
) {
$this->cleanupService = $cleanupService;
parent::__construct();
}
public function configure(): void public function configure(): void
{ {
$this->setDescription('Remove all event data'); $this->setDescription('Remove all event data');
@ -21,10 +32,7 @@ class RemoveAllCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output) protected function execute(InputInterface $input, OutputInterface $output)
{ {
Bootstrap::initializeBackendAuthentication(); Bootstrap::initializeBackendAuthentication();
$this->cleanupService->deleteAllData();
GeneralUtility::makeInstance(ObjectManager::class)
->get(CleanupService::class)
->deleteAllData();
return 0; return 0;
} }

View file

@ -6,12 +6,23 @@ use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Output\OutputInterface;
use TYPO3\CMS\Core\Core\Bootstrap; use TYPO3\CMS\Core\Core\Bootstrap;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use Wrm\Events\Service\CleanupService; use Wrm\Events\Service\CleanupService;
class RemovePastCommand extends Command class RemovePastCommand extends Command
{ {
/**
* @var CleanupService
*/
private $cleanupService;
public function __construct(
CleanupService $cleanupService
) {
$this->cleanupService = $cleanupService;
parent::__construct();
}
public function configure(): void public function configure(): void
{ {
$this->setDescription('Remove past events'); $this->setDescription('Remove past events');
@ -22,9 +33,7 @@ class RemovePastCommand extends Command
{ {
Bootstrap::initializeBackendAuthentication(); Bootstrap::initializeBackendAuthentication();
GeneralUtility::makeInstance(ObjectManager::class) $this->cleanupService->deletePastData();
->get(CleanupService::class)
->deletePastData();
return 0; return 0;
} }
} }

View file

@ -21,65 +21,68 @@ namespace Wrm\Events\Service\Cleanup;
* 02110-1301, USA. * 02110-1301, USA.
*/ */
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Database\Connection; use TYPO3\CMS\Core\Database\Connection;
use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\GeneralUtility;
class Database class Database
{ {
public const DATE_TABLE = 'tx_events_domain_model_date'; /**
public const EVENT_TABLE = 'tx_events_domain_model_event'; * @var ConnectionPool
public const ORGANIZER_TABLE = 'tx_events_domain_model_organizer'; */
private $connectionPool;
public function truncateTables(string ...$tableNames): void /**
* @var DataHandler
*/
private $dataHandler;
private const DATE_TABLE = 'tx_events_domain_model_date';
private const EVENT_TABLE = 'tx_events_domain_model_event';
private const ORGANIZER_TABLE = 'tx_events_domain_model_organizer';
private const REGION_TABLE = 'tx_events_domain_model_region';
public function __construct(
ConnectionPool $connectionPool,
DataHandler $dataHandler
) {
$this->connectionPool = $connectionPool;
$this->dataHandler = $dataHandler;
}
public function truncateTables(): void
{ {
$tableNames = [
Database::DATE_TABLE,
Database::ORGANIZER_TABLE,
Database::EVENT_TABLE,
Database::REGION_TABLE,
];
foreach ($tableNames as $tableName) { foreach ($tableNames as $tableName) {
GeneralUtility::makeInstance(ConnectionPool::class) $this->connectionPool
->getConnectionForTable($tableName) ->getConnectionForTable($tableName)
->truncate($tableName); ->truncate($tableName);
} }
} $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_category_record_mm');
$queryBuilder->delete('sys_category_record_mm')
public function getDeletionStructureForEvents(): array ->where($queryBuilder->expr()->like(
{ 'tablenames',
$dataStructure = [static::EVENT_TABLE => []]; $queryBuilder->createNamedParameter('tx_events_domain_model_%')
))
foreach ($this->getAllRecords(static::EVENT_TABLE) as $recordToDelete) { ->execute();
$dataStructure[static::EVENT_TABLE][$recordToDelete] = ['delete' => 1];
}
return $dataStructure;
}
private function getAllRecords(string $tableName): array
{
/* @var QueryBuilder $queryBuilder */
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getConnectionForTable($tableName)
->createQueryBuilder();
$records = $queryBuilder->select('uid')
->from($tableName)
->execute()
->fetchAll();
return array_map(function (array $record) {
return $record['uid'];
}, $records);
} }
public function getPastDates(): array public function getPastDates(): array
{ {
$midnightToday = new \DateTimeImmutable('midnight today'); $queryBuilder = $this->connectionPool
/* @var QueryBuilder $queryBuilder */
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getConnectionForTable(static::DATE_TABLE) ->getConnectionForTable(static::DATE_TABLE)
->createQueryBuilder(); ->createQueryBuilder();
$queryBuilder->getRestrictions()->removeAll(); $queryBuilder->getRestrictions()->removeAll();
$midnightToday = new \DateTimeImmutable('midnight today');
$records = $queryBuilder->select('uid') $records = $queryBuilder->select('uid')
->from(static::DATE_TABLE) ->from(static::DATE_TABLE)
->where($queryBuilder->expr()->lte( ->where($queryBuilder->expr()->lte(
@ -96,8 +99,7 @@ class Database
public function deleteDates(int ...$uids): void public function deleteDates(int ...$uids): void
{ {
/* @var QueryBuilder $queryBuilder */ $queryBuilder = $this->connectionPool
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable(static::DATE_TABLE); ->getQueryBuilderForTable(static::DATE_TABLE);
$queryBuilder->delete(static::DATE_TABLE) $queryBuilder->delete(static::DATE_TABLE)
@ -106,26 +108,42 @@ class Database
->execute(); ->execute();
} }
public function getDeletionStructureForEventsWithoutDates(): array public function deleteEventsWithoutDates(): void
{ {
$dataStructure = [static::EVENT_TABLE => []]; $queryBuilder = $this->connectionPool
/* @var QueryBuilder $queryBuilder */
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getConnectionForTable(static::EVENT_TABLE) ->getConnectionForTable(static::EVENT_TABLE)
->createQueryBuilder(); ->createQueryBuilder();
$queryBuilder->getRestrictions()->removeAll(); $queryBuilder->getRestrictions()->removeAll();
$records = $queryBuilder->select('event.uid') $recordUids = $queryBuilder->select('event.uid')
->from(static::EVENT_TABLE, 'event') ->from(static::EVENT_TABLE, 'event')
->leftJoin('event', static::DATE_TABLE, 'date', $queryBuilder->expr()->eq('date.event', 'event.uid')) ->leftJoin('event', static::DATE_TABLE, 'date', $queryBuilder->expr()->eq('date.event', 'event.uid'))
->where($queryBuilder->expr()->isNull('date.uid')) ->where($queryBuilder->expr()->isNull('date.uid'))
->execute() ->execute()
->fetchAll(); ->fetchAll(\PDO::FETCH_COLUMN);
foreach ($records as $record) { $dataStructure = [static::EVENT_TABLE => []];
$dataStructure[static::EVENT_TABLE][$record['uid']] = ['delete' => 1]; foreach ($recordUids as $recordUid) {
$dataStructure[static::EVENT_TABLE][$recordUid] = ['delete' => 1];
} }
return $dataStructure;
$dataHandler = clone $this->dataHandler;
$dataHandler->start([], $dataStructure);
$dataHandler->process_cmdmap();
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_category_record_mm');
$queryBuilder->delete('sys_category_record_mm')
->where($queryBuilder->expr()->andX(
$queryBuilder->expr()->like(
'tablenames',
$queryBuilder->createNamedParameter('tx_events_domain_model_%')
),
$queryBuilder->expr()->in(
'uid_foreign',
$queryBuilder->createNamedParameter($recordUids, Connection::PARAM_INT_ARRAY)
)
))
->execute();
} }
} }

View file

@ -26,10 +26,27 @@ use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Resource\ResourceStorage; use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\StorageRepository; use TYPO3\CMS\Core\Resource\StorageRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
class Files class Files
{ {
/**
* @var ConnectionPool
*/
private $connectionPool;
/**
* @var StorageRepository
*/
private $storageRepository;
public function __construct(
ConnectionPool $connectionPool,
StorageRepository $storageRepository
) {
$this->connectionPool = $connectionPool;
$this->storageRepository = $storageRepository;
}
public function deleteAll() public function deleteAll()
{ {
$this->delete($this->getFilesFromDb()); $this->delete($this->getFilesFromDb());
@ -54,23 +71,9 @@ class Files
})); }));
} }
private function delete(array $filesToDelete)
{
$uidsToRemove = [];
foreach ($filesToDelete as $fileToDelete) {
$this->deleteFromFal($fileToDelete['storage'], $fileToDelete['identifier']);
$uidsToRemove[] = $fileToDelete['uid'];
}
$this->deleteFromDb(...$uidsToRemove);
}
private function getFilesFromDb(callable $whereGenerator = null): array private function getFilesFromDb(callable $whereGenerator = null): array
{ {
/* @var QueryBuilder $queryBuilder */ $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('sys_file');
$queryBuilder->getRestrictions()->removeAll(); $queryBuilder->getRestrictions()->removeAll();
@ -88,11 +91,21 @@ class Files
return $queryBuilder->execute()->fetchAll(); return $queryBuilder->execute()->fetchAll();
} }
private function deleteFromFal(int $storageUid, string $filePath) private function delete(array $filesToDelete): void
{ {
/* @var ResourceStorage $storage */ $uidsToRemove = [];
$storage = GeneralUtility::makeInstance(StorageRepository::class)
->findByUid($storageUid); foreach ($filesToDelete as $fileToDelete) {
$this->deleteFromFal($fileToDelete['storage'], $fileToDelete['identifier']);
$uidsToRemove[] = $fileToDelete['uid'];
}
$this->deleteFromDb(...$uidsToRemove);
}
private function deleteFromFal(int $storageUid, string $filePath): void
{
$storage = $this->storageRepository->findByUid($storageUid);
if ($storage->hasFile($filePath) === false) { if ($storage->hasFile($filePath) === false) {
return; return;
@ -101,15 +114,24 @@ class Files
$storage->deleteFile($storage->getFile($filePath)); $storage->deleteFile($storage->getFile($filePath));
} }
private function deleteFromDb(int ...$uids) private function deleteFromDb(int ...$uids): void
{ {
/* @var QueryBuilder $queryBuilder */ $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file');
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('sys_file');
$queryBuilder->delete('sys_file') $queryBuilder->delete('sys_file')
->where('uid in (:uids)') ->where('uid in (:uids)')
->setParameter(':uids', $uids, Connection::PARAM_INT_ARRAY) ->setParameter(':uids', $uids, Connection::PARAM_INT_ARRAY)
->execute(); ->execute();
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_reference');
$queryBuilder->delete('sys_file_reference')
->where('uid_local in (:uids)')
->setParameter(':uids', $uids, Connection::PARAM_INT_ARRAY)
->execute();
$queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata');
$queryBuilder->delete('sys_file_metadata')
->where('file in (:uids)')
->setParameter(':uids', $uids, Connection::PARAM_INT_ARRAY)
->execute();
} }
} }

View file

@ -2,8 +2,6 @@
namespace Wrm\Events\Service; namespace Wrm\Events\Service;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use Wrm\Events\Service\Cleanup\Database; use Wrm\Events\Service\Cleanup\Database;
use Wrm\Events\Service\Cleanup\Files; use Wrm\Events\Service\Cleanup\Files;
@ -27,23 +25,14 @@ class CleanupService
public function deleteAllData(): void public function deleteAllData(): void
{ {
$this->database->truncateTables(...[Database::DATE_TABLE, Database::ORGANIZER_TABLE]); $this->database->truncateTables();
$this->removeViaDataHandler($this->database->getDeletionStructureForEvents());
$this->files->deleteAll(); $this->files->deleteAll();
} }
public function deletePastData(): void public function deletePastData(): void
{ {
$this->database->deleteDates(...$this->database->getPastDates()); $this->database->deleteDates(...$this->database->getPastDates());
$this->removeViaDataHandler($this->database->getDeletionStructureForEventsWithoutDates()); $this->database->deleteEventsWithoutDates();
$this->files->deleteDangling(); $this->files->deleteDangling();
} }
private function removeViaDataHandler(array $structure): void
{
/* @var DataHandler $dataHandler */
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
$dataHandler->start([], $structure);
$dataHandler->process_cmdmap();
}
} }