From 15dec4944504448621039e3b0cfed08dee8ad863 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 16 Dec 2021 10:00:51 +0100 Subject: [PATCH] Remove usage of uploads folder, use transient API instead Relates: #9533 --- .../Service/DestinationDataImportService.php | 33 ++++--- .../DestinationDataTest/AbstractTest.php | 4 - .../ImportCleansTransientFilesTest.php | 93 +++++++++++++++++++ .../ImportDoesNotUseUploadsFolderTest.php | 91 ++++++++++++++++++ ext_emconf.php | 1 - 5 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 Tests/Functional/Import/DestinationDataTest/ImportCleansTransientFilesTest.php create mode 100644 Tests/Functional/Import/DestinationDataTest/ImportDoesNotUseUploadsFolderTest.php diff --git a/Classes/Service/DestinationDataImportService.php b/Classes/Service/DestinationDataImportService.php index 5423574..0bef308 100644 --- a/Classes/Service/DestinationDataImportService.php +++ b/Classes/Service/DestinationDataImportService.php @@ -647,14 +647,14 @@ class DestinationDataImportService foreach ($assets as $media_object) { if ($media_object['rel'] == "default" && $media_object['type'] == "image/jpeg") { - $orgFileUrl = urldecode($media_object['url']); + $fileUrl = urldecode($media_object['url']); $orgFileNameSanitized = $storage->sanitizeFileName( basename( urldecode($media_object['url']) ) ); - $this->logger->info('File attached:' . $orgFileUrl); + $this->logger->info('File attached:' . $fileUrl); $this->logger->info('File attached sanitized:' . $orgFileNameSanitized); $targetFilePath = $this->environment->getPublicPath() . '/fileadmin/' . $this->filesFolder @@ -666,9 +666,9 @@ class DestinationDataImportService } else { $this->logger->info("File don't exist " . $orgFileNameSanitized); // Load the file - if ($file = $this->loadFile($orgFileUrl)) { + if ($filename = $this->loadFile($fileUrl)) { // Move file to defined folder - $this->logger->info('Adding file ' . $file); + $this->logger->info('Adding file ' . $filename); try { $targetFolder = $storage->getFolder($this->filesFolder); @@ -676,8 +676,7 @@ class DestinationDataImportService $targetFolder = $storage->createFolder($this->filesFolder); } - $tempFilePath = $this->environment->getPublicPath() . "/uploads/tx_events/" . $file; - $storage->addFile($tempFilePath, $targetFolder); + $storage->addFile($filename, $targetFolder, basename($fileUrl)); } else { $error = true; } @@ -719,19 +718,23 @@ class DestinationDataImportService private function loadFile(string $fileUrl): string { - $directory = $this->environment->getPublicPath() . "/uploads/tx_events/"; - $filename = basename($fileUrl); - $this->logger->info('Getting file ' . $fileUrl . ' as ' . $filename); + $this->logger->info('Getting file ' . $fileUrl); + + $file = new \SplFileInfo($fileUrl); + $temporaryFilename = GeneralUtility::tempnam($file->getBasename()); $response = $this->dataFetcher->fetchImage($fileUrl); - $asset = $response->getBody()->__toString(); - if ($response->getStatusCode() === 200 && $asset !== '') { - file_put_contents($directory . $filename, $asset); - return $filename; + $fileContent = $response->getBody()->__toString(); + if ($response->getStatusCode() !== 200) { + $this->logger->error('Cannot load file ' . $fileUrl); } - $this->logger->error('Cannot load file ' . $fileUrl); - return ''; + if (GeneralUtility::writeFile($temporaryFilename, $fileContent, true) === false) { + $this->logger->error('Could not write temporary file.'); + return ''; + } + + return $temporaryFilename; } private function createFileRelations( diff --git a/Tests/Functional/Import/DestinationDataTest/AbstractTest.php b/Tests/Functional/Import/DestinationDataTest/AbstractTest.php index a6fe0af..38847bd 100644 --- a/Tests/Functional/Import/DestinationDataTest/AbstractTest.php +++ b/Tests/Functional/Import/DestinationDataTest/AbstractTest.php @@ -11,10 +11,6 @@ abstract class AbstractTest extends FunctionalTestCase 'typo3conf/ext/events', ]; - protected $additionalFoldersToCreate = [ - 'uploads/tx_events/' - ]; - protected function setUp(): void { parent::setUp(); diff --git a/Tests/Functional/Import/DestinationDataTest/ImportCleansTransientFilesTest.php b/Tests/Functional/Import/DestinationDataTest/ImportCleansTransientFilesTest.php new file mode 100644 index 0000000..a85cda3 --- /dev/null +++ b/Tests/Functional/Import/DestinationDataTest/ImportCleansTransientFilesTest.php @@ -0,0 +1,93 @@ +setUpFrontendRootPage(1, [], [ + 'config' => implode(PHP_EOL, [ + 'module.tx_events_pi1.settings.destinationData {', + 'restUrl = ' . $this->getInstancePath() . '/typo3conf/ext/events/Tests/Functional/Import/DestinationDataTest/Fixtures/Response.json', + 'license = example-license', + 'restType = Event', + 'restLimit = 3', + 'restMode = next_months,12', + 'restTemplate = ET2014A.json', + 'categoriesPid = ', + 'categoryParentUid = ', + '}', + ]), + ]); + + $requests = []; + $client = ClientFactory::createClientWithHistory([ + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + ], $requests); + $container = $this->getContainer(); + if ($container instanceof Container) { + $container->set(ClientInterface::class, $client); + // For TYPO3 10 support + $container->set(GuzzleClientInterface::class, $client); + } + + $subject = $this->getContainer()->get(DestinationDataImportCommand::class); + self::assertInstanceOf(Command::class, $subject); + + $tester = new CommandTester($subject); + $tester->execute( + [ + 'storage-pid' => 2, + 'rest-experience' => 'beispielstadt', + 'files-folder' => $fileImportPath, + ], + [ + 'capture_stderr_separately' => true, + ] + ); + + self::assertSame(0, $tester->getStatusCode()); + + self::assertCount(3, $requests, 'Unexpected number of requests were made.'); + self::assertSame('https://dam.destination.one/849917/279ac45b3fc701a7197131f627164fffd9f8cc77bc75165e2fc2b864ed606920/theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg', (string)$requests[0]['request']->getUri()); + self::assertSame('https://dam.destination.one/828118/f13bbf5602ffc406ebae2faa3527654dea84194666bce4925a1ca8bd3f50c5e9/tueftlerzeit-sfz-rudolstadt-jpg.jpg', (string)$requests[1]['request']->getUri()); + self::assertSame('https://dam.destination.one/853436/109ac1cf87913e21b5e2b0ef0cc63d223a14374364952a855746a8e7c3fcfc36/lutherkirche-jpg.jpg', (string)$requests[2]['request']->getUri()); + + $importedFiles = GeneralUtility::getFilesInDir($this->getInstancePath() . '/fileadmin/' . $fileImportPath); + self::assertIsArray($importedFiles, 'Failed to retrieve imported files from filesystem.'); + self::assertSame( + [ + 'lutherkirche-jpg.jpg', + 'theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg', + 'tueftlerzeit-sfz-rudolstadt-jpg.jpg', + ], + array_values($importedFiles), + 'Got unexpected number of files' + ); + + $transientFiles = GeneralUtility::getFilesInDir(Environment::getVarPath() . '/transient/'); + self::assertIsArray($transientFiles, 'Failed to retrieve transient files from filesystem.'); + self::assertCount(0, $transientFiles, 'Got unexpected number of files'); + } +} diff --git a/Tests/Functional/Import/DestinationDataTest/ImportDoesNotUseUploadsFolderTest.php b/Tests/Functional/Import/DestinationDataTest/ImportDoesNotUseUploadsFolderTest.php new file mode 100644 index 0000000..87bb616 --- /dev/null +++ b/Tests/Functional/Import/DestinationDataTest/ImportDoesNotUseUploadsFolderTest.php @@ -0,0 +1,91 @@ +setUpFrontendRootPage(1, [], [ + 'config' => implode(PHP_EOL, [ + 'module.tx_events_pi1.settings.destinationData {', + 'restUrl = ' . $this->getInstancePath() . '/typo3conf/ext/events/Tests/Functional/Import/DestinationDataTest/Fixtures/Response.json', + 'license = example-license', + 'restType = Event', + 'restLimit = 3', + 'restMode = next_months,12', + 'restTemplate = ET2014A.json', + 'categoriesPid = ', + 'categoryParentUid = ', + '}', + ]), + ]); + + $requests = []; + $client = ClientFactory::createClientWithHistory([ + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + ], $requests); + $container = $this->getContainer(); + if ($container instanceof Container) { + $container->set(ClientInterface::class, $client); + // For TYPO3 10 support + $container->set(GuzzleClientInterface::class, $client); + } + + $subject = $this->getContainer()->get(DestinationDataImportCommand::class); + self::assertInstanceOf(Command::class, $subject); + + $tester = new CommandTester($subject); + $tester->execute( + [ + 'storage-pid' => 2, + 'rest-experience' => 'beispielstadt', + 'files-folder' => $fileImportPath, + ], + [ + 'capture_stderr_separately' => true, + ] + ); + + self::assertSame(0, $tester->getStatusCode()); + + self::assertCount(3, $requests, 'Unexpected number of requests were made.'); + self::assertSame('https://dam.destination.one/849917/279ac45b3fc701a7197131f627164fffd9f8cc77bc75165e2fc2b864ed606920/theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg', (string)$requests[0]['request']->getUri()); + self::assertSame('https://dam.destination.one/828118/f13bbf5602ffc406ebae2faa3527654dea84194666bce4925a1ca8bd3f50c5e9/tueftlerzeit-sfz-rudolstadt-jpg.jpg', (string)$requests[1]['request']->getUri()); + self::assertSame('https://dam.destination.one/853436/109ac1cf87913e21b5e2b0ef0cc63d223a14374364952a855746a8e7c3fcfc36/lutherkirche-jpg.jpg', (string)$requests[2]['request']->getUri()); + + $importedFiles = GeneralUtility::getFilesInDir($this->getInstancePath() . '/fileadmin/' . $fileImportPath); + self::assertIsArray($importedFiles, 'Failed to retrieve imported files from filesystem.'); + self::assertSame( + [ + 'lutherkirche-jpg.jpg', + 'theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg', + 'tueftlerzeit-sfz-rudolstadt-jpg.jpg', + ], + array_values($importedFiles), + 'Got unexpected number of files' + ); + + self::assertFalse(file_exists(Environment::getPublicPath() . '/uploads/tx_events/'), 'Uploads folder exists.'); + } +} diff --git a/ext_emconf.php b/ext_emconf.php index 4fbd39f..f86bda0 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,7 +7,6 @@ $EM_CONF['events'] = [ 'author' => 'Dirk Koritnik', 'author_email' => 'koritnik@werkraum-media.de', 'state' => 'alpha', - 'uploadfolder' => 1, 'createDirs' => '', 'clearCacheOnLoad' => 0, 'version' => '2.0.0',