Remove usage of uploads folder, use transient API instead

Relates: #9533

Classes/Service/DestinationDataImportService.php

@@ -647,14 +647,14 @@ class DestinationDataImportService
 
         foreach ($assets as $media_object) {
             if ($media_object['rel'] == "default" && $media_object['type'] == "image/jpeg") {
-                $orgFileUrl = urldecode($media_object['url']);
+                $fileUrl = urldecode($media_object['url']);
                 $orgFileNameSanitized = $storage->sanitizeFileName(
                     basename(
                         urldecode($media_object['url'])
                     )
                 );
 
-                $this->logger->info('File attached:' . $orgFileUrl);
+                $this->logger->info('File attached:' . $fileUrl);
                 $this->logger->info('File attached sanitized:' . $orgFileNameSanitized);
 
                 $targetFilePath = $this->environment->getPublicPath() . '/fileadmin/' . $this->filesFolder
@@ -666,9 +666,9 @@ class DestinationDataImportService
                 } else {
                     $this->logger->info("File don't exist " . $orgFileNameSanitized);
                     // Load the file
-                    if ($file = $this->loadFile($orgFileUrl)) {
+                    if ($filename = $this->loadFile($fileUrl)) {
                         // Move file to defined folder
-                        $this->logger->info('Adding file ' . $file);
+                        $this->logger->info('Adding file ' . $filename);
 
                         try {
                             $targetFolder = $storage->getFolder($this->filesFolder);
@@ -676,8 +676,7 @@ class DestinationDataImportService
                             $targetFolder = $storage->createFolder($this->filesFolder);
                         }
 
-                        $tempFilePath = $this->environment->getPublicPath() . "/uploads/tx_events/" . $file;
+                        $storage->addFile($filename, $targetFolder, basename($fileUrl));
-                        $storage->addFile($tempFilePath, $targetFolder);
                     } else {
                         $error = true;
                     }
@@ -719,21 +718,25 @@ class DestinationDataImportService
 
     private function loadFile(string $fileUrl): string
     {
-        $directory = $this->environment->getPublicPath() . "/uploads/tx_events/";
+        $this->logger->info('Getting file ' . $fileUrl);
-        $filename = basename($fileUrl);
+
-        $this->logger->info('Getting file ' . $fileUrl . ' as ' . $filename);
+        $file = new \SplFileInfo($fileUrl);
+        $temporaryFilename = GeneralUtility::tempnam($file->getBasename());
 
         $response = $this->dataFetcher->fetchImage($fileUrl);
-        $asset = $response->getBody()->__toString();
+        $fileContent = $response->getBody()->__toString();
-        if ($response->getStatusCode() === 200 && $asset !== '') {
+        if ($response->getStatusCode() !== 200) {
-            file_put_contents($directory . $filename, $asset);
+            $this->logger->error('Cannot load file ' . $fileUrl);
-            return $filename;
         }
 
-        $this->logger->error('Cannot load file ' . $fileUrl);
+        if (GeneralUtility::writeFile($temporaryFilename, $fileContent, true) === false) {
+            $this->logger->error('Could not write temporary file.');
             return '';
         }
 
+        return $temporaryFilename;
+    }
+
     private function createFileRelations(
         int $uid_local,
         string $tablenames,

Tests/Functional/Import/DestinationDataTest/AbstractTest.php

@@ -11,10 +11,6 @@ abstract class AbstractTest extends FunctionalTestCase
         'typo3conf/ext/events',
     ];
 
-    protected $additionalFoldersToCreate = [
-        'uploads/tx_events/'
-    ];
-
     protected function setUp(): void
     {
         parent::setUp();

Tests/Functional/Import/DestinationDataTest/ImportCleansTransientFilesTest.php

@@ -0,0 +1,93 @@
+<?php
+
+namespace Wrm\Events\Tests\Functional\Import\DestinationDataTest;
+
+use GuzzleHttp\ClientInterface as GuzzleClientInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Client\ClientInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Tester\CommandTester;
+use Symfony\Component\DependencyInjection\Container;
+use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use Wrm\Events\Command\DestinationDataImportCommand;
+use Wrm\Events\Tests\ClientFactory;
+
+/**
+ * @testdox DestinationData import
+ */
+class ImportCleansTransientFilesTest extends AbstractTest
+{
+    /**
+     * @test
+     */
+    public function importCleansTransientFiles(): void
+    {
+        $fileImportPath = 'staedte/beispielstadt/events/';
+        $this->setUpFrontendRootPage(1, [], [
+            'config' => implode(PHP_EOL, [
+                'module.tx_events_pi1.settings.destinationData {',
+                'restUrl = ' . $this->getInstancePath() . '/typo3conf/ext/events/Tests/Functional/Import/DestinationDataTest/Fixtures/Response.json',
+                'license = example-license',
+                'restType = Event',
+                'restLimit = 3',
+                'restMode = next_months,12',
+                'restTemplate = ET2014A.json',
+                'categoriesPid = ',
+                'categoryParentUid = ',
+                '}',
+            ]),
+        ]);
+
+        $requests = [];
+        $client = ClientFactory::createClientWithHistory([
+            new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''),
+            new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''),
+            new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''),
+        ], $requests);
+        $container = $this->getContainer();
+        if ($container instanceof Container) {
+            $container->set(ClientInterface::class, $client);
+            // For TYPO3 10 support
+            $container->set(GuzzleClientInterface::class, $client);
+        }
+
+        $subject = $this->getContainer()->get(DestinationDataImportCommand::class);
+        self::assertInstanceOf(Command::class, $subject);
+
+        $tester = new CommandTester($subject);
+        $tester->execute(
+            [
+                'storage-pid' => 2,
+                'rest-experience' => 'beispielstadt',
+                'files-folder' => $fileImportPath,
+            ],
+            [
+                'capture_stderr_separately' => true,
+            ]
+        );
+
+        self::assertSame(0, $tester->getStatusCode());
+
+        self::assertCount(3, $requests, 'Unexpected number of requests were made.');
+        self::assertSame('https://dam.destination.one/849917/279ac45b3fc701a7197131f627164fffd9f8cc77bc75165e2fc2b864ed606920/theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg', (string)$requests[0]['request']->getUri());
+        self::assertSame('https://dam.destination.one/828118/f13bbf5602ffc406ebae2faa3527654dea84194666bce4925a1ca8bd3f50c5e9/tueftlerzeit-sfz-rudolstadt-jpg.jpg', (string)$requests[1]['request']->getUri());
+        self::assertSame('https://dam.destination.one/853436/109ac1cf87913e21b5e2b0ef0cc63d223a14374364952a855746a8e7c3fcfc36/lutherkirche-jpg.jpg', (string)$requests[2]['request']->getUri());
+
+        $importedFiles = GeneralUtility::getFilesInDir($this->getInstancePath() . '/fileadmin/' . $fileImportPath);
+        self::assertIsArray($importedFiles, 'Failed to retrieve imported files from filesystem.');
+        self::assertSame(
+            [
+                'lutherkirche-jpg.jpg',
+                'theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg',
+                'tueftlerzeit-sfz-rudolstadt-jpg.jpg',
+            ],
+            array_values($importedFiles),
+            'Got unexpected number of files'
+        );
+
+        $transientFiles = GeneralUtility::getFilesInDir(Environment::getVarPath() . '/transient/');
+        self::assertIsArray($transientFiles, 'Failed to retrieve transient files from filesystem.');
+        self::assertCount(0, $transientFiles, 'Got unexpected number of files');
+    }
+}

Tests/Functional/Import/DestinationDataTest/ImportDoesNotUseUploadsFolderTest.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace Wrm\Events\Tests\Functional\Import\DestinationDataTest;
+
+use GuzzleHttp\ClientInterface as GuzzleClientInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Client\ClientInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Tester\CommandTester;
+use Symfony\Component\DependencyInjection\Container;
+use TYPO3\CMS\Core\Core\Environment;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use Wrm\Events\Command\DestinationDataImportCommand;
+use Wrm\Events\Tests\ClientFactory;
+
+/**
+ * @testdox DestinationData import
+ */
+class ImportDoesNotUserUploadsFolderTest extends AbstractTest
+{
+    /**
+     * @test
+     */
+    public function importDoesNotUserUploadsFolder(): void
+    {
+        $fileImportPath = 'staedte/beispielstadt/events/';
+        $this->setUpFrontendRootPage(1, [], [
+            'config' => implode(PHP_EOL, [
+                'module.tx_events_pi1.settings.destinationData {',
+                'restUrl = ' . $this->getInstancePath() . '/typo3conf/ext/events/Tests/Functional/Import/DestinationDataTest/Fixtures/Response.json',
+                'license = example-license',
+                'restType = Event',
+                'restLimit = 3',
+                'restMode = next_months,12',
+                'restTemplate = ET2014A.json',
+                'categoriesPid = ',
+                'categoryParentUid = ',
+                '}',
+            ]),
+        ]);
+
+        $requests = [];
+        $client = ClientFactory::createClientWithHistory([
+            new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''),
+            new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''),
+            new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''),
+        ], $requests);
+        $container = $this->getContainer();
+        if ($container instanceof Container) {
+            $container->set(ClientInterface::class, $client);
+            // For TYPO3 10 support
+            $container->set(GuzzleClientInterface::class, $client);
+        }
+
+        $subject = $this->getContainer()->get(DestinationDataImportCommand::class);
+        self::assertInstanceOf(Command::class, $subject);
+
+        $tester = new CommandTester($subject);
+        $tester->execute(
+            [
+                'storage-pid' => 2,
+                'rest-experience' => 'beispielstadt',
+                'files-folder' => $fileImportPath,
+            ],
+            [
+                'capture_stderr_separately' => true,
+            ]
+        );
+
+        self::assertSame(0, $tester->getStatusCode());
+
+        self::assertCount(3, $requests, 'Unexpected number of requests were made.');
+        self::assertSame('https://dam.destination.one/849917/279ac45b3fc701a7197131f627164fffd9f8cc77bc75165e2fc2b864ed606920/theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg', (string)$requests[0]['request']->getUri());
+        self::assertSame('https://dam.destination.one/828118/f13bbf5602ffc406ebae2faa3527654dea84194666bce4925a1ca8bd3f50c5e9/tueftlerzeit-sfz-rudolstadt-jpg.jpg', (string)$requests[1]['request']->getUri());
+        self::assertSame('https://dam.destination.one/853436/109ac1cf87913e21b5e2b0ef0cc63d223a14374364952a855746a8e7c3fcfc36/lutherkirche-jpg.jpg', (string)$requests[2]['request']->getUri());
+
+        $importedFiles = GeneralUtility::getFilesInDir($this->getInstancePath() . '/fileadmin/' . $fileImportPath);
+        self::assertIsArray($importedFiles, 'Failed to retrieve imported files from filesystem.');
+        self::assertSame(
+            [
+                'lutherkirche-jpg.jpg',
+                'theater-rudolstadt_johannes-gei-er_photo-by-lisa-stern_web_-jpg.jpg',
+                'tueftlerzeit-sfz-rudolstadt-jpg.jpg',
+            ],
+            array_values($importedFiles),
+            'Got unexpected number of files'
+        );
+
+        self::assertFalse(file_exists(Environment::getPublicPath() . '/uploads/tx_events/'), 'Uploads folder exists.');
+    }
+}

ext_emconf.php

@@ -7,7 +7,6 @@ $EM_CONF['events'] = [
     'author' => 'Dirk Koritnik',
     'author_email' => 'koritnik@werkraum-media.de',
     'state' => 'alpha',
-    'uploadfolder' => 1,
     'createDirs' => '',
     'clearCacheOnLoad' => 0,
     'version' => '2.0.0',