Work-Related/thuecat
Work-Related/thuecat
1
0
mirror of https://github.com/werkraum-media/thuecat.git synced 2024-12-05 03:26:13 +01:00
Code Issues Activity

Allow to configure storage pid for each import configuration

Browse source 
Each import configuration now offers an input to define storage pid.
That enables integrators to have different storage folders per
configuration.
This commit is contained in:
Daniel Siepmann 2021-02-17 15:33:15 +01:00
parent 92c8bbedaa
commit 2e0102894f
20 changed files with 138 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
Classes/Domain/Import/Converter/Converter.php
View file

@ -24,6 +24,7 @@ namespace WerkraumMedia\ThueCat\Domain\Import\Converter;
*/
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
interface Converter
{
@ -37,5 +38,5 @@ interface Converter
* A single JSONLD entity can have multiple languages.
* That may result in multiple entities in TYPO3.
*/
public function convert(array $jsonLD): EntityCollection;
public function convert(array $jsonLD, ImportConfiguration $configuration): EntityCollection;
}

5
Classes/Domain/Import/Converter/Organisation.php
View file

@ -27,6 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Import\Model\GenericEntity;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
class Organisation implements Converter
{
@ -38,11 +39,11 @@ class Organisation implements Converter
$this->parser = $parser;
}
public function convert(array $jsonLD): EntityCollection
public function convert(array $jsonLD, ImportConfiguration $configuration): EntityCollection
{
$entity = GeneralUtility::makeInstance(
GenericEntity::class,
10,
$configuration->getStoragePid(),
'tx_thuecat_organisation',
0,
$this->parser->getId($jsonLD),

5
Classes/Domain/Import/Converter/TouristAttraction.php
View file

@ -28,6 +28,7 @@ use WerkraumMedia\ThueCat\Domain\Import\Importer\LanguageHandling;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Import\Model\GenericEntity;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\OrganisationRepository;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\TownRepository;
@ -50,12 +51,12 @@ class TouristAttraction implements Converter
$this->townRepository = $townRepository;
}
public function convert(array $jsonLD): EntityCollection
public function convert(array $jsonLD, ImportConfiguration $configuration): EntityCollection
{
$storagePid = 10;
$manager = $this->organisationRepository->findOneByRemoteId($this->parser->getManagerId($jsonLD));
$town = $this->townRepository->findOneByRemoteIds($this->parser->getContainedInPlaceIds($jsonLD));
$entities = GeneralUtility::makeInstance(EntityCollection::class);
$storagePid = $configuration->getStoragePid();
foreach ($this->parser->getLanguages($jsonLD) as $language) {
if ($this->language->isUnknown($language, $storagePid)) {

5
Classes/Domain/Import/Converter/TouristInformation.php
View file

@ -27,6 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Import\Model\GenericEntity;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\OrganisationRepository;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\TownRepository;
@ -46,7 +47,7 @@ class TouristInformation implements Converter
$this->townRepository = $townRepository;
}
public function convert(array $jsonLD): EntityCollection
public function convert(array $jsonLD, ImportConfiguration $configuration): EntityCollection
{
$manager = $this->organisationRepository->findOneByRemoteId(
$this->parser->getManagerId($jsonLD)
@ -57,7 +58,7 @@ class TouristInformation implements Converter
$entity = GeneralUtility::makeInstance(
GenericEntity::class,
10,
$configuration->getStoragePid(),
'tx_thuecat_tourist_information',
0,
$this->parser->getId($jsonLD),

5
Classes/Domain/Import/Converter/Town.php
View file

@ -27,6 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Import\Model\GenericEntity;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\OrganisationRepository;
class Town implements Converter
@ -42,7 +43,7 @@ class Town implements Converter
$this->organisationRepository = $organisationRepository;
}
public function convert(array $jsonLD): EntityCollection
public function convert(array $jsonLD, ImportConfiguration $configuration): EntityCollection
{
$manager = $this->organisationRepository->findOneByRemoteId(
$this->parser->getManagerId($jsonLD)
@ -50,7 +51,7 @@ class Town implements Converter
$entity = GeneralUtility::makeInstance(
GenericEntity::class,
10,
$configuration->getStoragePid(),
'tx_thuecat_town',
0,
$this->parser->getId($jsonLD),

10
Classes/Domain/Import/Importer.php
View file

@ -42,6 +42,7 @@ class Importer
private SaveData $saveData;
private ImportLog $importLog;
private ImportLogRepository $importLogRepository;
private ImportConfiguration $configuration;
public function __construct(
UrlProviderRegistry $urls,
@ -59,9 +60,11 @@ class Importer
public function importConfiguration(ImportConfiguration $configuration): ImportLog
{
$this->importLog = GeneralUtility::makeInstance(ImportLog::class, $configuration);
$this->configuration = $configuration;
$urlProvider = $this->urls->getProviderForConfiguration($configuration);
$this->importLog = GeneralUtility::makeInstance(ImportLog::class, $this->configuration);
$urlProvider = $this->urls->getProviderForConfiguration($this->configuration);
if (!$urlProvider instanceof UrlProvider) {
return $this->importLog;
}
@ -91,7 +94,8 @@ class Importer
{
$converter = $this->converter->getConverterBasedOnType($jsonEntity['@type']);
if ($converter instanceof Converter) {
$this->saveData->import($converter->convert($jsonEntity), $this->importLog);
$entities = $converter->convert($jsonEntity, $this->configuration);
$this->saveData->import($entities, $this->importLog);
return;
}
}

18
Classes/Domain/Model/Backend/ImportConfiguration.php
View file

@ -54,6 +54,24 @@ class ImportConfiguration extends AbstractEntity
return $this->tstamp;
}
public function getStoragePid(): int
{
if ($this->configuration === '') {
return 0;
}
$storagePid = ArrayUtility::getValueByPath(
GeneralUtility::xml2array($this->configuration),
'data/sDEF/lDEF/storagePid/vDEF'
);
if (is_numeric($storagePid) && $storagePid > 0) {
return intval($storagePid);
}
return 0;
}
public function getUrls(): array
{
if ($this->configuration === '') {

9
Configuration/FlexForm/ImportConfiguration/Static.xml
View file

@ -10,6 +10,15 @@
</TCEforms>
<type>array</type>
<el>
<storagePid>
<TCEforms>
<label>LLL:EXT:thuecat/Resources/Private/Language/locallang_flexform.xlf:importConfiguration.static.storagePid</label>
<config>
<type>input</type>
<eval>int,required</eval>
</config>
</TCEforms>
</storagePid>
<urls>
<title>LLL:EXT:thuecat/Resources/Private/Language/locallang_flexform.xlf:importConfiguration.static.urls</title>
<section>1</section>

11
README.md
View file

@ -12,6 +12,8 @@ The extension already allows:
* Create static configuration to import specified resources,
e.g. defined organisation or towns.
* Support multiple languages
* Import of the following types (not all properties are supported):
* Organisations (responsible for content)
@ -34,13 +36,14 @@ The extension already allows:
* To inspect past imports and possible errors,
as well as number of affected records
* Frontend:
* Content element to display tourist attraction
## Short time goals
* Integrate proper icons
* Content element to display tourist attraction,
town, tourist information and organisation.
* Content element to display town, tourist information and organisation.
* Extending import to include further properties
* Support multiple languages

3
Resources/Private/Language/locallang_flexform.xlf
View file

@ -6,6 +6,9 @@
<trans-unit id="importConfiguration.static.sheetTitle" xml:space="preserve">
<source>Static import configuration</source>
</trans-unit>
<trans-unit id="importConfiguration.static.storagePid" xml:space="preserve">
<source>Storage Page UID</source>
</trans-unit>
<trans-unit id="importConfiguration.static.urls" xml:space="preserve">
<source>URLs</source>
</trans-unit>

9
Tests/Functional/Fixtures/Import/ImportsFreshOrganization.csv Normal file
View file

@ -0,0 +1,9 @@
tx_thuecat_organisation
,"uid","pid","remote_id","title"
,1,11,"https://thuecat.org/resources/018132452787-ngbe","Erfurt Tourismus und Marketing GmbH"
tx_thuecat_import_log
,"uid","pid","configuration"
,1,0,1
tx_thuecat_import_log_entry
,"uid","pid","import_log","record_uid","table_name","insertion","errors"
,1,0,1,1,"tx_thuecat_organisation",1,"[]"
Can't render this file because it has a wrong number of fields in line 2.

5
Tests/Functional/Fixtures/Import/ImportsFreshOrganization.xml
View file

@ -11,7 +11,7 @@
<is_siteroot>1</is_siteroot>
</pages>
<pages>
<uid>10</uid>
<uid>11</uid>
<pid>1</pid>
<tstamp>1613400587</tstamp>
<crdate>1613400558</crdate>
@ -34,6 +34,9 @@
<data>
<sheet index="sDEF">
<language index="lDEF">
<field index="storagePid">
<value index="vDEF">11</value>
</field>
<field index="urls">
<el index="el">
<field index="602a89e212237114263881">

3
Tests/Functional/Fixtures/Import/ImportsTouristAttractionsWithRelations.xml
View file

@ -50,6 +50,9 @@
<data>
<sheet index="sDEF">
<language index="lDEF">
<field index="storagePid">
<value index="vDEF">10</value>
</field>
<field index="urls">
<el index="el">
<field index="602a89f54d694654233086">

15
Tests/Functional/Fixtures/Import/UpdatesExistingOrganization.xml
View file

@ -1,11 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<dataset>
<pages>
<uid>10</uid>
<uid>1</uid>
<pid>0</pid>
<tstamp>1613400587</tstamp>
<crdate>1613400558</crdate>
<cruser_id>1</cruser_id>
<doktype>4</doktype>
<title>Rootpage</title>
<is_siteroot>1</is_siteroot>
</pages>
<pages>
<uid>10</uid>
<pid>1</pid>
<tstamp>1613400587</tstamp>
<crdate>1613400558</crdate>
<cruser_id>1</cruser_id>
<doktype>254</doktype>
<title>Storage folder</title>
</pages>
@ -24,6 +34,9 @@
<data>
<sheet index="sDEF">
<language index="lDEF">
<field index="storagePid">
<value index="vDEF">10</value>
</field>
<field index="urls">
<el index="el">
<field index="602a89e212237114263881">

34
Tests/Functional/ImportTest.php
View file

@ -141,24 +141,22 @@ class ImportTest extends TestCase
$extbaseBootstrap = $this->getContainer()->get(Bootstrap::class);
$extbaseBootstrap->handleBackendRequest($serverRequest->reveal());
$organisations = $this->getAllRecords('tx_thuecat_organisation');
self::assertCount(1, $organisations);
self::assertSame('https://thuecat.org/resources/018132452787-ngbe', $organisations[0]['remote_id']);
self::assertSame('Erfurt Tourismus und Marketing GmbH', $organisations[0]['title']);
self::assertSame('1', $organisations[0]['uid']);
$importLogs = $this->getAllRecords('tx_thuecat_import_log');
self::assertCount(1, $importLogs);
self::assertSame('1', $importLogs[0]['configuration']);
self::assertSame('1', $importLogs[0]['uid']);
$importLogEntries = $this->getAllRecords('tx_thuecat_import_log_entry');
self::assertCount(1, $importLogEntries);
self::assertSame('1', $importLogEntries[0]['import_log']);
self::assertSame('1', $importLogEntries[0]['record_uid']);
self::assertSame('tx_thuecat_organisation', $importLogEntries[0]['table_name']);
self::assertSame('1', $importLogEntries[0]['insertion']);
self::assertSame('[]', $importLogEntries[0]['errors']);
self::assertCount(
1,
$this->getAllRecords('tx_thuecat_organisation'),
'Did not create expected number of organisations.'
);
self::assertCount(
1,
$this->getAllRecords('tx_thuecat_import_log'),
'Did not create expected number of import logs.'
);
self::assertCount(
1,
$this->getAllRecords('tx_thuecat_import_log_entry'),
'Did not create expected number of import log entries.'
);
$this->assertCSVDataSet('EXT:thuecat/Tests/Functional/Fixtures/Import/ImportsFreshOrganization.csv');
}
/**

6
Tests/Unit/Domain/Import/Converter/OrganisationTest.php
View file

@ -29,6 +29,7 @@ use WerkraumMedia\ThueCat\Domain\Import\Converter\Converter;
use WerkraumMedia\ThueCat\Domain\Import\Converter\Organisation;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
/**
* @covers WerkraumMedia\ThueCat\Domain\Import\Converter\Organisation
@ -93,8 +94,11 @@ class OrganisationTest extends TestCase
$parser->getTitle($jsonLD)->willReturn('Title');
$parser->getDescription($jsonLD)->willReturn('Description');
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new Organisation($parser->reveal());
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());

11
Tests/Unit/Domain/Import/Converter/TouristAttractionTest.php
View file

@ -29,6 +29,7 @@ use WerkraumMedia\ThueCat\Domain\Import\Converter\TouristAttraction;
use WerkraumMedia\ThueCat\Domain\Import\Importer\LanguageHandling;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
use WerkraumMedia\ThueCat\Domain\Model\Backend\Organisation;
use WerkraumMedia\ThueCat\Domain\Model\Backend\Town;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\OrganisationRepository;
@ -132,6 +133,9 @@ class TouristAttractionTest extends TestCase
$organisationRepository = $this->prophesize(OrganisationRepository::class);
$townRepository = $this->prophesize(TownRepository::class);
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new TouristAttraction(
$parser->reveal(),
$language->reveal(),
@ -139,7 +143,7 @@ class TouristAttractionTest extends TestCase
$townRepository->reveal()
);
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());
@ -218,6 +222,9 @@ class TouristAttractionTest extends TestCase
'https://example.com/resources/573211638937-gmqb',
])->willReturn($town->reveal());
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new TouristAttraction(
$parser->reveal(),
$language->reveal(),
@ -225,7 +232,7 @@ class TouristAttractionTest extends TestCase
$townRepository->reveal()
);
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());

11
Tests/Unit/Domain/Import/Converter/TouristInformationTest.php
View file

@ -29,6 +29,7 @@ use WerkraumMedia\ThueCat\Domain\Import\Converter\Converter;
use WerkraumMedia\ThueCat\Domain\Import\Converter\TouristInformation;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
use WerkraumMedia\ThueCat\Domain\Model\Backend\Organisation;
use WerkraumMedia\ThueCat\Domain\Model\Backend\Town;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\OrganisationRepository;
@ -144,12 +145,15 @@ class TouristInformationTest extends TestCase
'https://example.com/resources/573211638937-gmqb',
])->willReturn(null);
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new TouristInformation(
$parser->reveal(),
$organisationRepository->reveal(),
$townRepository->reveal()
);
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());
@ -219,12 +223,15 @@ class TouristInformationTest extends TestCase
'https://example.com/resources/573211638937-gmqb',
])->willReturn($town->reveal());
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new TouristInformation(
$parser->reveal(),
$organisationRepository->reveal(),
$townRepository->reveal()
);
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());

11
Tests/Unit/Domain/Import/Converter/TownTest.php
View file

@ -29,6 +29,7 @@ use WerkraumMedia\ThueCat\Domain\Import\Converter\Converter;
use WerkraumMedia\ThueCat\Domain\Import\Converter\Town;
use WerkraumMedia\ThueCat\Domain\Import\JsonLD\Parser;
use WerkraumMedia\ThueCat\Domain\Import\Model\EntityCollection;
use WerkraumMedia\ThueCat\Domain\Model\Backend\ImportConfiguration;
use WerkraumMedia\ThueCat\Domain\Model\Backend\Organisation;
use WerkraumMedia\ThueCat\Domain\Repository\Backend\OrganisationRepository;
@ -115,11 +116,14 @@ class TownTest extends TestCase
$organisationRepository = $this->prophesize(OrganisationRepository::class);
$organisationRepository->findOneByRemoteId('https://example.com/resources/018132452787-xxxx')->willReturn(null);
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new Town(
$parser->reveal(),
$organisationRepository->reveal()
);
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());
@ -165,11 +169,14 @@ class TownTest extends TestCase
$organisationRepository = $this->prophesize(OrganisationRepository::class);
$organisationRepository->findOneByRemoteId('https://example.com/resources/018132452787-xxxx')->willReturn($organisation->reveal());
$configuration = $this->prophesize(ImportConfiguration::class);
$configuration->getStoragePid()->willReturn(10);
$subject = new Town(
$parser->reveal(),
$organisationRepository->reveal()
);
$entities = $subject->convert($jsonLD);
$entities = $subject->convert($jsonLD, $configuration->reveal());
self::assertInstanceOf(EntityCollection::class, $entities);
self::assertCount(1, $entities->getEntities());

4
Tests/Unit/Domain/Import/ImporterTest.php
View file

@ -173,10 +173,10 @@ class ImporterTest extends TestCase
$concreteConverter->convert(Argument::that(function (array $jsonEntity) {
return $jsonEntity['@id'] === 'https://example.com/resources/34343-ex';
}))->willReturn($entities1->reveal());
}), $configuration->reveal())->willReturn($entities1->reveal());
$concreteConverter->convert(Argument::that(function (array $jsonEntity) {
return $jsonEntity['@id'] === 'https://example.com/resources/34344-es';
}))->willReturn($entities2->reveal());
}), $configuration->reveal())->willReturn($entities2->reveal());
$saveData->import($entities1->reveal(), Argument::type(ImportLog::class))->shouldBeCalled();
$saveData->import($entities2->reveal(), Argument::type(ImportLog::class))->shouldBeCalled();