From 8f65d858bbb95acd4adff7e33fe4e1a1ff1887ca Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:47:08 +0100 Subject: [PATCH 01/10] TASK: Fix case in source code Fix case for method call. Fix a typo in test method name. --- .../Connection/Elasticsearch/IndexTcaTableTest.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index e9eab37..19245bf 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -49,7 +49,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertArraySubset( ['_source' => ['header' => 'indexed content element']], @@ -72,7 +72,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.'); $this->assertArraySubset( ['_source' => ['header' => 'indexed content element']], @@ -112,7 +112,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); } @@ -135,7 +135,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( @@ -167,7 +167,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); @@ -209,7 +209,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase /** * @test */ - public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() + public function indexingDeletedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() { \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) From a232ae0b3679ece95ae940b6a3bb3ea68ce6d5b1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 21:32:42 +0100 Subject: [PATCH 02/10] BUGFIX: Respect page uid of content records Do not index content records of all pages while indexing pages in CMS 8.x. We add same logic as for CMS 7.x, we extend the query to only fetch records from current page. Resolves: #135 --- .../Domain/Index/TcaIndexer/PagesIndexer.php | 10 +++++-- .../Elasticsearch/IndexTcaTableTest.php | 19 +++++++------ .../Fixtures/Indexing/IndexTcaTable.xml | 27 +++++++++++++++++++ .../Functional/Indexing/PagesIndexerTest.php | 2 +- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 7124ca1..3f0d39b 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -79,8 +79,14 @@ class PagesIndexer extends TcaIndexer protected function fetchContentForPage(int $uid) : array { if ($this->contentTableService instanceof TcaTableService) { - $contentElements = $this->contentTableService->getQuery() - ->execute()->fetchAll(); + $queryBuilder = $this->contentTableService->getQuery(); + $queryBuilder->andWhere( + $queryBuilder->expr()->eq( + $this->contentTableService->getTableName() . '.pid', + $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT) + ) + ); + $contentElements = $queryBuilder->execute()->fetchAll(); } else { $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( $this->contentTableService->getFields(), diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 19245bf..ae671dd 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -50,11 +50,10 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); - $this->assertArraySubset( - ['_source' => ['header' => 'indexed content element']], - $response->getData()['hits']['hits'][1], - false, + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $this->assertSame( + 'indexed content element', + $response->getData()['hits']['hits'][2]['_source']['header'], 'Record was not indexed.' ); } @@ -113,7 +112,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); } /** @@ -136,7 +135,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( ['_source' => ['header' => 'Also indexable record']], @@ -168,7 +167,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOk(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 5, 'Not exactly 5 documents were indexed.'); $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( @@ -218,7 +217,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); if ($this->isLegacyVersion()) { $this->getDatabaseConnection() @@ -239,6 +238,6 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ; $response = $this->client->request('typo3content/_search?q=*:*'); - $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 document is in index.'); } } diff --git a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml index c236f07..409e46c 100644 --- a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml +++ b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml @@ -99,4 +99,31 @@ 0 0 + + + 100 + 2 + 1480686370 + 1480686370 + 0 + 72 + header +
Indexed on page 2
+ This element is on a different page + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + 2 + 1 + Second page with content + Used to check whether content is indexed only for parent page. + diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php index 244413d..bc26dfd 100644 --- a/Tests/Functional/Indexing/PagesIndexerTest.php +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -48,7 +48,7 @@ class PagesIndexerTest extends AbstractFunctionalTestCase ->with( $this->stringContains($tableName), $this->callback(function ($documents) { - return count($documents) === 1 + return count($documents) === 2 && isset($documents[0]['content']) && $documents[0]['content'] === 'this is the content of header content element that should get indexed Some text in paragraph' && isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] === From 218d8d728992daf49c8d30ccaf0360ee49fad5c1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 08:29:44 +0100 Subject: [PATCH 03/10] FEATURE: Make content fields configurable Allows integrators to configure which fields should be used to produce field "content" for indexed pages. Before only "bodytext" was used. This is now configurable and "header" was added to defaults. Resolves: #134 --- .../Domain/Index/TcaIndexer/PagesIndexer.php | 21 ++++++++++++++++++- Configuration/TypoScript/constants.txt | 3 ++- Configuration/TypoScript/setup.txt | 1 + Documentation/source/changelog.rst | 1 + ...15-134-make-conent-fields-configurable.rst | 13 ++++++++++++ .../source/configuration/indexing.rst | 17 +++++++++++++++ Tests/Functional/Fixtures/BasicSetup.ts | 3 ++- .../Functional/Indexing/PagesIndexerTest.php | 4 +++- 8 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 7124ca1..7c6f8cc 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -24,6 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Specific indexer for Pages, will basically add content of page. @@ -103,7 +104,7 @@ class PagesIndexer extends TcaIndexer $images, $this->getContentElementImages($contentElement['uid']) ); - $content[] = $contentElement['bodytext']; + $content[] = $this->getContentFromContentElement($contentElement); } return [ @@ -136,4 +137,22 @@ class PagesIndexer extends TcaIndexer return $imageRelationUids; } + + protected function getContentFromContentElement(array $contentElement) : string + { + $content = ''; + + $fieldsWithContent = GeneralUtility::trimExplode( + ',', + $this->configuration->get('indexing.' . $this->identifier . '.contentFields'), + true + ); + foreach ($fieldsWithContent as $fieldWithContent) { + if (isset($contentElement[$fieldWithContent]) && trim($contentElement[$fieldWithContent])) { + $content .= trim($contentElement[$fieldWithContent]) . ' '; + } + } + + return trim($content); + } } diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index 1a39851..b818471 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -10,12 +10,13 @@ plugin { indexing { tt_content { - additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != '' + additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND (tt_content.bodytext != '' OR tt_content.header != '') } pages { additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255) abstractFields = abstract, description, bodytext + contentFields = header, bodytext } } } diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 1a1577f..2749904 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -19,6 +19,7 @@ plugin { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} + contentFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} } } diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 4917e1b..8b1c649 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,6 +5,7 @@ Changelog :maxdepth: 1 :glob: + changelog/20180415-134-make-conent-fields-configurable changelog/20180409-25-provide-sys-language-uid changelog/20180408-131-respect-page-cache-clear changelog/20180408-introduce-php70-type-hints diff --git a/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst b/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst new file mode 100644 index 0000000..7068119 --- /dev/null +++ b/Documentation/source/changelog/20180415-134-make-conent-fields-configurable.rst @@ -0,0 +1,13 @@ +FEATURE 134 "Enable indexing of tt_content records of CType Header" +=================================================================== + +Before, only ``bodytext`` was used to generate content while indexing pages. + +As there are content elements like ``header`` where this field is empty, but content is still +available, it's now possible to configure the fields. +This makes it also possible to configure further custom content elements with new columns. + +A new TypoScript option is now available, and ``header`` is added by default, see +:ref:`contentFields`. + +See :issue:`134`. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 050f0ab..ec342fc 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -86,6 +86,23 @@ Default:: abstract, description, bodytext +.. _contentFields: + +contentFields +------------- + +Used by: :ref:`PagesIndexer`. + +Define which fields should be used to provide the auto generated field "content". + +Example:: + + plugin.tx_searchcore.settings.indexing.pages.contentFields := addToList(table_caption) + +Default:: + + header, bodytext + .. _mapping: mapping diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 4129362..840a7b8 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -14,7 +14,7 @@ plugin { additionalWhereClause ( tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - AND tt_content.bodytext != '' + AND (tt_content.bodytext != '' OR tt_content.header != '') ) mapping { @@ -27,6 +27,7 @@ plugin { pages { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer abstractFields = abstract, description, bodytext + contentFields = header, bodytext mapping { CType { diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php index 244413d..61df297 100644 --- a/Tests/Functional/Indexing/PagesIndexerTest.php +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -50,7 +50,9 @@ class PagesIndexerTest extends AbstractFunctionalTestCase $this->callback(function ($documents) { return count($documents) === 1 && isset($documents[0]['content']) && $documents[0]['content'] === - 'this is the content of header content element that should get indexed Some text in paragraph' + 'indexed content element' . + ' this is the content of header content element that should get indexed' . + ' Indexed without html tags Some text in paragraph' && isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] === 'Used as abstract as no abstract is defined.' ; From 8ed477501c7c612ad44b5cc1f0d0ed739304e579 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 16 Mar 2018 17:28:08 +0100 Subject: [PATCH 04/10] BUGFIX: Use contentFields constant in setup.txt --- Configuration/TypoScript/setup.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 2749904..751e47c 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -19,7 +19,7 @@ plugin { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} - contentFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} + contentFields = {$plugin.tx_searchcore.settings.indexing.pages.contentFields} } } From 30aa8ed10bebc4793cac722f5f9b8fd7241b56af Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 11:02:43 +0100 Subject: [PATCH 05/10] TASK: Remove codacy from travis As we are no longer using codacy, we do not need to send any information. --- Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b1f4af0..21f6c31 100644 --- a/Makefile +++ b/Makefile @@ -41,16 +41,11 @@ unitTests: .Build/bin/phpunit --colors --debug -v \ -c Tests/Unit/UnitTests.xml -uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy +uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToScrutinizer: wget https://scrutinizer-ci.com/ocular.phar && \ php ocular.phar code-coverage:upload --format=php-clover .Build/report/functional/clover/coverage -uploadCodeCoverageToCodacy: - composer require -vv --dev codacy/coverage && \ - git checkout composer.json && \ - php .Build/bin/codacycoverage clover .Build/report/functional/clover/coverage - clean: rm -rf .Build composer.lock From 203b5d7adf4112273f55de57e9b877789811f69e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 16 Mar 2018 17:37:19 +0100 Subject: [PATCH 06/10] TASK: Prevent issue When hook is called with non uid If some issues occur outside of our extension, we might not get a valid uid inside of our hooks. We will therefore add additional checks and prevent any further execution. Resolves: #112 --- Classes/Hook/DataHandler.php | 6 ++++- Tests/Unit/Hook/DataHandlerTest.php | 42 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index e0b8e40..6c3da5c 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -72,7 +72,7 @@ class DataHandler implements Singleton /** * Called by CoreDataHandler on deletion of records. */ - public function processCmdmap_deleteAction(string $table, int $uid) : bool + public function processCmdmap_deleteAction(string $table, string $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { $this->logger->debug('Delete not processed.', [$table, $uid]); @@ -95,6 +95,10 @@ class DataHandler implements Singleton $uid = $dataHandler->substNEWwithIDs[$uid]; } + if (!is_numeric($uid) || $uid <= 0) { + continue; + } + $this->processRecord($table, $uid); } } diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php index ed68993..60da2e6 100644 --- a/Tests/Unit/Hook/DataHandlerTest.php +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -91,4 +91,46 @@ class DataHandlerToProcessorTest extends AbstractUnitTestCase ], ]; } + + /** + * @test + */ + public function indexingIsNotCalledForCacheClearIfDataIsInvalid() + { + $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); + $ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + + $subject = new DataHandler($ownDataHandlerMock); + + $ownDataHandlerMock->expects($this->never())->method('update'); + + $subject->clearCachePostProc([ + 'cacheCmd' => 'NEW343', + ], $coreDataHandlerMock); + } + /** + * @test + */ + public function indexingIsNotCalledForProcesIfDataIsInvalid() + { + $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); + $coreDataHandlerMock->datamap = [ + 'tt_content' => [ + 'NEW343' => [], + ], + ]; + $coreDataHandlerMock->substNEWwithIDs = []; + + $ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + + $subject = new DataHandler($ownDataHandlerMock); + + $ownDataHandlerMock->expects($this->never())->method('update'); + + $subject->processDatamap_afterAllOperations($coreDataHandlerMock); + } } From 5c493aad2117dbf9cf90b2d1c1c0cae9225d17b3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 14:18:47 +0100 Subject: [PATCH 07/10] TASK: Fix typo in test method name --- Tests/Unit/Hook/DataHandlerTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php index 60da2e6..aaf3935 100644 --- a/Tests/Unit/Hook/DataHandlerTest.php +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -113,7 +113,7 @@ class DataHandlerToProcessorTest extends AbstractUnitTestCase /** * @test */ - public function indexingIsNotCalledForProcesIfDataIsInvalid() + public function indexingIsNotCalledForProcessIfDataIsInvalid() { $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); $coreDataHandlerMock->datamap = [ From 372cd484710173e50bd99f363d636cf97c4932a3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 15:42:34 +0100 Subject: [PATCH 08/10] FEATURE: Allow to copy a single field with CopyToProcessor --- Classes/DataProcessing/CopyToProcessor.php | 18 ++++++++++++++---- .../dataProcessing/CopyToProcessor.rst | 7 ++++++- .../DataProcessing/CopyToProcessorTest.php | 12 ++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index 28c4294..159701f 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -27,11 +27,21 @@ class CopyToProcessor implements ProcessorInterface { public function processData(array $record, array $configuration) : array { - $all = []; + $target = []; - $this->addArray($all, $record); - $all = array_filter($all); - $record[$configuration['to']] = implode(PHP_EOL, $all); + $from = $record; + if (isset($configuration['from'])) { + $from = $record[$configuration['from']]; + } + + if (is_array($from)) { + $this->addArray($target, $from); + } else { + $target[] = (string) $from; + } + + $target = array_filter($target); + $record[$configuration['to']] = implode(PHP_EOL, $target); return $record; } diff --git a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst index 6d83d70..07faef1 100644 --- a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst @@ -8,6 +8,10 @@ Possible Options: ``to`` Defines the field to copy the values into. All values not false will be copied at the moment. +``from`` + Optional, defines the field to copy, can only be one field. + If empty, all existing fields will be copied. + Example:: plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { @@ -17,7 +21,8 @@ Example:: } 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor 2 { - to = spellcheck + from = uid + to = backup_uid } } diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php index 28545a3..6968d25 100644 --- a/Tests/Unit/DataProcessing/CopyToProcessorTest.php +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -80,6 +80,18 @@ class CopyToProcessorTest extends AbstractUnitTestCase 'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2', ], ], + 'Copy single field to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedData' => [ + 'field 1' => 'Some content like lorem', + 'new_field' => 'Some content like lorem', + ], + ], ]; } } From c8dc95c4d37a831398b03f8d1f567fd6a2e583c4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Mar 2018 16:07:28 +0100 Subject: [PATCH 09/10] FEATURE: Provide type of result item In order to make further usage easier, we provide the type for each result item. This makes it possible to call {f:render(section: resultItem.type)}. --- .../Connection/Elasticsearch/SearchResult.php | 2 +- Classes/Connection/ResultItemInterface.php | 8 +++ Classes/Domain/Model/ResultItem.php | 13 +++- Classes/Domain/Model/SearchResult.php | 2 +- Classes/Domain/Search/SearchService.php | 11 ++-- Tests/Unit/Domain/Model/ResultItemTest.php | 59 +++++++++++++++++-- Tests/Unit/Domain/Model/SearchResultTest.php | 30 +++++++--- .../Unit/Domain/Search/SearchServiceTest.php | 17 +++++- 8 files changed, 119 insertions(+), 23 deletions(-) diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 5b3d381..0f81463 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -108,7 +108,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->result->getResults() as $result) { - $this->results[] = new ResultItem($result->getData()); + $this->results[] = new ResultItem($result->getData(), $result->getParam('_type')); } } diff --git a/Classes/Connection/ResultItemInterface.php b/Classes/Connection/ResultItemInterface.php index d7fe1a9..f5456c9 100644 --- a/Classes/Connection/ResultItemInterface.php +++ b/Classes/Connection/ResultItemInterface.php @@ -33,4 +33,12 @@ interface ResultItemInterface extends \ArrayAccess * Used e.g. for dataprocessing. */ public function getPlainData() : array; + + /** + * Returns the type of the item. + * + * That should make it easier to differentiate if multiple + * types are returned for one query. + */ + public function getType() : string; } diff --git a/Classes/Domain/Model/ResultItem.php b/Classes/Domain/Model/ResultItem.php index d0e370a..9ff45d2 100644 --- a/Classes/Domain/Model/ResultItem.php +++ b/Classes/Domain/Model/ResultItem.php @@ -29,9 +29,20 @@ class ResultItem implements ResultItemInterface */ protected $data = []; - public function __construct(array $result) + /** + * @var string + */ + protected $type = ''; + + public function __construct(array $result, string $type) { $this->data = $result; + $this->type = $type; + } + + public function getType() : string + { + return $this->type; } public function getPlainData() : array diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php index 163b996..516c333 100644 --- a/Classes/Domain/Model/SearchResult.php +++ b/Classes/Domain/Model/SearchResult.php @@ -76,7 +76,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->resultItems as $item) { - $this->results[] = new ResultItem($item); + $this->results[] = new ResultItem($item['data'], $item['type']); } } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 3a83b7b..1fc792e 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -146,10 +146,13 @@ class SearchService $newSearchResultItems = []; foreach ($this->configuration->get('searching.dataProcessing') as $configuration) { foreach ($searchResult as $resultItem) { - $newSearchResultItems[] = $this->dataProcessorService->executeDataProcessor( - $configuration, - $resultItem->getPlainData() - ); + $newSearchResultItems[] = [ + 'data' => $this->dataProcessorService->executeDataProcessor( + $configuration, + $resultItem->getPlainData() + ), + 'type' => $resultItem->getType(), + ]; } } diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php index 53477c7..a4c18fd 100644 --- a/Tests/Unit/Domain/Model/ResultItemTest.php +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -36,7 +36,7 @@ class ResultItemTest extends AbstractUnitTestCase ]; $expectedData = $originalData; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->assertSame( $expectedData, $subject->getPlainData(), @@ -55,7 +55,7 @@ class ResultItemTest extends AbstractUnitTestCase ]; $expectedData = $originalData; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->assertSame( $originalData['title'], $subject['title'], @@ -73,7 +73,7 @@ class ResultItemTest extends AbstractUnitTestCase 'title' => 'Some title', ]; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->assertTrue(isset($subject['title']), 'Could not determine that title exists.'); $this->assertFalse(isset($subject['title2']), 'Could not determine that title2 does not exists.'); } @@ -88,7 +88,7 @@ class ResultItemTest extends AbstractUnitTestCase 'title' => 'Some title', ]; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->expectException(\BadMethodCallException::class); $subject['title'] = 'New Title'; } @@ -103,8 +103,57 @@ class ResultItemTest extends AbstractUnitTestCase 'title' => 'Some title', ]; - $subject = new ResultItem($originalData); + $subject = new ResultItem($originalData, 'testType'); $this->expectException(\BadMethodCallException::class); unset($subject['title']); } + + /** + * @test + */ + public function typeCanBeRetrievedAfterConstruction() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData, 'testType'); + $this->assertSame( + 'testType', + $subject->getType(), + 'Could not retrieve type.' + ); + } + + /** + * @test + */ + public function typeCanNotBeChanged() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + + $subject = new ResultItem($originalData, 'testType'); + $this->expectException(\BadMethodCallException::class); + $subject['type'] = 'New Title'; + } + + /** + * @test + */ + public function typeCanNotBeRemoved() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + + $subject = new ResultItem($originalData, 'testType'); + $this->expectException(\BadMethodCallException::class); + unset($subject['type']); + } } diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php index eebb3c1..db5ccb5 100644 --- a/Tests/Unit/Domain/Model/SearchResultTest.php +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -71,16 +71,25 @@ class SearchResultTest extends AbstractUnitTestCase $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); $data = [ [ - 'uid' => 10, - 'title' => 'Some Title', + 'data' => [ + 'uid' => 10, + 'title' => 'Some Title', + ], + 'type' => 'testType1', ], [ - 'uid' => 11, - 'title' => 'Some Title 2', + 'data' => [ + 'uid' => 11, + 'title' => 'Some Title 2', + ], + 'type' => 'testType2', ], [ - 'uid' => 12, - 'title' => 'Some Title 3', + 'data' => [ + 'uid' => 12, + 'title' => 'Some Title 3', + ], + 'type' => 'testType2', ], ]; @@ -89,9 +98,12 @@ class SearchResultTest extends AbstractUnitTestCase $this->assertCount(3, $resultItems); - $this->assertSame($data[0]['uid'], $resultItems[0]['uid']); - $this->assertSame($data[1]['uid'], $resultItems[1]['uid']); - $this->assertSame($data[2]['uid'], $resultItems[2]['uid']); + $this->assertSame($data[0]['data']['uid'], $resultItems[0]['uid']); + $this->assertSame($data[0]['type'], $resultItems[0]->getType()); + $this->assertSame($data[1]['data']['uid'], $resultItems[1]['uid']); + $this->assertSame($data[1]['type'], $resultItems[1]->getType()); + $this->assertSame($data[2]['data']['uid'], $resultItems[2]['uid']); + $this->assertSame($data[2]['type'], $resultItems[2]->getType()); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]); diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 3d6791e..50c1f49 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -294,7 +294,14 @@ class SearchServiceTest extends AbstractUnitTestCase )); $searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); - $searchResult = new SearchResult($searchResultMock, [['field 1' => 'value 1']]); + $searchResult = new SearchResult($searchResultMock, [ + [ + 'data' => [ + 'field 1' => 'value 1' + ], + 'type' => 'testType', + ], + ]); $this->connection->expects($this->once()) ->method('search') @@ -311,7 +318,13 @@ class SearchServiceTest extends AbstractUnitTestCase $this->objectManager->expects($this->once()) ->method('get') ->with(SearchResult::class, $searchResult, [ - ['field 1' => 'value 1', 'field 2' => 'value 2'] + [ + 'data' => [ + 'field 1' => 'value 1', + 'field 2' => 'value 2', + ], + 'type' => 'testType', + ] ]) ->willReturn($searchResultMock); From f9da96e7df96132182476cbf87f0e96b00692bf9 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Mar 2018 13:43:25 +0100 Subject: [PATCH 10/10] TASK: Raise release version --- Documentation/source/conf.py | 6 +++--- Documentation/source/installation.rst | 2 +- ext_emconf.php | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index aa846e0..92dc768 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -59,9 +59,9 @@ author = u'Daniel Siepmann' # built documents. # # The short X.Y version. -version = u'0.0.1' +version = u'0.0.3' # The full version, including alpha/beta/rc tags. -release = u'0.0.1' +release = u'0.0.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -133,7 +133,7 @@ html_theme_options = { # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'TYPO3 Extension search_core v0.0.1' +#html_title = u'TYPO3 Extension search_core v0.0.3' # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index 0a0e5b6..174d9f7 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -9,7 +9,7 @@ Composer The extension can be installed through composer:: - composer require "codappix/search_core" "~1.0.0" + composer require "codappix/search_core" "~0.0.3" Note that you have to allow unstable packages: diff --git a/ext_emconf.php b/ext_emconf.php index 7c3a087..e2b4d1b 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -18,7 +18,7 @@ $EM_CONF[$_EXTKEY] = [ ], ], 'state' => 'beta', - 'version' => '0.0.1', + 'version' => '0.0.3', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', ];