From 5dd0759bb603c590b0a6031c1a12e3cb4a1ea583 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Jun 2017 12:11:28 +0200 Subject: [PATCH 001/158] TASK: Fix file permissions --- Configuration/TypoScript/constants.txt | 0 Configuration/TypoScript/setup.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 Configuration/TypoScript/constants.txt mode change 100755 => 100644 Configuration/TypoScript/setup.txt diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt old mode 100755 new mode 100644 diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt old mode 100755 new mode 100644 From 3a2523e1d2e8e61350150a29c0f4ce5e098d2ee4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Jun 2017 12:32:27 +0200 Subject: [PATCH 002/158] WIP|FEATURE: First basic implementation of filter * Working version without further architecture. * Manually tested. * Still need to move to new architecture and cover with tests. --- Classes/Connection/Elasticsearch.php | 27 ++++++++++++++++++++-- Classes/Domain/Model/SearchRequest.php | 31 +++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 560ca07..7f963ea 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -146,12 +146,35 @@ class Elasticsearch implements Singleton, ConnectionInterface { $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); + $query = [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => $searchRequest->getSearchTerm() + ], + ], + ], + ], + ]; + + if ($searchRequest->hasFilter()) { + $queryFilter = []; + foreach ($searchRequest->getFilter() as $field => $value) { + $queryFilter[$field] = $value; + } + + $query['bool']['filter'] = [ + 'term' => $queryFilter, + ]; + } + $search = new \Elastica\Search($this->connection->getClient()); $search->addIndex('typo3content'); - + $search->setQuery(new \Elastica\Query(['query' => $query])); // TODO: Return wrapped result to implement our interface. // Also update php doc to reflect the change. - return $search->search('"' . $searchRequest->getSearchTerm() . '"'); + return $search->search(); } /** diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 1de2f71..105cc45 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -32,7 +32,12 @@ class SearchRequest implements SearchRequestInterface * * @var string */ - protected $query; + protected $query = ''; + + /** + * @var array + */ + protected $filter = []; /** * @param string $query @@ -57,4 +62,28 @@ class SearchRequest implements SearchRequestInterface { return $this->query; } + + /** + * @param array $filter + */ + public function setFilter(array $filter) + { + $this->filter = $filter; + } + + /** + * @return bool + */ + public function hasFilter() + { + return count($this->filter); + } + + /** + * @return array + */ + public function getFilter() + { + return $this->filter; + } } From 1a41c5e237729026696034d04a67bcfca7384a71 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Jun 2017 13:53:08 +0200 Subject: [PATCH 003/158] FEATURE: Add filter to search * Allow to filter results by field values. --- .travis.yml | 4 +- Classes/Connection/Elasticsearch.php | 37 ++++------ Classes/Connection/SearchRequestInterface.php | 10 +++ Classes/Domain/Search/QueryFactory.php | 72 +++++++++++++++++++ Makefile | 5 ++ .../AbstractFunctionalTestCase.php | 7 ++ .../Connection/Elasticsearch/FilterTest.php | 61 ++++++++++++++++ .../Functional/Fixtures/Searching/Filter.xml | 42 +++++++++++ Tests/Unit/AbstractUnitTestCase.php | 27 +++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 56 +++++++++++++++ Tests/Unit/UnitTests.xml | 28 ++++++++ 11 files changed, 323 insertions(+), 26 deletions(-) create mode 100644 Classes/Domain/Search/QueryFactory.php create mode 100644 Tests/Functional/Connection/Elasticsearch/FilterTest.php create mode 100644 Tests/Functional/Fixtures/Searching/Filter.xml create mode 100644 Tests/Unit/AbstractUnitTestCase.php create mode 100644 Tests/Unit/Domain/Search/QueryFactoryTest.php create mode 100644 Tests/Unit/UnitTests.xml diff --git a/.travis.yml b/.travis.yml index e583e51..123363a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,9 @@ services: install: make install -script: make functionalTests +script: + - make unitTests + - make functionalTests after_script: - make uploadCodeCoverage diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 7f963ea..386e19c 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Connection; */ use TYPO3\CMS\Core\SingletonInterface as Singleton; +use Leonmrni\SearchCore\Domain\Search\QueryFactory; /** * Outer wrapper to elasticsearch. @@ -47,6 +48,11 @@ class Elasticsearch implements Singleton, ConnectionInterface */ protected $documentFactory; + /** + * @var QueryFactory + */ + protected $queryFactory; + /** * @var \TYPO3\CMS\Core\Log\Logger */ @@ -67,17 +73,20 @@ class Elasticsearch implements Singleton, ConnectionInterface * @param Elasticsearch\IndexFactory $indexFactory * @param Elasticsearch\TypeFactory $typeFactory * @param Elasticsearch\DocumentFactory $documentFactory + * @param QueryFactory $queryFactory */ public function __construct( Elasticsearch\Connection $connection, Elasticsearch\IndexFactory $indexFactory, Elasticsearch\TypeFactory $typeFactory, - Elasticsearch\DocumentFactory $documentFactory + Elasticsearch\DocumentFactory $documentFactory, + QueryFactory $queryFactory ) { $this->connection = $connection; $this->indexFactory = $indexFactory; $this->typeFactory = $typeFactory; $this->documentFactory = $documentFactory; + $this->queryFactory = $queryFactory; } public function addDocument($documentType, array $document) @@ -146,32 +155,10 @@ class Elasticsearch implements Singleton, ConnectionInterface { $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); - $query = [ - 'bool' => [ - 'must' => [ - [ - 'match' => [ - '_all' => $searchRequest->getSearchTerm() - ], - ], - ], - ], - ]; - - if ($searchRequest->hasFilter()) { - $queryFilter = []; - foreach ($searchRequest->getFilter() as $field => $value) { - $queryFilter[$field] = $value; - } - - $query['bool']['filter'] = [ - 'term' => $queryFilter, - ]; - } - $search = new \Elastica\Search($this->connection->getClient()); $search->addIndex('typo3content'); - $search->setQuery(new \Elastica\Query(['query' => $query])); + $search->setQuery($this->queryFactory->create($this, $searchRequest)); + // TODO: Return wrapped result to implement our interface. // Also update php doc to reflect the change. return $search->search(); diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index ecf7a74..603c02f 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -31,4 +31,14 @@ interface SearchRequestInterface * @return string */ public function getSearchTerm(); + + /** + * @return bool + */ + public function hasFilter(); + + /** + * @return array + */ + public function getFilter(); } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php new file mode 100644 index 0000000..2f3ac25 --- /dev/null +++ b/Classes/Domain/Search/QueryFactory.php @@ -0,0 +1,72 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Connection\ConnectionInterface; +use Leonmrni\SearchCore\Connection\Elasticsearch\Query; +use Leonmrni\SearchCore\Connection\SearchRequestInterface; + +class QueryFactory +{ + /** + * @param ConnectionInterface $connection + * @param SearchRequestInterface $searchRequest + * + * @return \Elastica\Query + */ + public function create( + ConnectionInterface $connection, + SearchRequestInterface $searchRequest + ) { + return $this->createElasticaQuery($searchRequest); + } + + /** + * @param SearchRequestInterface $searchRequest + * @return \Elastica\Query + */ + protected function createElasticaQuery(SearchRequestInterface $searchRequest) + { + $query = [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => $searchRequest->getSearchTerm() + ], + ], + ], + ], + ]; + $queryFilter = []; + + if ($searchRequest->hasFilter()) { + foreach ($searchRequest->getFilter() as $field => $value) { + $queryFilter[$field] = $value; + } + $query['bool']['filter'] = [ + 'term' => $queryFilter, + ]; + } + + return new \Elastica\Query(['query' => $query]); + } +} diff --git a/Makefile b/Makefile index 2520ab0..6d058c0 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,11 @@ functionalTests: .Build/bin/phpunit --colors --debug -v \ -c Tests/Functional/FunctionalTests.xml +unitTests: + TYPO3_PATH_WEB=$(TYPO3_WEB_DIR) \ + .Build/bin/phpunit --colors --debug -v \ + -c Tests/Unit/UnitTests.xml + uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy uploadCodeCoverageToScrutinizer: diff --git a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php index cdd8ddf..363ad6d 100644 --- a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php +++ b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php @@ -43,11 +43,18 @@ abstract class AbstractFunctionalTestCase extends BaseFunctionalTestCase 'host' => getenv('ES_HOST') ?: \Elastica\Connection::DEFAULT_HOST, 'port' => getenv('ES_PORT') ?: \Elastica\Connection::DEFAULT_PORT, ]); + + $this->cleanUp(); } public function tearDown() { // Delete everything so next test starts clean. + $this->cleanUp(); + } + + protected function cleanUp() + { $this->client->getIndex('_all')->delete(); $this->client->getIndex('_all')->clearCache(); } diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php new file mode 100644 index 0000000..88ae37f --- /dev/null +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -0,0 +1,61 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Leonmrni\SearchCore\Domain\Model\SearchRequest; +use Leonmrni\SearchCore\Domain\Search\SearchService; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class FilterTest extends AbstractFunctionalTestCase +{ + protected function getDataSets() + { + return array_merge( + parent::getDataSets(), + ['Tests/Functional/Fixtures/Searching/Filter.xml'] + ); + } + + /** + * @test + */ + public function itsPossibleToFilterResultsByASingleField() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(SearchService::class); + $searchRequest = new SearchRequest('Search Word'); + + $result = $searchService->search($searchRequest); + $this->assertSame(2, count($result), 'Did not receive both indexed elements without filter.'); + + $searchRequest->setFilter(['CType' => 'html']); + $result = $searchService->search($searchRequest); + $this->assertSame('5', $result[0]->getData()['uid'], 'Did not get the expected result entry.'); + $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); + } +} diff --git a/Tests/Functional/Fixtures/Searching/Filter.xml b/Tests/Functional/Fixtures/Searching/Filter.xml new file mode 100644 index 0000000..103a4da --- /dev/null +++ b/Tests/Functional/Fixtures/Searching/Filter.xml @@ -0,0 +1,42 @@ + + + + 5 + 1 + 1480686370 + 1480686370 + 0 + 72 + html +
indexed content element with html ctype
+ Search Word + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + 6 + 1 + 1480686370 + 1480686370 + 0 + 72 + header +
indexed content element with header ctype
+ Search Word + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php new file mode 100644 index 0000000..f3281c1 --- /dev/null +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -0,0 +1,27 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; + +abstract class AbstractUnitTestCase extends CoreTestCase +{ +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php new file mode 100644 index 0000000..7fc6b15 --- /dev/null +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -0,0 +1,56 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Connection; +use Leonmrni\SearchCore\Domain\Model\SearchRequest; +use Leonmrni\SearchCore\Domain\Search\QueryFactory; +use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class QueryFactoryTest extends AbstractUnitTestCase +{ + protected $subject; + + public function setUp() + { + parent::setUp(); + + $this->subject = new QueryFactory; + } + + /** + * @test + */ + public function creatonOfQueryWorksInGeneral() + { + $connection = $this->getMockBuilder(Connection\Elasticsearch::class) + ->disableOriginalConstructor() + ->getMock(); + $searchRequest = new SearchRequest('SearchWord'); + + $query = $this->subject->create($connection, $searchRequest); + $this->assertInstanceOf( + \Elastica\Query::class, + $query, + 'Factory did not create the expected instance.' + ); + } +} diff --git a/Tests/Unit/UnitTests.xml b/Tests/Unit/UnitTests.xml new file mode 100644 index 0000000..6456405 --- /dev/null +++ b/Tests/Unit/UnitTests.xml @@ -0,0 +1,28 @@ + + + + + . + + + + + + ../../Classes + + + From bb9e29574f84e16a81e2888a78fb60e41f63b8a4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Jun 2017 14:25:03 +0200 Subject: [PATCH 004/158] TASK: Use elasticsearch 5.2 on travis * As we make use of newer features that are incompatible with travis own elasticsearch version 1.x. --- .travis.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 123363a..03a193c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,12 @@ +sudo: true + +addons: + apt: + packages: + - oracle-java8-set-default +before_install: + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start + language: php php: @@ -49,7 +58,6 @@ matrix: services: - mysql - - elasticsearch install: make install From f5729c276325e8d8696eb8a51bd0cc80b1f60bc5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Jun 2017 15:33:06 +0200 Subject: [PATCH 005/158] BUGFIX: Keep return type * Return boolean type. --- Classes/Domain/Model/SearchRequest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 105cc45..b88c866 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -76,7 +76,7 @@ class SearchRequest implements SearchRequestInterface */ public function hasFilter() { - return count($this->filter); + return count($this->filter) > 0; } /** From f4a9531fe59df1e40b6090342121e3b6664fee59 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Jun 2017 16:25:35 +0200 Subject: [PATCH 006/158] TASK: Remove unnecessary code * As filter is already in the format we need, we can just use it instead of using a foreach. --- Classes/Domain/Search/QueryFactory.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 2f3ac25..31c50df 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -56,15 +56,9 @@ class QueryFactory ], ], ]; - $queryFilter = []; if ($searchRequest->hasFilter()) { - foreach ($searchRequest->getFilter() as $field => $value) { - $queryFilter[$field] = $value; - } - $query['bool']['filter'] = [ - 'term' => $queryFilter, - ]; + $query['bool']['filter'] = ['term' => $searchRequest->getFilter()]; } return new \Elastica\Query(['query' => $query]); From f453592b39158e41a2f46121253e0755de1b618c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Jun 2017 08:38:14 +0200 Subject: [PATCH 007/158] TASK: Add further tests and cast search input * Map user input to string in any case. * Add tests to check whether filter is added to query. * Add test to check whether input is casted to string. --- Classes/Domain/Model/SearchRequest.php | 4 +- Tests/Unit/Domain/Search/QueryFactoryTest.php | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index b88c866..f1ad18e 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -44,7 +44,7 @@ class SearchRequest implements SearchRequestInterface */ public function __construct($query) { - $this->query = $query; + $this->query = (string) $query; } /** @@ -68,7 +68,7 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = $filter; + $this->filter = array_map('strval', $filter); } /** diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 7fc6b15..83f031e 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -53,4 +53,47 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Factory did not create the expected instance.' ); } + + /** + * @test + */ + public function filterIsAddedToQuery() + { + $connection = $this->getMockBuilder(Connection\Elasticsearch::class) + ->disableOriginalConstructor() + ->getMock(); + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter(['field' => 'content']); + + $query = $this->subject->create($connection, $searchRequest); + $this->assertSame( + ['field' => 'content'], + $query->toArray()['query']['bool']['filter']['term'], + 'Filter was not added to query.' + ); + } + + /** + * @test + */ + public function userInputIsAlwaysString() + { + $connection = $this->getMockBuilder(Connection\Elasticsearch::class) + ->disableOriginalConstructor() + ->getMock(); + $searchRequest = new SearchRequest(10); + $searchRequest->setFilter(['field' => 20]); + + $query = $this->subject->create($connection, $searchRequest); + $this->assertSame( + '10', + $query->toArray()['query']['bool']['must'][0]['match']['_all'], + 'Search word was not escaped as expected.' + ); + $this->assertSame( + '20', + $query->toArray()['query']['bool']['filter']['term']['field'], + 'Search word was not escaped as expected.' + ); + } } From 3553c443e2e06da2cdc44aed4477c7470579a23f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 27 Jun 2017 16:51:33 +0200 Subject: [PATCH 008/158] FEATURE: Adjust configuration structure * To support further configuration, specific to identifiers / tables. * E.g. mapping and boost configuration should be possible. * Adjust docs and settings. * Adjust tests and code. --- Classes/Command/IndexCommandController.php | 3 ++ .../Index/TcaIndexer/TcaTableService.php | 7 +++-- Classes/Domain/Service/DataHandler.php | 13 ++++++++- Configuration/TypoScript/constants.txt | 24 +++++++-------- Configuration/TypoScript/setup.txt | 11 +++---- Documentation/source/concepts.rst | 2 +- Documentation/source/configuration.rst | 29 +++++-------------- Tests/Functional/Fixtures/BasicSetup.ts | 6 ++-- .../TcaIndexer/RespectRootLineBlacklist.ts | 4 +-- .../Fixtures/Indexing/UserWhereClause.ts | 8 ++--- 10 files changed, 49 insertions(+), 58 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index c1ade27..55ac4df 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -57,6 +57,9 @@ class IndexCommandController extends CommandController { // TODO: Allow to index multiple tables at once? // TODO: Also allow to index everything? + + // TODO: There is no test yet! + // TODO: Add test, adjust config path if (! in_array($table, GeneralUtility::trimExplode(',', $this->configuration->get('indexer.tca.allowedTables')))) { $this->outputLine('Table is not allowed for indexing.'); $this->quit(1); diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 936d425..7d69e5b 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -151,7 +151,8 @@ class TcaTableService . ' AND pages.no_search = 0' ; - $userDefinedWhere = $this->configuration->getIfExists('indexer.tca.' . $this->tableName . '.additionalWhereClause'); + // TODO: There is no test covering this option yet? + $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); if (is_string($userDefinedWhere)) { $whereClause .= ' AND ' . $userDefinedWhere; } @@ -269,7 +270,7 @@ class TcaTableService */ protected function isBlackListedRootLineConfigured() { - return (bool) $this->configuration->getIfExists('indexer.tca.rootLineBlacklist'); + return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); } /** @@ -279,6 +280,6 @@ class TcaTableService */ protected function getBlackListedRootLine() { - return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexer.tca.rootLineBlacklist')); + return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); } } diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 213b9cc..578589a 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Domain\Service; */ use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -86,7 +87,17 @@ class DataHandler implements Singleton */ public function getAllowedTablesForIndexing() { - return GeneralUtility::trimExplode(',', $this->configuration->get('indexer.tca.allowedTables')); + $tables = []; + foreach ($this->configuration->get('indexing') as $tableName => $indexConfiguration) { + if ($indexConfiguration['indexer'] === TcaIndexer::class) { + $tables[] = $tableName; + continue; + } + // TODO: Support custom indexer. + // Define "interface" / option which is used. + } + + return $tables; } /** diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index 50f57c3..f97c039 100755 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -8,20 +8,16 @@ plugin { } } - indexer { - tca { - # Pages are not supported yet, see - # https://github.com/DanielSiepmann/search_core/issues/24 but - # should also be added, together with additionalWhereClause - # based on doktypes - allowedTables = tt_content - - tt_content { - additionalWhereClause ( - pages.doktype NOT IN (3, 199) - AND tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - ) - } + indexing { + # Pages are not supported yet, see + # https://github.com/DanielSiepmann/search_core/issues/24 but + # should also be added, together with additionalWhereClause + # based on doktypes + tt_content { + additionalWhereClause ( + pages.doktype NOT IN (3, 199) + AND tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') + ) } } } diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index afa15be..2d58b35 100755 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -8,13 +8,10 @@ plugin { } } - indexer { - tca { - allowedTables = {$plugin.tx_searchcore.settings.indexer.tca.allowedTables} - - tt_content { - additionalWhereClause = {$plugin.tx_searchcore.settings.indexer.tca.tt_content.additionalWhereClause} - } + indexing { + tt_content { + indexer = Leonmrni\SearchCore\Domain\Index\TcaIndexer + additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause} } } } diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index 4d48d35..fc0569a 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -25,6 +25,6 @@ Indexing -------- The indexing is done by one of the available indexer. It should be possible to define the indexer to -use for certein document types. Also it should be possible to write custom indexer to use. +use for certain document types. Also it should be possible to write custom indexer to use. Currently only the :ref:`TcaIndexer` is provided. diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index fb51d11..1f16e06 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -38,7 +38,7 @@ Options The following section contains the different options, e.g. for :ref:`connections` and :ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or -``plugin.tx_searchcore.settings.index``. +``plugin.tx_searchcore.settings.indexing``. .. _configuration_options_connection: @@ -106,8 +106,9 @@ Configured as:: plugin { tx_searchcore { settings { - indexer { - indexerName { + indexing { + identifier { + indexer = Fully Qualified Classname // the settings } } @@ -115,26 +116,10 @@ Configured as:: } } -Where ``indexerName`` is one of the available :ref:`indexer`. +Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. The following settings are available. For each setting its documented which indexer consumes it. -.. _allowedTables: - -``allowedTables`` -""""""""""""""""" - - Used by: :ref:`TcaIndexer`. - - Defines which TYPO3 tables are allowed to be indexed. Only white listed tables will be processed - through Command Line Interface and Hooks. - - Contains a comma separated list of table names. Spaces are trimmed. - - Example:: - - plugin.tx_searchcore.settings.indexer.tca.allowedTables = tt_content, fe_users - .. _rootLineBlacklist: ``rootLineBlacklist`` @@ -151,7 +136,7 @@ The following settings are available. For each setting its documented which inde Example:: - plugin.tx_searchcore.settings.index.tca.rootLineBlacklist = 3, 10, 100 + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 Also it's possible to define some behaviour for the different document types. In context of TYPO3 tables are used as document types 1:1. It's possible to configure different tables. The following @@ -170,7 +155,7 @@ options are available: Example:: - plugin.tx_searchcore.settings.index.tca.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') .. attention:: diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 510e2a1..72163e3 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -8,9 +8,9 @@ plugin { } } - indexer { - tca { - allowedTables = tt_content + indexing { + tt_content { + indexer = Leonmrni\SearchCore\Domain\Index\TcaIndexer } } } diff --git a/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts b/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts index 44013e7..c0c1cd6 100644 --- a/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts +++ b/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts @@ -1,8 +1,8 @@ plugin { tx_searchcore { settings { - indexer { - tca { + indexing { + tt_content { rootLineBlacklist = 3 } } diff --git a/Tests/Functional/Fixtures/Indexing/UserWhereClause.ts b/Tests/Functional/Fixtures/Indexing/UserWhereClause.ts index d79bb14..7478ef1 100644 --- a/Tests/Functional/Fixtures/Indexing/UserWhereClause.ts +++ b/Tests/Functional/Fixtures/Indexing/UserWhereClause.ts @@ -1,11 +1,9 @@ plugin { tx_searchcore { settings { - indexer { - tca { - tt_content { - additionalWhereClause = tt_content.CType NOT IN ('div') - } + indexing { + tt_content { + additionalWhereClause = tt_content.CType NOT IN ('div') } } } From dfed17bb6c51c1c49164386e896663c75412ca0b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 29 Jun 2017 08:42:38 +0200 Subject: [PATCH 009/158] TASK: Update docs * Cover new features and how to use them. --- Documentation/source/features.rst | 41 +++++++++++++++++++++++++++++++ Documentation/source/index.rst | 1 + Documentation/source/usage.rst | 27 ++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 Documentation/source/features.rst diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst new file mode 100644 index 0000000..c715512 --- /dev/null +++ b/Documentation/source/features.rst @@ -0,0 +1,41 @@ +.. _features: + +Features +======== + +The following features are currently provided: + +.. _features_indexing: + +Indexing +-------- + +Indexing data to Elasticsearch is provided. The extension delivers an indexer for TCA with zero +configuration needs. Still it's possible to configure the indexer. + +Own indexer are not possible yet, but will. + +.. _features_search: + +Searching +--------- + +Currently all fields are searched for a single search input. + +Also multiple filter are supported. Filtering results by fields for string contents. + +.. _features_planned: + +Planned +--------- + +The following features are currently planned and will be integrated: + +#. Mapping Configuration + Allowing to configure the whole mapping, to define type of input, e.g. integer, keyword. + + +#. Facets / Aggregates + Based on the mapping configuration, facets will be configurable and fetched. Therefore mapping is + required and we will adjust the result set to be of a custom model providing all information in a + more clean way. diff --git a/Documentation/source/index.rst b/Documentation/source/index.rst index dfda13b..5427fc9 100644 --- a/Documentation/source/index.rst +++ b/Documentation/source/index.rst @@ -7,6 +7,7 @@ Table of Contents :maxdepth: 1 :glob: + features installation configuration usage diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 1fc0e51..453b17d 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -37,3 +37,30 @@ Searching / Frontend Plugin To provide a search interface you can insert the frontend Plugin as normal content element of type plugin. The plugin is named *Search Core*. + +Please provide your own template, the extension will not deliver a useful template for now. + +The extbase mapping is used, this way you can create a form: + +.. code-block:: html + + + + + + +.. _usage_searching_filter: + +Filter +"""""" + +Thanks to extbase mapping, filter are added to the form: + +.. code-block:: html + :emphasize-lines: 3 + + + + + + From aa8d7e36e6e9e8735a5b0ef3eb283d68c125347b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 29 Jun 2017 09:13:39 +0200 Subject: [PATCH 010/158] TASK: Add test covering index command * To enable testing new configuration structure. --- Classes/Command/IndexCommandController.php | 4 +- .../Command/IndexCommandControllerTest.php | 106 ++++++++++++++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 3 + 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 Tests/Unit/Command/IndexCommandControllerTest.php diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 55ac4df..78ee643 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -57,9 +57,7 @@ class IndexCommandController extends CommandController { // TODO: Allow to index multiple tables at once? // TODO: Also allow to index everything? - - // TODO: There is no test yet! - // TODO: Add test, adjust config path + // TODO: Adjust config path if (! in_array($table, GeneralUtility::trimExplode(',', $this->configuration->get('indexer.tca.allowedTables')))) { $this->outputLine('Table is not allowed for indexing.'); $this->quit(1); diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php new file mode 100644 index 0000000..04e0dd0 --- /dev/null +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -0,0 +1,106 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Command\IndexCommandController; +use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Leonmrni\SearchCore\Domain\Index\TcaIndexer; +use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; +use TYPO3\CMS\Extbase\Mvc\Exception\StopActionException; + +class IndexCommandControllerTest extends AbstractUnitTestCase +{ + /** + * @var IndexCommandController + */ + protected $subject; + + /** + * @var IndexerFactory + */ + protected $indexerFactory; + + public function setUp() + { + parent::setUp(); + + $this->indexerFactory = $this->getMockBuilder(IndexerFactory::class) + ->disableOriginalConstructor() + ->getMock(); + $configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $configuration->expects($this->once()) + ->method('get') + ->with('indexer.tca.allowedTables') + ->will($this->returnValue('allowedTable, anotherTable,YetAnotherTable')); + + $this->subject = $this->getMockBuilder(IndexCommandController::class) + ->disableOriginalConstructor() + ->setMethods(['quit', 'outputLine']) + ->getMock(); + $this->subject->injectIndexerFactory($this->indexerFactory); + $this->inject($this->subject, 'configuration', $configuration); + } + + /** + * @test + */ + public function indexerStopsForNonAllowedTable() + { + $this->expectException(StopActionException::class); + $this->subject->expects($this->once()) + ->method('quit') + ->with(1) + ->will($this->throwException(new StopActionException)); + + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('Table is not allowed for indexing.'); + $this->indexerFactory->expects($this->never()) + ->method('getIndexer'); + + $this->subject->indexCommand('nonAllowedTable'); + } + + /** + * @test + */ + public function indexerExecutesForAllowedTable() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $this->subject->expects($this->never()) + ->method('quit'); + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('Table was indexed.'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('allowedTable') + ->will($this->returnValue($indexerMock)); + + $this->subject->indexCommand('allowedTable'); + } +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 83f031e..9cb0a3d 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -27,6 +27,9 @@ use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; class QueryFactoryTest extends AbstractUnitTestCase { + /** + * @var QueryFactory + */ protected $subject; public function setUp() From fde592f2e3f2258bb9c1330275cf0f3e6c1ab995 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 29 Jun 2017 09:18:31 +0200 Subject: [PATCH 011/158] TASK: Adjust configuration for indexing * Adjust used configuration in command. --- Classes/Command/IndexCommandController.php | 3 +-- .../Command/IndexCommandControllerTest.php | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 78ee643..aef0863 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -57,8 +57,7 @@ class IndexCommandController extends CommandController { // TODO: Allow to index multiple tables at once? // TODO: Also allow to index everything? - // TODO: Adjust config path - if (! in_array($table, GeneralUtility::trimExplode(',', $this->configuration->get('indexer.tca.allowedTables')))) { + if ($this->configuration->getIfExists('indexing.' . $table) === null) { $this->outputLine('Table is not allowed for indexing.'); $this->quit(1); } diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php index 04e0dd0..c2559f9 100644 --- a/Tests/Unit/Command/IndexCommandControllerTest.php +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -40,6 +40,11 @@ class IndexCommandControllerTest extends AbstractUnitTestCase */ protected $indexerFactory; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + public function setUp() { parent::setUp(); @@ -47,20 +52,16 @@ class IndexCommandControllerTest extends AbstractUnitTestCase $this->indexerFactory = $this->getMockBuilder(IndexerFactory::class) ->disableOriginalConstructor() ->getMock(); - $configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) ->disableOriginalConstructor() ->getMock(); - $configuration->expects($this->once()) - ->method('get') - ->with('indexer.tca.allowedTables') - ->will($this->returnValue('allowedTable, anotherTable,YetAnotherTable')); $this->subject = $this->getMockBuilder(IndexCommandController::class) ->disableOriginalConstructor() ->setMethods(['quit', 'outputLine']) ->getMock(); $this->subject->injectIndexerFactory($this->indexerFactory); - $this->inject($this->subject, 'configuration', $configuration); + $this->inject($this->subject, 'configuration', $this->configuration); } /** @@ -80,6 +81,10 @@ class IndexCommandControllerTest extends AbstractUnitTestCase $this->indexerFactory->expects($this->never()) ->method('getIndexer'); + $this->configuration->expects($this->once()) + ->method('getIfExists') + ->with('indexing.nonAllowedTable') + ->will($this->returnValue(null)); $this->subject->indexCommand('nonAllowedTable'); } @@ -101,6 +106,12 @@ class IndexCommandControllerTest extends AbstractUnitTestCase ->with('allowedTable') ->will($this->returnValue($indexerMock)); + $this->configuration->expects($this->once()) + ->method('getIfExists') + ->with('indexing.allowedTable') + ->will($this->returnValue([ + 'indexer' => 'Leonmrni\SearchCore\Domain\Index\TcaIndexer', + ])); $this->subject->indexCommand('allowedTable'); } } From 9769ee1cb68b465c5a845d79bb3a185185aa9aef Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 3 Jul 2017 22:59:18 +0200 Subject: [PATCH 012/158] TASK: Update composer Require TYPO3 CMS 8.2 and PHP 7.1 as minimum. Use conventionally lower case web folder. --- composer.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index debb71f..2aba101 100644 --- a/composer.json +++ b/composer.json @@ -16,8 +16,8 @@ } }, "require" : { - "php": ">=5.6.0", - "typo3/cms": "~7.6", + "php": ">=7.1.0", + "typo3/cms": "~8.2", "ruflin/elastica": "~3.2" }, "require-dev": { @@ -30,14 +30,14 @@ }, "scripts": { "post-autoload-dump": [ - "mkdir -p .Build/Web/typo3conf/ext/", - "[ -L .Build/Web/typo3conf/ext/search_core ] || ln -snvf ../../../../. .Build/Web/typo3conf/ext/search_core" + "mkdir -p .Build/web/typo3conf/ext/", + "[ -L .Build/web/typo3conf/ext/search_core ] || ln -snvf ../../../../. .Build/web/typo3conf/ext/search_core" ] }, "extra": { "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", - "web-dir": ".Build/Web" + "web-dir": ".Build/web" } }, "authors": [ From 9d20524706053368ff9c0e82fc839cbbbb4cd16a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 4 Jul 2017 10:12:47 +0200 Subject: [PATCH 013/158] WIP|TASK: Use new TYPO3 Use testing framework configuration. Use new db. Begin with replacement of old TYPO3_DB. --- Classes/Domain/Index/TcaIndexer.php | 10 ++++++++-- Makefile | 8 +++++--- Tests/Functional/FunctionalTests.xml | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index b50c6fa..3abdcb3 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -112,7 +112,7 @@ class TcaIndexer implements IndexerInterface */ protected function getRecords($offset, $limit) { - $records = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( + $records = $this->getDatabaseConnection()->exec_SELECTgetRows( $this->tcaTableService->getFields(), $this->tcaTableService->getTableClause(), $this->tcaTableService->getWhereClause(), @@ -139,7 +139,7 @@ class TcaIndexer implements IndexerInterface */ protected function getRecord($identifier) { - $record = $GLOBALS['TYPO3_DB']->exec_SELECTgetSingleRow( + $record = $this->getDatabaseConnection()->exec_SELECTgetSingleRow( $this->tcaTableService->getFields(), $this->tcaTableService->getTableClause(), $this->tcaTableService->getWhereClause() @@ -156,4 +156,10 @@ class TcaIndexer implements IndexerInterface return $record; } + + protected function getDatabaseConnection() + { + return GeneralUtility::makeInstance(ConnectionPool::class) + ->getConnectionByName('Default'); + } } diff --git a/Makefile b/Makefile index 589dd7d..566709a 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) current_dir := $(dir $(mkfile_path)) -TYPO3_WEB_DIR := $(current_dir).Build/Web +TYPO3_WEB_DIR := $(current_dir).Build/web +TYPO3_PATH_ROOT := $(current_dir).Build/web # Allow different versions on travis -TYPO3_VERSION ?= ~7.6 -typo3DatabaseName ?= "searchcore_test" +TYPO3_VERSION ?= ~8.2 +typo3DatabaseName ?= "searchcore_test2" typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" @@ -21,6 +22,7 @@ functionalTests: typo3DatabaseHost=$(typo3DatabaseHost) \ TYPO3_PATH_WEB=$(TYPO3_WEB_DIR) \ .Build/bin/phpunit --colors --debug -v \ + --stop-on-error \ -c Tests/Functional/FunctionalTests.xml uploadCodeCoverage: uploadCodeCoverageToScrutinizer uploadCodeCoverageToCodacy diff --git a/Tests/Functional/FunctionalTests.xml b/Tests/Functional/FunctionalTests.xml index d42ef21..a318f65 100644 --- a/Tests/Functional/FunctionalTests.xml +++ b/Tests/Functional/FunctionalTests.xml @@ -1,7 +1,7 @@ Date: Tue, 4 Jul 2017 10:27:03 +0200 Subject: [PATCH 014/158] TASK: Add unit test covering configuration option Add test to cover option to configure user defined additionalWhereClause. --- .../Index/TcaIndexer/TcaTableService.php | 1 - .../Index/TcaIndexer/TcaTableServiceTest.php | 96 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 7d69e5b..ae819bc 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -151,7 +151,6 @@ class TcaTableService . ' AND pages.no_search = 0' ; - // TODO: There is no test covering this option yet? $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); if (is_string($userDefinedWhere)) { $whereClause .= ' AND ' . $userDefinedWhere; diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php new file mode 100644 index 0000000..3a5b7fd --- /dev/null +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -0,0 +1,96 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class TcaTableServiceTest extends AbstractUnitTestCase +{ + /** + * @var TcaTableService + */ + protected $subject; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + public function setUp() + { + parent::setUp(); + + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $logger = $this->getMockBuilder(\TYPO3\CMS\Core\Log\LogManager::class) + ->disableOriginalConstructor() + ->setMethods(['getLogger']) + ->getMock(); + $logger->expects($this->once()) + ->method('getLogger') + ->will($this->returnValue( + $this->getMockBuilder(\TYPO3\CMS\Core\Log\Logger::class) + ->disableOriginalConstructor() + ->getMock() + )); + + $this->subject = $this->getMockBuilder(TcaTableService::class) + ->disableOriginalConstructor() + ->setMethodsExcept(['getWhereClause', 'injectLogger', 'getTableName']) + ->getMock(); + $this->inject($this->subject, 'configuration', $this->configuration); + $this->inject($this->subject, 'logger', $logger); + $this->inject($this->subject, 'tableName', 'table'); + } + + /** + * @test + */ + public function doUsePlainQueryIfNoAdditionalWhereClauseIsDefined() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) + ->will($this->onConsecutiveCalls(null, false)); + + $this->assertSame( + '1=1 AND pages.no_search = 0', + $this->subject->getWhereClause() + ); + } + + /** + * @test + */ + public function configuredAdditionalWhereClauseIsAdded() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) + ->will($this->onConsecutiveCalls('table.field = "someValue"', false)); + + $this->assertSame( + '1=1 AND pages.no_search = 0 AND table.field = "someValue"', + $this->subject->getWhereClause() + ); + } +} From 432335c80dce545dea133fe057b4ab0b12adeb65 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 4 Jul 2017 12:12:36 +0200 Subject: [PATCH 015/158] FEATURE: Allow custom indexer Adjust code to use defined indexer as FQCN, to allow any class to be used as indexer. Also classes defined by user. --- Classes/Command/IndexCommandController.php | 24 ++-- Classes/Domain/Index/AbstractIndexer.php | 113 ++++++++++++++++++ Classes/Domain/Index/IndexerFactory.php | 62 ++++++++-- .../Index/NoMatchingIndexerException.php | 25 ++++ Classes/Domain/Index/TcaIndexer.php | 70 ++--------- Classes/Domain/Service/DataHandler.php | 60 ++++++---- Classes/Hook/DataHandler.php | 12 +- .../DataHandler/AbstractDataHandlerTest.php | 6 +- .../Command/IndexCommandControllerTest.php | 37 ++---- 9 files changed, 255 insertions(+), 154 deletions(-) create mode 100644 Classes/Domain/Index/AbstractIndexer.php create mode 100644 Classes/Domain/Index/NoMatchingIndexerException.php diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index aef0863..227918f 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Command; */ use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; @@ -34,12 +35,6 @@ class IndexCommandController extends CommandController */ protected $indexerFactory; - /** - * @var \Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface - * @inject - */ - protected $configuration; - /** * @param IndexerFactory $factory */ @@ -49,19 +44,18 @@ class IndexCommandController extends CommandController } /** - * Will index the given table or everything. + * Will index the given identifier. * - * @param string $table + * @param string $identifier */ - public function indexCommand($table) + public function indexCommand($identifier) { - // TODO: Allow to index multiple tables at once? // TODO: Also allow to index everything? - if ($this->configuration->getIfExists('indexing.' . $table) === null) { - $this->outputLine('Table is not allowed for indexing.'); - $this->quit(1); + try { + $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); + $this->outputLine($identifier . ' was indexed.'); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $identifier); } - $this->indexerFactory->getIndexer($table)->indexAllDocuments(); - $this->outputLine('Table was indexed.'); } } diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php new file mode 100644 index 0000000..acd046e --- /dev/null +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Connection\ConnectionInterface; + +abstract class AbstractIndexer implements IndexerInterface +{ + /** + * @var ConnectionInterface + */ + protected $connection; + + /** + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * Inject log manager to get concrete logger from it. + * + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + + /** + * @param ConnectionInterface $connection + */ + public function __construct(ConnectionInterface $connection) + { + $this->connection = $connection; + } + + public function indexAllDocuments() + { + $this->logger->info('Start indexing'); + foreach ($this->getRecordGenerator() as $records) { + $this->logger->debug('Index records.', [$records]); + if ($records === null) { + break; + } + + $this->connection->addDocuments($this->getDocumentName(), $records); + } + $this->logger->info('Finish indexing'); + } + + public function indexDocument($identifier) + { + $this->logger->info('Start indexing single record.', [$identifier]); + try { + $this->connection->addDocument($this->getDocumentName(), $this->getRecord($identifier)); + } catch (NoRecordFoundException $e) { + $this->logger->info('Could not index document.', [$e->getMessage()]); + } + $this->logger->info('Finish indexing'); + } + + /** + * @return \Generator + */ + protected function getRecordGenerator() + { + $offset = 0; + // TODO: Make configurable. + $limit = 50; + + while (($records = $this->getRecords($offset, $limit)) !== []) { + yield $records; + $offset += $limit; + } + } + + /** + * @param int $offset + * @param int $limit + * @return array|null + */ + abstract protected function getRecords($offset, $limit); + + /** + * @param int $identifier + * @return array + * @throws NoRecordFoundException If record could not be found. + */ + abstract protected function getRecord($identifier); + + /** + * @return string + */ + abstract protected function getDocumentName(); +} diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 4ecbfb9..0140122 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -20,6 +20,11 @@ namespace Leonmrni\SearchCore\Domain\Index; * 02110-1301, USA. */ +use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Configuration\InvalidArgumentException; +use Leonmrni\SearchCore\Domain\Index\IndexerInterface; +use Leonmrni\SearchCore\Domain\Index\TcaIndexer; +use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -33,28 +38,61 @@ class IndexerFactory implements Singleton */ protected $objectManager; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + /** * @param ObjectManagerInterface $objectManager */ - public function __construct(ObjectManagerInterface $objectManager) - { + public function __construct( + ObjectManagerInterface $objectManager, + ConfigurationContainerInterface $configuration + ) { $this->objectManager = $objectManager; + $this->configuration = $configuration; } /** - * @param string $tableName + * @param string $identifier * * @return IndexerInterface + * @throws NoMatchingIndexer */ - public function getIndexer($tableName) + public function getIndexer($identifier) { - // This is the place to use configuration to return different indexer. - return $this->objectManager->get( - TcaIndexer::Class, - $this->objectManager->get( - TcaIndexer\TcaTableService::class, - $tableName - ) - ); + try { + return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier); + } catch (NoMatchingIndexerException $e) { + // Nothing to do, we throw exception below + } catch (InvalidArgumentException $e) { + // Nothing to do, we throw exception below + } + + throw new NoMatchingIndexerException('Could not find an indexer for ' . $identifier, 1497341442); + } + + /** + * @param string $indexerClass + * @param string $identifier + * + * @return IndexerInterface + * @throws NoMatchingIndexer + */ + protected function buildIndexer($indexerClass, $identifier) + { + if ($indexerClass === TcaIndexer::class) { + return $this->objectManager->get( + TcaIndexer::class, + $this->objectManager->get(TcaTableService::class, $identifier) + ); + } + + if (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) { + return $this->objectManager->get($indexerClass); + } + + throw new NoMatchingIndexerException('Could not find indexer: ' . $indexerClass, 1497341442); } } diff --git a/Classes/Domain/Index/NoMatchingIndexerException.php b/Classes/Domain/Index/NoMatchingIndexerException.php new file mode 100644 index 0000000..02904cc --- /dev/null +++ b/Classes/Domain/Index/NoMatchingIndexerException.php @@ -0,0 +1,25 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class NoMatchingIndexerException extends IndexingException +{ +} diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index b50c6fa..52f1efa 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -26,33 +26,13 @@ use Leonmrni\SearchCore\Connection\ConnectionInterface; /** * Will index the given table using configuration from TCA. */ -class TcaIndexer implements IndexerInterface +class TcaIndexer extends AbstractIndexer { - /** - * @var ConnectionInterface - */ - protected $connection; - /** * @var TcaIndexer\TcaTableService */ protected $tcaTableService; - /** - * @var \TYPO3\CMS\Core\Log\Logger - */ - protected $logger; - - /** - * Inject log manager to get concrete logger from it. - * - * @param \TYPO3\CMS\Core\Log\LogManager $logManager - */ - public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) - { - $this->logger = $logManager->getLogger(__CLASS__); - } - /** * @param TcaIndexer\TcaTableService $tcaTableService * @param ConnectionInterface $connection @@ -65,46 +45,6 @@ class TcaIndexer implements IndexerInterface $this->connection = $connection; } - public function indexAllDocuments() - { - $this->logger->info('Start indexing'); - foreach ($this->getRecordGenerator() as $records) { - $this->logger->debug('Index records.', [$records]); - if ($records === null) { - break; - } - - $this->connection->addDocuments($this->tcaTableService->getTableName(), $records); - } - $this->logger->info('Finish indexing'); - } - - public function indexDocument($identifier) - { - $this->logger->info('Start indexing single record.', [$identifier]); - try { - $this->connection->addDocument($this->tcaTableService->getTableName(), $this->getRecord($identifier)); - } catch (NoRecordFoundException $e) { - $this->logger->info('Could not index document.', [$e->getMessage()]); - } - $this->logger->info('Finish indexing'); - } - - /** - * @return \Generator - */ - protected function getRecordGenerator() - { - $offset = 0; - // TODO: Make configurable. - $limit = 50; - - while (($records = $this->getRecords($offset, $limit)) !== []) { - yield $records; - $offset += $limit; - } - } - /** * @param int $offset * @param int $limit @@ -156,4 +96,12 @@ class TcaIndexer implements IndexerInterface return $record; } + + /** + * @return string + */ + protected function getDocumentName() + { + return $this->tcaTableService->getTableName(); + } } diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 578589a..bec9eb3 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -21,6 +21,8 @@ namespace Leonmrni\SearchCore\Domain\Service; */ use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; use Leonmrni\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -47,8 +49,7 @@ class DataHandler implements Singleton protected $connection; /** - * @var \Leonmrni\SearchCore\Domain\Index\IndexerFactory - * @inject + * @var IndexerFactory */ protected $indexerFactory; @@ -74,30 +75,12 @@ class DataHandler implements Singleton /** * @param ConfigurationContainerInterface $configuration + * @param IndexerFactory $indexerFactory */ - public function __construct(ConfigurationContainerInterface $configuration) + public function __construct(ConfigurationContainerInterface $configuration, IndexerFactory $indexerFactory) { $this->configuration = $configuration; - } - - /** - * Get all tables that are allowed for indexing. - * - * @return array - */ - public function getAllowedTablesForIndexing() - { - $tables = []; - foreach ($this->configuration->get('indexing') as $tableName => $indexConfiguration) { - if ($indexConfiguration['indexer'] === TcaIndexer::class) { - $tables[] = $tableName; - continue; - } - // TODO: Support custom indexer. - // Define "interface" / option which is used. - } - - return $tables; + $this->indexerFactory = $indexerFactory; } /** @@ -107,7 +90,7 @@ class DataHandler implements Singleton public function add($table, array $record) { $this->logger->debug('Record received for add.', [$table, $record]); - $this->indexerFactory->getIndexer($table)->indexDocument($record['uid']); + $this->getIndexer($table)->indexDocument($record['uid']); } /** @@ -116,7 +99,7 @@ class DataHandler implements Singleton public function update($table, array $record) { $this->logger->debug('Record received for update.', [$table, $record]); - $this->indexerFactory->getIndexer($table)->indexDocument($record['uid']); + $this->getIndexer($table)->indexDocument($record['uid']); } /** @@ -128,4 +111,31 @@ class DataHandler implements Singleton $this->logger->debug('Record received for delete.', [$table, $identifier]); $this->connection->deleteDocument($table, $identifier); } + + /** + * @param string $table + * @return IndexerInterface + * + * @throws NoMatchingIndexerException + */ + protected function getIndexer($table) + { + return $this->indexerFactory->getIndexer($table); + } + + /** + * @param string $table + * @return bool + */ + public function canHandle($table) + { + try { + $this->getIndexer($table); + return true; + } catch (NoMatchingIndexerException $e) { + return false; + } + + return false; + } } diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 1bcb4a5..34cf3f3 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Hook; */ use Leonmrni\SearchCore\Configuration\NoConfigurationException; +use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; use Leonmrni\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; @@ -140,7 +141,7 @@ class DataHandler implements Singleton $this->logger->debug('Datahandler could not be setup.'); return false; } - if (! $this->shouldProcessTable($table)) { + if (! $this->dataHandler->canHandle($table)) { $this->logger->debug('Table is not allowed.', [$table]); return false; } @@ -148,15 +149,6 @@ class DataHandler implements Singleton return true; } - /** - * @param string $table - * @return bool - */ - protected function shouldProcessTable($table) - { - return in_array($table, $this->dataHandler->getAllowedTablesForIndexing()); - } - /** * Wrapper to allow unit testing. * diff --git a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php index a26021d..b3c384b 100644 --- a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php +++ b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php @@ -21,6 +21,7 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; */ use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Domain\Index\IndexerFactory; use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; @@ -42,7 +43,10 @@ abstract class AbstractDataHandlerTest extends AbstractFunctionalTestCase $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $this->subject = $this->getMockBuilder(DataHandlerService::class) - ->setConstructorArgs([$objectManager->get(ConfigurationContainerInterface::class)]) + ->setConstructorArgs([ + $objectManager->get(ConfigurationContainerInterface::class), + $objectManager->get(IndexerFactory::class) + ]) ->setMethods(['add', 'update', 'delete']) ->getMock(); diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php index c2559f9..3dff968 100644 --- a/Tests/Unit/Command/IndexCommandControllerTest.php +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -21,8 +21,8 @@ namespace Leonmrni\SearchCore\Tests\Unit\Command; */ use Leonmrni\SearchCore\Command\IndexCommandController; -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; use Leonmrni\SearchCore\Domain\Index\TcaIndexer; use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; @@ -40,11 +40,6 @@ class IndexCommandControllerTest extends AbstractUnitTestCase */ protected $indexerFactory; - /** - * @var ConfigurationContainerInterface - */ - protected $configuration; - public function setUp() { parent::setUp(); @@ -52,16 +47,12 @@ class IndexCommandControllerTest extends AbstractUnitTestCase $this->indexerFactory = $this->getMockBuilder(IndexerFactory::class) ->disableOriginalConstructor() ->getMock(); - $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) - ->disableOriginalConstructor() - ->getMock(); $this->subject = $this->getMockBuilder(IndexCommandController::class) ->disableOriginalConstructor() ->setMethods(['quit', 'outputLine']) ->getMock(); $this->subject->injectIndexerFactory($this->indexerFactory); - $this->inject($this->subject, 'configuration', $this->configuration); } /** @@ -69,22 +60,14 @@ class IndexCommandControllerTest extends AbstractUnitTestCase */ public function indexerStopsForNonAllowedTable() { - $this->expectException(StopActionException::class); - $this->subject->expects($this->once()) - ->method('quit') - ->with(1) - ->will($this->throwException(new StopActionException)); - $this->subject->expects($this->once()) ->method('outputLine') - ->with('Table is not allowed for indexing.'); - $this->indexerFactory->expects($this->never()) - ->method('getIndexer'); + ->with('No indexer found for: nonAllowedTable'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('nonAllowedTable') + ->will($this->throwException(new NoMatchingIndexerException)); - $this->configuration->expects($this->once()) - ->method('getIfExists') - ->with('indexing.nonAllowedTable') - ->will($this->returnValue(null)); $this->subject->indexCommand('nonAllowedTable'); } @@ -100,18 +83,12 @@ class IndexCommandControllerTest extends AbstractUnitTestCase ->method('quit'); $this->subject->expects($this->once()) ->method('outputLine') - ->with('Table was indexed.'); + ->with('allowedTable was indexed.'); $this->indexerFactory->expects($this->once()) ->method('getIndexer') ->with('allowedTable') ->will($this->returnValue($indexerMock)); - $this->configuration->expects($this->once()) - ->method('getIfExists') - ->with('indexing.allowedTable') - ->will($this->returnValue([ - 'indexer' => 'Leonmrni\SearchCore\Domain\Index\TcaIndexer', - ])); $this->subject->indexCommand('allowedTable'); } } From 975381cc4a25bb480b2ba6266b932139795c81d0 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 12:03:52 +0200 Subject: [PATCH 016/158] TASK: Integrate working code Copied code from customer installation with working implementation. --- .../ConfigurationContainerInterface.php | 2 + Classes/Connection/Elasticsearch.php | 40 +++++- Classes/Connection/Elasticsearch/Facet.php | 93 +++++++++++++ .../Connection/Elasticsearch/FacetOption.php | 61 +++++++++ .../Elasticsearch/MappingFactory.php | 73 +++++++++++ .../Connection/Elasticsearch/ResultItem.php | 56 ++++++++ .../Connection/Elasticsearch/SearchResult.php | 123 +++++++++++++++++- Classes/Connection/FacetInterface.php | 39 ++++++ Classes/Connection/FacetOptionInterface.php | 42 ++++++ Classes/Connection/FacetRequestInterface.php | 42 ++++++ Classes/Connection/ResultItemInterface.php | 29 +++++ Classes/Connection/SearchResultInterface.php | 14 +- Classes/Domain/Index/IndexerFactory.php | 1 + Classes/Domain/Index/TcaIndexer.php | 1 - Classes/Domain/Model/FacetRequest.php | 62 +++++++++ Classes/Domain/Model/SearchRequest.php | 28 +++- Classes/Domain/Search/QueryFactory.php | 103 ++++++++++++--- Classes/Domain/Search/SearchService.php | 56 +++++++- 18 files changed, 832 insertions(+), 33 deletions(-) create mode 100644 Classes/Connection/Elasticsearch/Facet.php create mode 100644 Classes/Connection/Elasticsearch/FacetOption.php create mode 100644 Classes/Connection/Elasticsearch/MappingFactory.php create mode 100644 Classes/Connection/Elasticsearch/ResultItem.php create mode 100644 Classes/Connection/FacetInterface.php create mode 100644 Classes/Connection/FacetOptionInterface.php create mode 100644 Classes/Connection/FacetRequestInterface.php create mode 100644 Classes/Connection/ResultItemInterface.php create mode 100644 Classes/Domain/Model/FacetRequest.php diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index 7bc45ef..d0d9828 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -34,6 +34,8 @@ interface ConfigurationContainerInterface extends Singleton * * @param string $path In dot notation. E.g. indexer.tca.allowedTables * @return mixed + * + * @throws InvalidArgumentException */ public function get($path); diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 386e19c..88b93c2 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -20,8 +20,10 @@ namespace Leonmrni\SearchCore\Connection; * 02110-1301, USA. */ -use TYPO3\CMS\Core\SingletonInterface as Singleton; +use Leonmrni\SearchCore\Connection\Elasticsearch\SearchResult; use Leonmrni\SearchCore\Domain\Search\QueryFactory; +use TYPO3\CMS\Core\SingletonInterface as Singleton; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** * Outer wrapper to elasticsearch. @@ -43,6 +45,11 @@ class Elasticsearch implements Singleton, ConnectionInterface */ protected $typeFactory; + /** + * @var Elasticsearch\MappingFactory + */ + protected $mappingFactory; + /** * @var Elasticsearch\DocumentFactory */ @@ -58,6 +65,11 @@ class Elasticsearch implements Singleton, ConnectionInterface */ protected $logger; + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + /** * Inject log manager to get concrete logger from it. * @@ -68,10 +80,19 @@ class Elasticsearch implements Singleton, ConnectionInterface $this->logger = $logManager->getLogger(__CLASS__); } + /** + * @param ObjectManagerInterface $objectManager + */ + public function injectObjectManager(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + /** * @param Elasticsearch\Connection $connection * @param Elasticsearch\IndexFactory $indexFactory * @param Elasticsearch\TypeFactory $typeFactory + * @param Elasticsearch\MappingFactory $mappingFactory * @param Elasticsearch\DocumentFactory $documentFactory * @param QueryFactory $queryFactory */ @@ -79,12 +100,14 @@ class Elasticsearch implements Singleton, ConnectionInterface Elasticsearch\Connection $connection, Elasticsearch\IndexFactory $indexFactory, Elasticsearch\TypeFactory $typeFactory, + Elasticsearch\MappingFactory $mappingFactory, Elasticsearch\DocumentFactory $documentFactory, QueryFactory $queryFactory ) { $this->connection = $connection; $this->indexFactory = $indexFactory; $this->typeFactory = $typeFactory; + $this->mappingFactory = $mappingFactory; $this->documentFactory = $documentFactory; $this->queryFactory = $queryFactory; } @@ -142,6 +165,13 @@ class Elasticsearch implements Singleton, ConnectionInterface protected function withType($documentType, callable $callback) { $type = $this->getType($documentType); + // TODO: Check whether it's to heavy to send it so often e.g. for every single document. + // Perhaps add command controller to submit mapping?! + // Also it's not possible to change mapping without deleting index first. + // Mattes told about a solution. + // So command looks like the best way so far, except we manage mattes solution. + // Still then this should be done once. So perhaps singleton which tracks state and does only once? + $this->mappingFactory->getMapping($type)->send(); $callback($type); $type->getIndex()->refresh(); } @@ -149,7 +179,7 @@ class Elasticsearch implements Singleton, ConnectionInterface /** * @param SearchRequestInterface $searchRequest * - * @return \Elastica\ResultSet + * @return SearchResultInterface */ public function search(SearchRequestInterface $searchRequest) { @@ -157,11 +187,9 @@ class Elasticsearch implements Singleton, ConnectionInterface $search = new \Elastica\Search($this->connection->getClient()); $search->addIndex('typo3content'); - $search->setQuery($this->queryFactory->create($this, $searchRequest)); + $search->setQuery($this->queryFactory->create($searchRequest)); - // TODO: Return wrapped result to implement our interface. - // Also update php doc to reflect the change. - return $search->search(); + return $this->objectManager->get(SearchResult::class, $search->search()); } /** diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php new file mode 100644 index 0000000..79a87b3 --- /dev/null +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -0,0 +1,93 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Connection\FacetInterface; + +class Facet implements FacetInterface +{ + /** + * @var string + */ + protected $name = ''; + + /** + * @var string + */ + protected $field = ''; + + /** + * @var array + */ + protected $buckets = []; + + /** + * @var array + */ + protected $options = []; + + public function __construct($name, array $aggregation, ConfigurationContainerInterface $configuration) + { + $this->name = $name; + $this->buckets = $aggregation['buckets']; + $this->field = $configuration->getIfExists('searching.facets.' . $this->name . '.field') ?: ''; + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } + + /** + * @return string + */ + public function getField() + { + return $this->field; + } + + /** + * Returns all possible options for this facet. + * + * @return array + */ + public function getOptions() + { + $this->initOptions(); + + return $this->options; + } + + protected function initOptions() + { + if ($this->options !== []) { + return; + } + + foreach ($this->buckets as $bucket) { + $this->options[] = new FacetOption($bucket); + } + } +} diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php new file mode 100644 index 0000000..a618be1 --- /dev/null +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -0,0 +1,61 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Connection\FacetOptionInterface; + +class FacetOption implements FacetOptionInterface +{ + /** + * @var string + */ + protected $name = ''; + + /** + * @var int + */ + protected $count = 0; + + /** + * @param array $bucket + */ + public function __construct(array $bucket) + { + $this->name = $bucket['key']; + $this->count = $bucket['doc_count']; + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } + + /** + * @return int + */ + public function getCount() + { + return $this->count; + } +} diff --git a/Classes/Connection/Elasticsearch/MappingFactory.php b/Classes/Connection/Elasticsearch/MappingFactory.php new file mode 100644 index 0000000..f029cd1 --- /dev/null +++ b/Classes/Connection/Elasticsearch/MappingFactory.php @@ -0,0 +1,73 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Configuration\InvalidArgumentException; +use TYPO3\CMS\Core\SingletonInterface as Singleton; + +/** + * Factory to get mappings. + */ +class MappingFactory implements Singleton +{ + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @param ConfigurationContainerInterface $configuration + */ + public function __construct(ConfigurationContainerInterface $configuration) + { + $this->configuration = $configuration; + } + + /** + * Get an mapping based on type. + * + * @param \Elastica\Type $type + * + * @return \Elastica\Mapping + */ + public function getMapping(\Elastica\Type $type) + { + $mapping = new \Elastica\Type\Mapping(); + $mapping->setType($type); + $mapping->setProperties($this->getConfiguration($type->getName())); + + return $mapping; + } + + /** + * @param string $identifier + * @return array + */ + protected function getConfiguration($identifier) + { + try { + return $this->configuration->get('indexing.' . $identifier . '.mapping'); + } catch (InvalidArgumentException $e) { + return []; + } + } +} diff --git a/Classes/Connection/Elasticsearch/ResultItem.php b/Classes/Connection/Elasticsearch/ResultItem.php new file mode 100644 index 0000000..dd8cb62 --- /dev/null +++ b/Classes/Connection/Elasticsearch/ResultItem.php @@ -0,0 +1,56 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Connection\ResultItemInterface; + +class ResultItem implements ResultItemInterface +{ + /** + * @var array + */ + protected $data = []; + + public function __construct(\Elastica\Result $result) + { + $this->data = $result->getData(); + } + + public function offsetExists($offset) + { + return isset($this->data[$offset]); + } + + public function offsetGet($offset) + { + return $this->data[$offset]; + } + + public function offsetSet() + { + throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077); + } + + public function offsetUnset() + { + throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077); + } +} diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 3f90d60..63c93b0 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -20,12 +20,127 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ +use Leonmrni\SearchCore\Connection\FacetInterface; +use Leonmrni\SearchCore\Connection\ResultItemInterface; use Leonmrni\SearchCore\Connection\SearchResultInterface; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; -/** - * - */ -class SearchResult extends \Elastica\SearchResult implements SearchResultInterface +class SearchResult implements SearchResultInterface { + /** + * @var \Elastica\ResultSet + */ + protected $result; + /** + * @var array + */ + protected $facets = []; + + /** + * @var array + */ + protected $results = []; + + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + public function __construct(\Elastica\ResultSet $result, ObjectManagerInterface $objectManager) + { + $this->result = $result; + $this->objectManager = $objectManager; + } + + /** + * @return array + */ + public function getResults() + { + $this->initResults(); + + return $this->results; + } + + /** + * Return all facets, if any. + * + * @return array + */ + public function getFacets() + { + $this->initFacets(); + + return $this->facets; + } + + /** + * Returns the total sum of matching results. + * + * @return int + */ + public function getTotalCount() + { + return $this->result->getTotalHits(); + } + + // Countable - Interface + /** + * Returns the total sum of results contained in this result. + * + * @return int + */ + public function count() + { + return $this->result->count(); + } + + // Iterator - Interface + public function current() + { + return $this->result->current(); + } + + public function next() + { + return $this->result->next(); + } + + public function key() + { + return $this->result->key(); + } + + public function valid() + { + return $this->result->valid(); + } + + public function rewind() + { + $this->result->rewind(); + } + + protected function initResults() + { + if ($this->results !== []) { + return; + } + + foreach ($this->result->getResults() as $result) { + $this->results[] = new ResultItem($result); + } + } + + protected function initFacets() + { + if ($this->facets !== [] || !$this->result->hasAggregations()) { + return; + } + + foreach ($this->result->getAggregations() as $aggregationName => $aggregation) { + $this->facets[] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation); + } + } } diff --git a/Classes/Connection/FacetInterface.php b/Classes/Connection/FacetInterface.php new file mode 100644 index 0000000..f422e98 --- /dev/null +++ b/Classes/Connection/FacetInterface.php @@ -0,0 +1,39 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * A single facet, e.g. keyword based. + */ +interface FacetInterface +{ + /** + * @return string + */ + public function getName(); + + /** + * Returns all possible options for this facet. + * + * @return array + */ + public function getOptions(); +} diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php new file mode 100644 index 0000000..b05445b --- /dev/null +++ b/Classes/Connection/FacetOptionInterface.php @@ -0,0 +1,42 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * A single possible option of a facet. + */ +interface FacetOptionInterface +{ + /** + * Returns the name of this option. Equivalent + * to value used for filtering. + * + * @return string + */ + public function getName(); + + /** + * Returns the number of found results for this option. + * + * @return int + */ + public function getCount(); +} diff --git a/Classes/Connection/FacetRequestInterface.php b/Classes/Connection/FacetRequestInterface.php new file mode 100644 index 0000000..e5a60a1 --- /dev/null +++ b/Classes/Connection/FacetRequestInterface.php @@ -0,0 +1,42 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Used to request facets / aggregates from connection. + */ +interface FacetRequestInterface +{ + /** + * The identifier of the facet, used as key in arrays and to get the facet + * from search request, etc. + * + * @return string + */ + public function getIdentifier(); + + /** + * The field to use for facet building. + * + * @return string + */ + public function getField(); +} diff --git a/Classes/Connection/ResultItemInterface.php b/Classes/Connection/ResultItemInterface.php new file mode 100644 index 0000000..f4c9185 --- /dev/null +++ b/Classes/Connection/ResultItemInterface.php @@ -0,0 +1,29 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Use ArrayAccess to enable retrieval of information in fluid. + */ +interface ResultItemInterface extends \ArrayAccess +{ + +} diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index f00e97c..a6b2c90 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -21,9 +21,19 @@ namespace Leonmrni\SearchCore\Connection; */ /** - * + * A search result. */ -interface SearchResultInterface extends \Iterator, \Countable, \ArrayAccess +interface SearchResultInterface extends \Iterator, \Countable { + /** + * @return array + */ + public function getResults(); + /** + * Return all facets, if any. + * + * @return array + */ + public function getFacets(); } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 0140122..24addd1 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -45,6 +45,7 @@ class IndexerFactory implements Singleton /** * @param ObjectManagerInterface $objectManager + * @param ConfigurationContainerInterface $configuration */ public function __construct( ObjectManagerInterface $objectManager, diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 52f1efa..17dc61e 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -20,7 +20,6 @@ namespace Leonmrni\SearchCore\Domain\Index; * 02110-1301, USA. */ -use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; use Leonmrni\SearchCore\Connection\ConnectionInterface; /** diff --git a/Classes/Domain/Model/FacetRequest.php b/Classes/Domain/Model/FacetRequest.php new file mode 100644 index 0000000..715e74a --- /dev/null +++ b/Classes/Domain/Model/FacetRequest.php @@ -0,0 +1,62 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Leonmrni\SearchCore\Connection\FacetRequestInterface; + +class FacetRequest implements FacetRequestInterface +{ + /** + * @var string + */ + protected $identifier = ''; + + /** + * @var string + */ + protected $field = ''; + + /** + * @param string $identifier + * @param string $field + */ + public function __construct($identifier, $field) + { + $this->identifier = $identifier; + $this->field = $field; + } + + /** + * @return string + */ + public function getIdentifier() + { + return $this->identifier; + } + + /** + * @return string + */ + public function getField() + { + return $this->field; + } +} diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index f1ad18e..65ab979 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -20,6 +20,7 @@ namespace Leonmrni\SearchCore\Domain\Model; * 02110-1301, USA. */ +use Leonmrni\SearchCore\Connection\FacetRequestInterface; use Leonmrni\SearchCore\Connection\SearchRequestInterface; /** @@ -39,6 +40,11 @@ class SearchRequest implements SearchRequestInterface */ protected $filter = []; + /** + * @var array + */ + protected $facets = []; + /** * @param string $query */ @@ -68,7 +74,7 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_map('strval', $filter); + $this->filter = array_filter(array_map('strval', $filter)); } /** @@ -86,4 +92,24 @@ class SearchRequest implements SearchRequestInterface { return $this->filter; } + + /** + * Add a facet to gather in this search request. + * + * @param FacetRequestInterface $facet + */ + public function addFacet(FacetRequestInterface $facet) + { + $this->facets[$facet->getIdentifier()] = $facet; + } + + /** + * Returns all configured facets to fetch in this search request. + * + * @return array + */ + public function getFacets() + { + return $this->facets; + } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 31c50df..778b575 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -23,44 +23,117 @@ namespace Leonmrni\SearchCore\Domain\Search; use Leonmrni\SearchCore\Connection\ConnectionInterface; use Leonmrni\SearchCore\Connection\Elasticsearch\Query; use Leonmrni\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Extbase\Utility\ArrayUtility; class QueryFactory { /** - * @param ConnectionInterface $connection + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * @var array + */ + protected $query = []; + + /** + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function __construct(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + + /** * @param SearchRequestInterface $searchRequest * * @return \Elastica\Query */ - public function create( - ConnectionInterface $connection, - SearchRequestInterface $searchRequest - ) { + public function create(SearchRequestInterface $searchRequest) + { return $this->createElasticaQuery($searchRequest); } /** * @param SearchRequestInterface $searchRequest + * + * TODO: This is not in scope Elasticsearch, therefore should not return elastica. * @return \Elastica\Query */ protected function createElasticaQuery(SearchRequestInterface $searchRequest) { - $query = [ - 'bool' => [ - 'must' => [ - [ - 'match' => [ - '_all' => $searchRequest->getSearchTerm() + $this->addSearch($searchRequest); + $this->addFilter($searchRequest); + $this->addFacets($searchRequest); + + // TODO: Add logging here. + $this->logger->debug('Generated elasticsearch query.', [$this->query]); + return new \Elastica\Query($this->query); + } + + /** + * @param SearchRequestInterface $searchRequest + */ + protected function addSearch(SearchRequestInterface $searchRequest) + { + $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + 'query' => [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => $searchRequest->getSearchTerm() + ], ], ], ], ], - ]; + ]); + } - if ($searchRequest->hasFilter()) { - $query['bool']['filter'] = ['term' => $searchRequest->getFilter()]; + /** + * @param SearchRequestInterface $searchRequest + */ + protected function addFilter(SearchRequestInterface $searchRequest) + { + if (! $searchRequest->hasFilter()) { + return; } - return new \Elastica\Query(['query' => $query]); + $terms = []; + foreach ($searchRequest->getFilter() as $name => $value) { + $terms[] = [ + 'term' => [ + $name => $value, + ], + ]; + } + + $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + 'query' => [ + 'bool' => [ + 'filter' => $terms, + ], + ], + ]); + } + + /** + * @param SearchRequestInterface $searchRequest + */ + protected function addFacets(SearchRequestInterface $searchRequest) + { + foreach ($searchRequest->getFacets() as $facet) { + $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + 'aggs' => [ + $facet->getIdentifier() => [ + 'terms' => [ + 'field' => $facet->getField(), + ], + ], + ], + ]); + } } } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index afbaf7e..ac991cd 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -20,10 +20,13 @@ namespace Leonmrni\SearchCore\Domain\Search; * 02110-1301, USA. */ +use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Leonmrni\SearchCore\Configuration\InvalidArgumentException; use Leonmrni\SearchCore\Connection\ConnectionInterface; use Leonmrni\SearchCore\Connection\SearchRequestInterface; use Leonmrni\SearchCore\Connection\SearchResultInterface; -use Leonmrni\SearchCore\Domain\Model\SearchRequest; +use Leonmrni\SearchCore\Domain\Model\FacetRequest; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** * Service to process a search request. @@ -36,11 +39,28 @@ class SearchService protected $connection; /** - * @param ConnectionInterface $connection + * @var ConfigurationContainerInterface */ - public function __construct(ConnectionInterface $connection) - { + protected $configuration; + + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + /** + * @param ConnectionInterface $connection + * @param ConfigurationContainerInterface $configuration + * @param ObjectManagerInterface $objectManager + */ + public function __construct( + ConnectionInterface $connection, + ConfigurationContainerInterface $configuration, + ObjectManagerInterface $objectManager + ) { $this->connection = $connection; + $this->configuration = $configuration; + $this->objectManager = $objectManager; } /** @@ -49,6 +69,34 @@ class SearchService */ public function search(SearchRequestInterface $searchRequest) { + $this->addConfiguredFacets($searchRequest); + return $this->connection->search($searchRequest); } + + /** + * Add facets from configuration to request. + * + * @param SearchRequestInterface $searchRequest + */ + protected function addConfiguredFacets(SearchRequestInterface $searchRequest) + { + $facetsConfig = $this->configuration->getIfExists('searching.facets'); + if ($facetsConfig === null) { + return; + } + + foreach ($facetsConfig as $identifier => $facetConfig) { + if (!isset($facetConfig['field']) || trim($facetConfig['field']) === '') { + // TODO: Finish throw + throw new \Exception('message', 1499171142); + } + + $searchRequest->addFacet($this->objectManager->get( + FacetRequest::class, + $identifier, + $facetConfig['field'] + )); + } + } } From 05f846a1cfc2a6b0b04cfaa726ee6225ae151646 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 13:53:29 +0200 Subject: [PATCH 017/158] Update existing unit tests Make existing unit tests work with new code base. Also add new tests for new code to existing tests. --- Classes/Domain/Model/FacetRequest.php | 4 + Classes/Domain/Search/QueryFactory.php | 1 - Tests/Unit/AbstractUnitTestCase.php | 19 +++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 13 +-- Tests/Unit/Domain/Search/QueryFactoryTest.php | 80 +++++++++++++++---- 5 files changed, 87 insertions(+), 30 deletions(-) diff --git a/Classes/Domain/Model/FacetRequest.php b/Classes/Domain/Model/FacetRequest.php index 715e74a..5b8bea6 100644 --- a/Classes/Domain/Model/FacetRequest.php +++ b/Classes/Domain/Model/FacetRequest.php @@ -35,6 +35,10 @@ class FacetRequest implements FacetRequestInterface protected $field = ''; /** + * TODO: Add validation / exception? + * As the facets come from configuration this might be a good idea to help + * integrators find issues. + * * @param string $identifier * @param string $field */ diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 778b575..70f42ff 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -67,7 +67,6 @@ class QueryFactory $this->addFilter($searchRequest); $this->addFacets($searchRequest); - // TODO: Add logging here. $this->logger->debug('Generated elasticsearch query.', [$this->query]); return new \Elastica\Query($this->query); } diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index f3281c1..88496a1 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -24,4 +24,23 @@ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; abstract class AbstractUnitTestCase extends CoreTestCase { + /** + * @return \TYPO3\CMS\Core\Log\LogManager + */ + protected function getMockedLogger() + { + $logger = $this->getMockBuilder(\TYPO3\CMS\Core\Log\LogManager::class) + ->disableOriginalConstructor() + ->setMethods(['getLogger']) + ->getMock(); + $logger->expects($this->once()) + ->method('getLogger') + ->will($this->returnValue( + $this->getMockBuilder(\TYPO3\CMS\Core\Log\Logger::class) + ->disableOriginalConstructor() + ->getMock() + )); + + return $logger; + } } diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 3a5b7fd..837d68c 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -41,24 +41,13 @@ class TcaTableServiceTest extends AbstractUnitTestCase parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); - $logger = $this->getMockBuilder(\TYPO3\CMS\Core\Log\LogManager::class) - ->disableOriginalConstructor() - ->setMethods(['getLogger']) - ->getMock(); - $logger->expects($this->once()) - ->method('getLogger') - ->will($this->returnValue( - $this->getMockBuilder(\TYPO3\CMS\Core\Log\Logger::class) - ->disableOriginalConstructor() - ->getMock() - )); $this->subject = $this->getMockBuilder(TcaTableService::class) ->disableOriginalConstructor() ->setMethodsExcept(['getWhereClause', 'injectLogger', 'getTableName']) ->getMock(); $this->inject($this->subject, 'configuration', $this->configuration); - $this->inject($this->subject, 'logger', $logger); + $this->inject($this->subject, 'logger', $this->getMockedLogger()); $this->inject($this->subject, 'tableName', 'table'); } diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 9cb0a3d..725754f 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Tests\Unit\Domain\Search; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection; +use Leonmrni\SearchCore\Domain\Model\FacetRequest; use Leonmrni\SearchCore\Domain\Model\SearchRequest; use Leonmrni\SearchCore\Domain\Search\QueryFactory; use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -36,7 +36,7 @@ class QueryFactoryTest extends AbstractUnitTestCase { parent::setUp(); - $this->subject = new QueryFactory; + $this->subject = new QueryFactory($this->getMockedLogger()); } /** @@ -44,12 +44,9 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function creatonOfQueryWorksInGeneral() { - $connection = $this->getMockBuilder(Connection\Elasticsearch::class) - ->disableOriginalConstructor() - ->getMock(); $searchRequest = new SearchRequest('SearchWord'); - $query = $this->subject->create($connection, $searchRequest); + $query = $this->subject->create($searchRequest); $this->assertInstanceOf( \Elastica\Query::class, $query, @@ -62,32 +59,53 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { - $connection = $this->getMockBuilder(Connection\Elasticsearch::class) - ->disableOriginalConstructor() - ->getMock(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); - $query = $this->subject->create($connection, $searchRequest); + $query = $this->subject->create($searchRequest); $this->assertSame( - ['field' => 'content'], - $query->toArray()['query']['bool']['filter']['term'], + [ + ['term' => ['field' => 'content']] + ], + $query->toArray()['query']['bool']['filter'], 'Filter was not added to query.' ); } + /** + * @test + */ + public function emptyFilterIsNotAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter([ + 'field' => '', + 'field1' => 0, + 'field2' => false, + ]); + + $this->assertFalse( + $searchRequest->hasFilter(), + 'Search request contains filter even if it should not.' + ); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + null, + $query->toArray()['query']['bool']['filter'], + 'Filter was added to query, even if no filter exists.' + ); + } + /** * @test */ public function userInputIsAlwaysString() { - $connection = $this->getMockBuilder(Connection\Elasticsearch::class) - ->disableOriginalConstructor() - ->getMock(); $searchRequest = new SearchRequest(10); $searchRequest->setFilter(['field' => 20]); - $query = $this->subject->create($connection, $searchRequest); + $query = $this->subject->create($searchRequest); $this->assertSame( '10', $query->toArray()['query']['bool']['must'][0]['match']['_all'], @@ -95,8 +113,36 @@ class QueryFactoryTest extends AbstractUnitTestCase ); $this->assertSame( '20', - $query->toArray()['query']['bool']['filter']['term']['field'], + $query->toArray()['query']['bool']['filter'][0]['term']['field'], 'Search word was not escaped as expected.' ); } + + /** + * @test + */ + public function facetsAreAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); + $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'Identifier' => [ + 'terms' => [ + 'field' => 'FieldName', + ], + ], + 'Identifier 2' => [ + 'terms' => [ + 'field' => 'FieldName 2', + ], + ], + ], + $query->toArray()['aggs'], + 'Facets were not added to query.' + ); + } } From c6a199a7a47a6fefb7184eb4419fa311e37e481e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 14:13:26 +0200 Subject: [PATCH 018/158] BUGFIX: Fix broken functional test --- Classes/Connection/Elasticsearch/ResultItem.php | 4 ++-- Tests/Functional/Connection/Elasticsearch/FilterTest.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Connection/Elasticsearch/ResultItem.php b/Classes/Connection/Elasticsearch/ResultItem.php index dd8cb62..863e21b 100644 --- a/Classes/Connection/Elasticsearch/ResultItem.php +++ b/Classes/Connection/Elasticsearch/ResultItem.php @@ -44,12 +44,12 @@ class ResultItem implements ResultItemInterface return $this->data[$offset]; } - public function offsetSet() + public function offsetSet($offset, $value) { throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077); } - public function offsetUnset() + public function offsetUnset($offset) { throw new \BadMethodCallException('It\'s not possible to change the search result.', 1499179077); } diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index 88ae37f..3492e06 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -55,7 +55,7 @@ class FilterTest extends AbstractFunctionalTestCase $searchRequest->setFilter(['CType' => 'html']); $result = $searchService->search($searchRequest); - $this->assertSame('5', $result[0]->getData()['uid'], 'Did not get the expected result entry.'); + $this->assertSame('5', $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } } From 38f908711114c8fc550a969c6d31e14943a3ed5c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 14:13:39 +0200 Subject: [PATCH 019/158] TASK: Finish interface To stay compatible with implementation. --- Classes/Connection/SearchResultInterface.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index a6b2c90..74ac741 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -36,4 +36,20 @@ interface SearchResultInterface extends \Iterator, \Countable * @return array */ public function getFacets(); + + /** + * Returns the total sum of matching results. + * + * @return int + */ + public function getTotalCount(); + + // Countable - Interface + + /** + * Returns the total sum of results contained in this result. + * + * @return int + */ + public function count(); } From 3e2e889e277425b01577df6112e0a4e665391296 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 15:51:29 +0200 Subject: [PATCH 020/158] TASK: Add tests for new facets Add functional test to test facet configuration including: - Mapping (indexing) - Searching (retrieving of facets) - Filter (already existed, now based on facet case sensitive) --- .../AbstractFunctionalTestCase.php | 3 +- .../Connection/Elasticsearch/FilterTest.php | 35 ++++++++++++++++++- Tests/Functional/Fixtures/BasicSetup.ts | 14 ++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php index 363ad6d..1ecdd38 100644 --- a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php +++ b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php @@ -44,12 +44,13 @@ abstract class AbstractFunctionalTestCase extends BaseFunctionalTestCase 'port' => getenv('ES_PORT') ?: \Elastica\Connection::DEFAULT_PORT, ]); + // Start with clean system for test. $this->cleanUp(); } public function tearDown() { - // Delete everything so next test starts clean. + // Make system clean again. $this->cleanUp(); } diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index 3492e06..3a0ead8 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -53,9 +53,42 @@ class FilterTest extends AbstractFunctionalTestCase $result = $searchService->search($searchRequest); $this->assertSame(2, count($result), 'Did not receive both indexed elements without filter.'); - $searchRequest->setFilter(['CType' => 'html']); + $searchRequest->setFilter(['CType' => 'HTML']); $result = $searchService->search($searchRequest); $this->assertSame('5', $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } + + /** + * @test + */ + public function itsPossibleToFetchFacetsForField() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(SearchService::class); + + $searchRequest = new SearchRequest('Search Word'); + $result = $searchService->search($searchRequest); + + $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); + + $facet = $result->getFacets()[0]; + $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.'); + $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.'); + + $options = $facet->getOptions(); + $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.'); + $option = $options[0]; + $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.'); + $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); + $option = $options[1]; + $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.'); + $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); + } } diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 72163e3..094fd07 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -11,6 +11,20 @@ plugin { indexing { tt_content { indexer = Leonmrni\SearchCore\Domain\Index\TcaIndexer + + mapping { + CType { + type = keyword + } + } + } + } + + searching { + facets { + contentTypes { + field = CType + } } } } From dfde4b8f6b41f64ed7b5ed20b492ba2c12d46a3f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 16:17:30 +0200 Subject: [PATCH 021/158] TASK: Update docs Document new configuration options and adjust configuration where necessary. --- Documentation/source/concepts.rst | 4 +- Documentation/source/configuration.rst | 54 ++++++++++++++++++++++++-- Documentation/source/connections.rst | 4 ++ Documentation/source/development.rst | 9 +++-- Documentation/source/features.rst | 15 +++---- Documentation/source/installation.rst | 7 ++-- Documentation/source/readme.rst | 2 +- Documentation/source/usage.rst | 38 +++++++++++++----- ext_emconf.php | 6 +-- 9 files changed, 103 insertions(+), 36 deletions(-) diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index fc0569a..0ce3a0e 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -24,7 +24,7 @@ Currently only :ref:`Elasticsearch` is provided. Indexing -------- -The indexing is done by one of the available indexer. It should be possible to define the indexer to -use for certain document types. Also it should be possible to write custom indexer to use. +The indexing is done by one of the available indexer. For each identifier it's possible to define +the indexer to use. Also it's possible to write custom indexer to use. Currently only the :ref:`TcaIndexer` is provided. diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 1f16e06..3d8db75 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -96,8 +96,8 @@ The following settings are available. For each setting its documented which conn .. _configuration_options_index: -index -^^^^^ +Indexing +^^^^^^^^ Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. @@ -108,7 +108,7 @@ Configured as:: settings { indexing { identifier { - indexer = Fully Qualified Classname + indexer = FullyQualifiedClassname // the settings } } @@ -161,3 +161,51 @@ options are available: Make sure to prefix all fields with the corresponding table name. The selection from database will contain joins and can lead to SQL errors if a field exists in multiple tables. + +.. _mapping: + +``mapping`` +""""""""""" + + Used by: Elasticsearch connection while indexing. + + Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html + You are able to define the mapping for each property / columns. + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword + } + } + + The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This + makes building a facet possible. + + +.. _configuration_options_search: + +Searching +^^^^^^^^^ + +.. _facets: + +``facets`` +""""""""""" + + Used by: Elasticsearch connection while building search query. + + Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html + Currently only the term facet is provided. + + Example:: + + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType + } + } + + The above example will provide a facet with options for all found ``CType`` results together + with a count. diff --git a/Documentation/source/connections.rst b/Documentation/source/connections.rst index 678a92d..b51f280 100644 --- a/Documentation/source/connections.rst +++ b/Documentation/source/connections.rst @@ -22,5 +22,9 @@ The connection is configurable through the following options: * :ref:`port` +* :ref:`mapping` + +* :ref:`facets` + .. _elastic Elasticsearch: https://www.elastic.co/products/elasticsearch .. _elastica: http://elastica.io/ diff --git a/Documentation/source/development.rst b/Documentation/source/development.rst index 8026473..e73425c 100644 --- a/Documentation/source/development.rst +++ b/Documentation/source/development.rst @@ -32,9 +32,10 @@ Then setup your system:: git clone git@github.com:DanielSiepmann/search_core.git \ && cd search_core \ - && export typo3DatabaseName="searchcoretest62" \ - && export TYPO3_VERSION="~6.2" \ + && export typo3DatabaseName="searchcoretest76" \ + && export TYPO3_VERSION="~7.6" \ && make install \ + && make unitTests \ && make functionalTests If all tests are okay, start your work. @@ -42,8 +43,8 @@ If all tests are okay, start your work. If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and `TYPO3_VERSION` in your environment like:: - export typo3DatabaseName="searchcoretest76" - export TYPO3_VERSION="~7.6" + export typo3DatabaseName="searchcoretest62" + export TYPO3_VERSION="~6.2" Also run the install command for each version before running any tests. Only this will make sure you are testing against the actual TYPO3 Version and database scheme. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index c715512..669e632 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -13,7 +13,7 @@ Indexing Indexing data to Elasticsearch is provided. The extension delivers an indexer for TCA with zero configuration needs. Still it's possible to configure the indexer. -Own indexer are not possible yet, but will. +Also custom classes can be used as indexers. .. _features_search: @@ -24,6 +24,9 @@ Currently all fields are searched for a single search input. Also multiple filter are supported. Filtering results by fields for string contents. +Even facets / aggregates are now possible. Therefore a mapping has to be defined in TypoScript for +indexing, and the facets itself while searching. + .. _features_planned: Planned @@ -31,11 +34,5 @@ Planned The following features are currently planned and will be integrated: -#. Mapping Configuration - Allowing to configure the whole mapping, to define type of input, e.g. integer, keyword. - - -#. Facets / Aggregates - Based on the mapping configuration, facets will be configurable and fetched. Therefore mapping is - required and we will adjust the result set to be of a custom model providing all information in a - more clean way. +#. Pagination + Add a pagination to search results, to allow users to walk through all results. diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index 614b6e6..ddfb065 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -6,7 +6,7 @@ Installation The extension can be installed through composer:: - composer require "leonmrni/search_core dev-feature/integrate-elasticsearch" + composer require "leonmrni/search_core dev-master as 1.0.x-dev" or by `downloading`_ and placing it inside the :file:`typo3conf/ext`-Folder of your installation. In that case you need to install all dependencies yourself. Dependencies are: @@ -16,8 +16,7 @@ In that case you need to install all dependencies yourself. Dependencies are: :lines: 19-21 :dedent: 8 - Afterwards you need to enable the extension through the extension manager and include the static -typoscript setup. +TypoScript setup. -.. _downloading: https://github.com/DanielSiepmann/search_core/archive/feature/integrate-elasticsearch.zip +.. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index ec5e0e0..95350fa 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -18,7 +18,7 @@ reindexing. Current state ------------- -This is still a very early alpha version. More information can be taken from Github at +This is still a very early beta version. More information can be taken from Github at `current issues`_ and `current projects`_. We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_. diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 453b17d..1ca50d2 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -11,12 +11,12 @@ Manual indexing You can trigger indexing from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:index --table 'tt_content' + ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'tt_content' This will index the table ``tt_content`` using the :ref:`TcaIndexer`. -Only one table per call is available, to index multiple tables just make multiple calls. -The tables have to be white listed through :ref:`allowedTables` option. +Only one index per call is available, to run multiple indexers, just make multiple calls. +The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. .. _usage_auto_indexing: @@ -24,7 +24,7 @@ Auto indexing ------------- Indexing is done through hooks every time an record is changed. -The tables have to be white listed through :ref:`allowedTables` option. +The tables have to be configured via :ref:`configuration_options_index`. .. note:: @@ -57,10 +57,28 @@ Filter Thanks to extbase mapping, filter are added to the form: .. code-block:: html - :emphasize-lines: 3 - - - - - + + + +.. _usage_searching_facets: + +Facets +"""""" + +To add a facet as criteria for searching, use :ref:`usage_searching_filter`. + +To display facet results use: + +.. code-block:: html + + + + + + + diff --git a/ext_emconf.php b/ext_emconf.php index 3aab0c7..452a095 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -4,6 +4,7 @@ $EM_CONF[$_EXTKEY] = [ 'title' => 'Search Core', 'description' => 'Search core for implementing various search types.', 'category' => 'be', + 'clearCacheOnLoad' => 1, 'constraints' => [ 'depends' => [ 'typo3' => '7.6.0-7.6.99', @@ -16,9 +17,8 @@ $EM_CONF[$_EXTKEY] = [ 'Leonmrni\\SearchCore\\' => 'Classes', ], ], - 'state' => 'alpha', - 'clearCacheOnLoad' => 1, + 'state' => 'beta', + 'version' => '1.0.0', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', - 'version' => '1.0.0', ]; From c58e13cdf67fec8684f06103f1f2d89e381f8d90 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 6 Jul 2017 23:48:47 +0200 Subject: [PATCH 022/158] TASK: Rename vendor As we move it from private repo to codappix. --- Classes/Command/IndexCommandController.php | 6 +++--- Classes/Configuration/ConfigurationContainer.php | 2 +- .../ConfigurationContainerInterface.php | 2 +- Classes/Configuration/InvalidArgumentException.php | 2 +- Classes/Configuration/NoConfigurationException.php | 2 +- Classes/Connection/ConnectionInterface.php | 2 +- Classes/Connection/Elasticsearch.php | 6 +++--- Classes/Connection/Elasticsearch/Connection.php | 4 ++-- .../Connection/Elasticsearch/DocumentFactory.php | 2 +- Classes/Connection/Elasticsearch/Facet.php | 6 +++--- Classes/Connection/Elasticsearch/FacetOption.php | 4 ++-- Classes/Connection/Elasticsearch/IndexFactory.php | 2 +- .../Connection/Elasticsearch/MappingFactory.php | 6 +++--- Classes/Connection/Elasticsearch/ResultItem.php | 4 ++-- Classes/Connection/Elasticsearch/SearchResult.php | 8 ++++---- Classes/Connection/Elasticsearch/TypeFactory.php | 2 +- Classes/Connection/FacetInterface.php | 2 +- Classes/Connection/FacetOptionInterface.php | 2 +- Classes/Connection/FacetRequestInterface.php | 2 +- Classes/Connection/ResultItemInterface.php | 2 +- Classes/Connection/SearchRequestInterface.php | 2 +- Classes/Connection/SearchResultInterface.php | 2 +- Classes/Controller/SearchController.php | 6 +++--- Classes/Domain/Index/AbstractIndexer.php | 4 ++-- Classes/Domain/Index/IndexerFactory.php | 12 ++++++------ Classes/Domain/Index/IndexerInterface.php | 2 +- Classes/Domain/Index/IndexingException.php | 2 +- .../Domain/Index/NoMatchingIndexerException.php | 2 +- Classes/Domain/Index/NoRecordFoundException.php | 2 +- Classes/Domain/Index/TcaIndexer.php | 4 ++-- .../Index/TcaIndexer/InvalidArgumentException.php | 2 +- .../Domain/Index/TcaIndexer/RelationResolver.php | 2 +- .../Domain/Index/TcaIndexer/TcaTableService.php | 6 +++--- Classes/Domain/Model/FacetRequest.php | 4 ++-- Classes/Domain/Model/SearchRequest.php | 6 +++--- Classes/Domain/Search/QueryFactory.php | 8 ++++---- Classes/Domain/Search/SearchService.php | 14 +++++++------- Classes/Domain/Service/DataHandler.php | 12 ++++++------ Classes/Hook/DataHandler.php | 8 ++++---- Configuration/TypoScript/setup.txt | 2 +- Documentation/source/conf.py | 8 ++++---- Documentation/source/readme.rst | 10 +++++----- Tests/Functional/AbstractFunctionalTestCase.php | 2 +- .../Elasticsearch/AbstractFunctionalTestCase.php | 4 ++-- .../Connection/Elasticsearch/FilterTest.php | 8 ++++---- .../Connection/Elasticsearch/IndexTcaTableTest.php | 6 +++--- Tests/Functional/Fixtures/BasicSetup.ts | 2 +- .../Hooks/DataHandler/AbstractDataHandlerTest.php | 12 ++++++------ .../DataHandler/IgnoresUnkownOperationTest.php | 8 ++++---- .../Hooks/DataHandler/NonAllowedTablesTest.php | 8 ++++---- ...lowedTablesWithMultipleTablesConfiguredTest.php | 2 +- .../DataHandler/ProcessesAllowedTablesTest.php | 8 ++++---- ...lowedTablesWithMultipleTablesConfiguredTest.php | 2 +- .../Indexing/TcaIndexer/RelationResolverTest.php | 6 +++--- Tests/Functional/Indexing/TcaIndexerTest.php | 14 +++++++------- Tests/Unit/AbstractUnitTestCase.php | 2 +- Tests/Unit/Command/IndexCommandControllerTest.php | 12 ++++++------ .../Index/TcaIndexer/TcaTableServiceTest.php | 8 ++++---- Tests/Unit/Domain/Search/QueryFactoryTest.php | 10 +++++----- composer.json | 10 +++++----- ext_emconf.php | 2 +- ext_localconf.php | 12 ++++++------ ext_tables.php | 2 +- 63 files changed, 164 insertions(+), 164 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 227918f..b14d4b3 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Command; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; -use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; diff --git a/Classes/Configuration/ConfigurationContainer.php b/Classes/Configuration/ConfigurationContainer.php index db66a10..7481692 100644 --- a/Classes/Configuration/ConfigurationContainer.php +++ b/Classes/Configuration/ConfigurationContainer.php @@ -1,5 +1,5 @@ diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index d0d9828..9429535 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Configuration/InvalidArgumentException.php b/Classes/Configuration/InvalidArgumentException.php index fd9e048..1f5f8f0 100644 --- a/Classes/Configuration/InvalidArgumentException.php +++ b/Classes/Configuration/InvalidArgumentException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Configuration/NoConfigurationException.php b/Classes/Configuration/NoConfigurationException.php index bc6c1ba..db4e511 100644 --- a/Classes/Configuration/NoConfigurationException.php +++ b/Classes/Configuration/NoConfigurationException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index fdd09e3..a130f9e 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 88b93c2..f6321bc 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Connection; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\Elasticsearch\SearchResult; -use Leonmrni\SearchCore\Domain\Search\QueryFactory; +use Codappix\SearchCore\Connection\Elasticsearch\SearchResult; +use Codappix\SearchCore\Domain\Search\QueryFactory; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index 25c7361..cd0a1fd 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use TYPO3\CMS\Core\SingletonInterface as Singleton; /** diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 43462fe..99d29e5 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index 79a87b3..7893f5f 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Connection\FacetInterface; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\FacetInterface; class Facet implements FacetInterface { diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php index a618be1..7359434 100644 --- a/Classes/Connection/Elasticsearch/FacetOption.php +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\FacetOptionInterface; +use Codappix\SearchCore\Connection\FacetOptionInterface; class FacetOption implements FacetOptionInterface { diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 0a06ad4..019b091 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/Elasticsearch/MappingFactory.php b/Classes/Connection/Elasticsearch/MappingFactory.php index f029cd1..d4ac037 100644 --- a/Classes/Connection/Elasticsearch/MappingFactory.php +++ b/Classes/Connection/Elasticsearch/MappingFactory.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Configuration\InvalidArgumentException; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use TYPO3\CMS\Core\SingletonInterface as Singleton; /** diff --git a/Classes/Connection/Elasticsearch/ResultItem.php b/Classes/Connection/Elasticsearch/ResultItem.php index 863e21b..e56783c 100644 --- a/Classes/Connection/Elasticsearch/ResultItem.php +++ b/Classes/Connection/Elasticsearch/ResultItem.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\ResultItemInterface; class ResultItem implements ResultItemInterface { diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 63c93b0..f8ffdd8 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\FacetInterface; -use Leonmrni\SearchCore\Connection\ResultItemInterface; -use Leonmrni\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Connection\FacetInterface; +use Codappix\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; class SearchResult implements SearchResultInterface diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index b529937..d5283f8 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/FacetInterface.php b/Classes/Connection/FacetInterface.php index f422e98..b1cc421 100644 --- a/Classes/Connection/FacetInterface.php +++ b/Classes/Connection/FacetInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php index b05445b..51a1efd 100644 --- a/Classes/Connection/FacetOptionInterface.php +++ b/Classes/Connection/FacetOptionInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/FacetRequestInterface.php b/Classes/Connection/FacetRequestInterface.php index e5a60a1..9352f96 100644 --- a/Classes/Connection/FacetRequestInterface.php +++ b/Classes/Connection/FacetRequestInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/ResultItemInterface.php b/Classes/Connection/ResultItemInterface.php index f4c9185..56add2a 100644 --- a/Classes/Connection/ResultItemInterface.php +++ b/Classes/Connection/ResultItemInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index 603c02f..ee779f7 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index 74ac741..5c45bcd 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index b7624f7..068f9a1 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Controller; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Model\SearchRequest; -use Leonmrni\SearchCore\Domain\Search\SearchService; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Mvc\Controller\ActionController; /** diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index acd046e..b780dc5 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Domain\Index; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\ConnectionInterface; abstract class AbstractIndexer implements IndexerInterface { diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 24addd1..6618d01 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -1,5 +1,5 @@ @@ -20,11 +20,11 @@ namespace Leonmrni\SearchCore\Domain\Index; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Configuration\InvalidArgumentException; -use Leonmrni\SearchCore\Domain\Index\IndexerInterface; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; +use Codappix\SearchCore\Domain\Index\IndexerInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index d70b410..5fef64f 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/IndexingException.php b/Classes/Domain/Index/IndexingException.php index a22660d..d8ca71e 100644 --- a/Classes/Domain/Index/IndexingException.php +++ b/Classes/Domain/Index/IndexingException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/NoMatchingIndexerException.php b/Classes/Domain/Index/NoMatchingIndexerException.php index 02904cc..3f6c094 100644 --- a/Classes/Domain/Index/NoMatchingIndexerException.php +++ b/Classes/Domain/Index/NoMatchingIndexerException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/NoRecordFoundException.php b/Classes/Domain/Index/NoRecordFoundException.php index 8a92408..469359d 100644 --- a/Classes/Domain/Index/NoRecordFoundException.php +++ b/Classes/Domain/Index/NoRecordFoundException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 17dc61e..c29cf60 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Domain\Index; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\ConnectionInterface; /** * Will index the given table using configuration from TCA. diff --git a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php index 0f3dc9a..bc2036f 100644 --- a/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php +++ b/Classes/Domain/Index/TcaIndexer/InvalidArgumentException.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 69eb4a2..b09e483 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -1,5 +1,5 @@ diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index ae819bc..21e6374 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Index\IndexingException; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; diff --git a/Classes/Domain/Model/FacetRequest.php b/Classes/Domain/Model/FacetRequest.php index 5b8bea6..b1dbc4c 100644 --- a/Classes/Domain/Model/FacetRequest.php +++ b/Classes/Domain/Model/FacetRequest.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Domain\Model; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\FacetRequestInterface; +use Codappix\SearchCore\Connection\FacetRequestInterface; class FacetRequest implements FacetRequestInterface { diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 65ab979..163f65a 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Domain\Model; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\FacetRequestInterface; -use Leonmrni\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Connection\FacetRequestInterface; +use Codappix\SearchCore\Connection\SearchRequestInterface; /** * Represents a search request used to process an actual search. diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 70f42ff..7809fb6 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Domain\Search; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Connection\ConnectionInterface; -use Leonmrni\SearchCore\Connection\Elasticsearch\Query; -use Leonmrni\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\Elasticsearch\Query; +use Codappix\SearchCore\Connection\SearchRequestInterface; use TYPO3\CMS\Extbase\Utility\ArrayUtility; class QueryFactory diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index ac991cd..158d9d1 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -1,5 +1,5 @@ @@ -20,12 +20,12 @@ namespace Leonmrni\SearchCore\Domain\Search; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Configuration\InvalidArgumentException; -use Leonmrni\SearchCore\Connection\ConnectionInterface; -use Leonmrni\SearchCore\Connection\SearchRequestInterface; -use Leonmrni\SearchCore\Connection\SearchResultInterface; -use Leonmrni\SearchCore\Domain\Model\FacetRequest; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\FacetRequest; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index bec9eb3..6ac8069 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -1,5 +1,5 @@ @@ -20,10 +20,10 @@ namespace Leonmrni\SearchCore\Domain\Service; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; -use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; +use Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -43,7 +43,7 @@ class DataHandler implements Singleton /** * TODO: Only inject on first use?! * - * @var \Leonmrni\SearchCore\Connection\ConnectionInterface + * @var \Codappix\SearchCore\Connection\ConnectionInterface * @inject */ protected $connection; diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 34cf3f3..d0eb1ba 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Hook; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\NoConfigurationException; -use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; -use Leonmrni\SearchCore\Domain\Service\DataHandler as OwnDataHandler; +use Codappix\SearchCore\Configuration\NoConfigurationException; +use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; +use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; use TYPO3\CMS\Core\Log\LogManager; diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 2d58b35..0efcf4d 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -10,7 +10,7 @@ plugin { indexing { tt_content { - indexer = Leonmrni\SearchCore\Domain\Index\TcaIndexer + indexer = Codappix\SearchCore\Domain\Index\TcaIndexer additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause} } } diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index 07129e7..a708209 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -120,7 +120,7 @@ html_theme = 'alabaster' # documentation. html_theme_options = { 'description': 'TYPO3 Extension to integrate search services.', - 'github_user': 'DanielSiepmann', + 'github_user': 'Codappix', 'github_repo': 'search_core', 'github_button': True, 'github_banner': True, @@ -306,7 +306,7 @@ intersphinx_mapping = { 't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None), } extlinks = { - 'project': ('https://github.com/DanielSiepmann/search_core/projects/%s', 'Github project: '), - 'pr': ('https://github.com/DanielSiepmann/search_core/pull/%s', 'Github pull request: '), - 'issue': ('https://github.com/DanielSiepmann/search_core/issues/%s', 'Github issue: '), + 'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '), + 'pr': ('https://github.com/Codappix/search_core/pull/%s', 'Github pull request: '), + 'issue': ('https://github.com/Codappix/search_core/issues/%s', 'Github issue: '), } diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index 95350fa..ba77ae8 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -23,9 +23,9 @@ This is still a very early beta version. More information can be taken from Gith We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_. -.. _current issues: https://github.com/DanielSiepmann/search_core/issues -.. _current projects: https://github.com/DanielSiepmann/search_core/projects -.. _travis ci: https://travis-ci.org/DanielSiepmann/search_core -.. _scrutinizer: https://scrutinizer-ci.com/g/DanielSiepmann/search_core/inspections -.. _codacy: https://www.codacy.com/app/daniel-siepmann/search_core/dashboard +.. _current issues: https://github.com/Codappix/search_core/issues +.. _current projects: https://github.com/Codappix/search_core/projects +.. _travis ci: https://travis-ci.org/Codappix/search_core +.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections +.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index c231784..7f808c7 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -1,5 +1,5 @@ diff --git a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php index 1ecdd38..92f9db4 100644 --- a/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php +++ b/Tests/Functional/Connection/Elasticsearch/AbstractFunctionalTestCase.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase as BaseFunctionalTestCase; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase as BaseFunctionalTestCase; /** * All functional tests should extend this base class. diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index 3a0ead8..aa5a5dc 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; -use Leonmrni\SearchCore\Domain\Model\SearchRequest; -use Leonmrni\SearchCore\Domain\Search\SearchService; +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Object\ObjectManager; class FilterTest extends AbstractFunctionalTestCase diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 5e0c1a7..9678d83 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -1,5 +1,5 @@ @@ -20,7 +20,7 @@ namespace Leonmrni\SearchCore\Tests\Functional\Connection\Elasticsearch; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\IndexerFactory; use TYPO3\CMS\Extbase\Object\ObjectManager; /** @@ -61,7 +61,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase /** * @test - * @expectedException \Leonmrni\SearchCore\Domain\Index\IndexingException + * @expectedException \Codappix\SearchCore\Domain\Index\IndexingException */ public function indexingNonConfiguredTableWillThrowException() { diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 094fd07..e24aee8 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -10,7 +10,7 @@ plugin { indexing { tt_content { - indexer = Leonmrni\SearchCore\Domain\Index\TcaIndexer + indexer = Codappix\SearchCore\Domain\Index\TcaIndexer mapping { CType { diff --git a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php index b3c384b..40b64ea 100644 --- a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php +++ b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php @@ -1,5 +1,5 @@ @@ -20,11 +20,11 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php b/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php index 44f5186..b1676e3 100644 --- a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php +++ b/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index 6073633..c33701d 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php index 0bdf563..ebbee99 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesWithMultipleTablesConfiguredTest.php @@ -1,5 +1,5 @@ diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index e9e834e..715fe29 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Functional\Hooks\DataHandler; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Leonmrni\SearchCore\Hook\DataHandler as DataHandlerHook; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; +use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php index f743bdb..c0b0aa3 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesWithMultipleTablesConfiguredTest.php @@ -1,5 +1,5 @@ diff --git a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php b/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php index 360cacd..5577ac0 100644 --- a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php +++ b/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php @@ -1,5 +1,5 @@ @@ -20,8 +20,8 @@ namespace Leonmrni\SearchCore\Tests\Indexing\TcaIndexer; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Object\ObjectManager; diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 056af39..2b3f817 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -1,5 +1,5 @@ @@ -20,12 +20,12 @@ namespace Leonmrni\SearchCore\Tests\Indexing; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Connection\Elasticsearch; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\RelationResolver; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use Leonmrni\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\Elasticsearch; +use Codappix\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Extbase\Object\ObjectManager; class TcaIndexerTest extends AbstractFunctionalTestCase diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index 88496a1..c11e74e 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -1,5 +1,5 @@ diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php index 3dff968..4c81672 100644 --- a/Tests/Unit/Command/IndexCommandControllerTest.php +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -1,5 +1,5 @@ @@ -20,11 +20,11 @@ namespace Leonmrni\SearchCore\Tests\Unit\Command; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Command\IndexCommandController; -use Leonmrni\SearchCore\Domain\Index\IndexerFactory; -use Leonmrni\SearchCore\Domain\Index\NoMatchingIndexerException; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer; -use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; +use Codappix\SearchCore\Command\IndexCommandController; +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; +use Codappix\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; use TYPO3\CMS\Extbase\Mvc\Exception\StopActionException; diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 837d68c..9e88d4b 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -1,5 +1,5 @@ @@ -20,9 +20,9 @@ namespace Leonmrni\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Configuration\ConfigurationContainerInterface; -use Leonmrni\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; class TcaTableServiceTest extends AbstractUnitTestCase { diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 725754f..32c8fbd 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -1,5 +1,5 @@ @@ -20,10 +20,10 @@ namespace Leonmrni\SearchCore\Tests\Unit\Domain\Search; * 02110-1301, USA. */ -use Leonmrni\SearchCore\Domain\Model\FacetRequest; -use Leonmrni\SearchCore\Domain\Model\SearchRequest; -use Leonmrni\SearchCore\Domain\Search\QueryFactory; -use Leonmrni\SearchCore\Tests\Unit\AbstractUnitTestCase; +use Codappix\SearchCore\Domain\Model\FacetRequest; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\QueryFactory; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; class QueryFactoryTest extends AbstractUnitTestCase { diff --git a/composer.json b/composer.json index debb71f..61aedce 100644 --- a/composer.json +++ b/composer.json @@ -1,17 +1,17 @@ { - "name": "leonmrni/search_core", + "name": "codappix/search_core", "type": "typo3-cms-extension", - "description": "Leonmrni Search Core.", - "homepage": "http://www.leonmrni.com", + "description": "Codappix Search Core.", + "homepage": "https://github.com/Codappix/search_core", "license": ["GPL-2.0+"], "autoload": { "psr-4": { - "Leonmrni\\SearchCore\\": "Classes" + "Codappix\\SearchCore\\": "Classes" } }, "autoload-dev": { "psr-4": { - "Leonmrni\\SearchCore\\Tests\\": "Tests/", + "Codappix\\SearchCore\\Tests\\": "Tests/", "TYPO3\\CMS\\Core\\Tests\\": ".Build/vendor/typo3/cms/typo3/sysext/core/Tests/" } }, diff --git a/ext_emconf.php b/ext_emconf.php index 452a095..118e2df 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -14,7 +14,7 @@ $EM_CONF[$_EXTKEY] = [ ], 'autoload' => [ 'psr-4' => [ - 'Leonmrni\\SearchCore\\' => 'Classes', + 'Codappix\\SearchCore\\' => 'Classes', ], ], 'state' => 'beta', diff --git a/ext_localconf.php b/ext_localconf.php index 7495d73..1c1f9cd 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -11,15 +11,15 @@ call_user_func( 'SC_OPTIONS' => [ 'extbase' => [ 'commandControllers' => [ - Leonmrni\SearchCore\Command\IndexCommandController::class, + Codappix\SearchCore\Command\IndexCommandController::class, ], ], 't3lib/class.t3lib_tcemain.php' => [ 'processCmdmapClass' => [ - $extensionKey => '&' . \Leonmrni\SearchCore\Hook\DataHandler::class, + $extensionKey => '&' . \Codappix\SearchCore\Hook\DataHandler::class, ], 'processDatamapClass' => [ - $extensionKey => '&' . \Leonmrni\SearchCore\Hook\DataHandler::class, + $extensionKey => '&' . \Codappix\SearchCore\Hook\DataHandler::class, ], ], ], @@ -27,7 +27,7 @@ call_user_func( ); TYPO3\CMS\Extbase\Utility\ExtensionUtility::configurePlugin( - 'Leonmrni.' . $extensionKey, + 'Codappix.' . $extensionKey, 'search', [ 'Search' => 'search' @@ -39,8 +39,8 @@ call_user_func( \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Extbase\Object\Container\Container') ->registerImplementation( - 'Leonmrni\SearchCore\Connection\ConnectionInterface', - 'Leonmrni\SearchCore\Connection\Elasticsearch' + 'Codappix\SearchCore\Connection\ConnectionInterface', + 'Codappix\SearchCore\Connection\Elasticsearch' ); }, $_EXTKEY diff --git a/ext_tables.php b/ext_tables.php index d6d07ed..1a54787 100644 --- a/ext_tables.php +++ b/ext_tables.php @@ -7,7 +7,7 @@ ); TYPO3\CMS\Extbase\Utility\ExtensionUtility::registerPlugin( - 'Leonmrni.' . $_EXTKEY, + 'Codappix.' . $_EXTKEY, 'search', 'Search Core' ); From 705e3be85aac85e77e0f3564f7ea28d8ae25af21 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 7 Jul 2017 12:03:06 +0200 Subject: [PATCH 023/158] TASK: Migrate dev dependencies As testing framework is used, we can prefer dist again to speed up composer installation. --- Makefile | 4 ++-- Tests/Unit/UnitTests.xml | 2 +- composer.json | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e8177b9..87bcdfc 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ current_dir := $(dir $(mkfile_path)) TYPO3_WEB_DIR := $(current_dir).Build/web TYPO3_PATH_ROOT := $(current_dir).Build/web # Allow different versions on travis -TYPO3_VERSION ?= ~8.2 +TYPO3_VERSION ?= ~8.7 typo3DatabaseName ?= "searchcore_test2" typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" @@ -12,7 +12,7 @@ typo3DatabaseHost ?= "127.0.0.1" .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-source --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" git checkout composer.json functionalTests: diff --git a/Tests/Unit/UnitTests.xml b/Tests/Unit/UnitTests.xml index 6456405..6486594 100644 --- a/Tests/Unit/UnitTests.xml +++ b/Tests/Unit/UnitTests.xml @@ -1,7 +1,7 @@ Date: Fri, 7 Jul 2017 12:19:35 +0200 Subject: [PATCH 024/158] BUGFIX: Allow tests to run without database connection Ad TYPO3 Core now makes use of Doctrine, a connection is required to build system where. Therefore we move it to an own method to exchange the execution inside of tests. --- .../Index/TcaIndexer/TcaTableService.php | 26 ++++++++++++------- .../Index/TcaIndexer/TcaTableServiceTest.php | 6 +++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 21e6374..65adf70 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -142,15 +142,7 @@ class TcaTableService */ public function getWhereClause() { - $whereClause = '1=1' - . BackendUtility::BEenableFields($this->tableName) - . BackendUtility::deleteClause($this->tableName) - - . BackendUtility::BEenableFields('pages') - . BackendUtility::deleteClause('pages') - . ' AND pages.no_search = 0' - ; - + $whereClause = $this->getSystemWhereClause(); $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); if (is_string($userDefinedWhere)) { $whereClause .= ' AND ' . $userDefinedWhere; @@ -169,6 +161,22 @@ class TcaTableService return $whereClause; } + /** + * Generate SQL for TYPO3 as a system, to make sure only available records + * are fetched. + */ + public function getSystemWhereClause() : string + { + return '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + + . BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + . ' AND pages.no_search = 0' + ; + } + /** * @return string */ diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 9e88d4b..330c16e 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -60,6 +60,9 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) ->will($this->onConsecutiveCalls(null, false)); + $this->subject->expects($this->once()) + ->method('getSystemWhereClause') + ->will($this->returnValue('1=1 AND pages.no_search = 0')); $this->assertSame( '1=1 AND pages.no_search = 0', @@ -76,6 +79,9 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) ->will($this->onConsecutiveCalls('table.field = "someValue"', false)); + $this->subject->expects($this->once()) + ->method('getSystemWhereClause') + ->will($this->returnValue('1=1 AND pages.no_search = 0')); $this->assertSame( '1=1 AND pages.no_search = 0 AND table.field = "someValue"', From cf902dde83f64aac4a3326810012d38f69f5f622 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 7 Jul 2017 14:44:32 +0200 Subject: [PATCH 025/158] TASK: Make extension more compatible Migrate sql to doctrine. Migrate relation resolver to use new API. --- Classes/Database/Doctrine/Join.php | 50 ++++++++++ Classes/Database/Doctrine/Where.php | 53 +++++++++++ Classes/Domain/Index/TcaIndexer.php | 29 ++++-- .../Index/TcaIndexer/RelationResolver.php | 94 ++++--------------- .../Index/TcaIndexer/TcaTableService.php | 90 +++++++++--------- Makefile | 1 - .../Connection/Elasticsearch/FilterTest.php | 2 +- .../Elasticsearch/IndexTcaTableTest.php | 2 +- .../Index/TcaIndexer/TcaTableServiceTest.php | 14 ++- 9 files changed, 202 insertions(+), 133 deletions(-) create mode 100644 Classes/Database/Doctrine/Join.php create mode 100644 Classes/Database/Doctrine/Where.php diff --git a/Classes/Database/Doctrine/Join.php b/Classes/Database/Doctrine/Join.php new file mode 100644 index 0000000..df1a8c6 --- /dev/null +++ b/Classes/Database/Doctrine/Join.php @@ -0,0 +1,50 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class Join +{ + /** + * @var string + */ + protected $table = ''; + + /** + * @var string + */ + protected $condition = ''; + + public function __construct(string $table, string $condition) + { + $this->table = $table; + $this->condition = $condition; + } + + public function getTable() : string + { + return $this->table; + } + + public function getCondition() : string + { + return $this->condition; + } +} diff --git a/Classes/Database/Doctrine/Where.php b/Classes/Database/Doctrine/Where.php new file mode 100644 index 0000000..3398991 --- /dev/null +++ b/Classes/Database/Doctrine/Where.php @@ -0,0 +1,53 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * + */ +class Where +{ + /** + * @var string + */ + protected $statement = ''; + + /** + * @var array + */ + protected $parameters = []; + + public function __construct(string $statement, array $parameters) + { + $this->statement = $statement; + $this->parameters = $parameters; + } + + public function getStatement() : string + { + return $this->statement; + } + + public function getParameters() : array + { + return $this->parameters; + } +} diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index c0a77a1..18444ce 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Domain\Index; */ use Codappix\SearchCore\Connection\ConnectionInterface; +use TYPO3\CMS\Core\Database\ConnectionPool; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Will index the given table using configuration from TCA. @@ -51,14 +53,22 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecords($offset, $limit) { - $records = $this->getDatabaseConnection()->exec_SELECTgetRows( - $this->tcaTableService->getFields(), - $this->tcaTableService->getTableClause(), - $this->tcaTableService->getWhereClause(), - '', - '', - (int) $offset . ',' . (int) $limit - ); + $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->tcaTableService->getTableName()); + $where = $this->tcaTableService->getWhereClause(); + $query = $queryBuilder->select(... $this->tcaTableService->getFields()) + ->from($this->tcaTableService->getTableClause()) + ->where($where->getStatement()) + ->setParameters($where->getParameters()) + ->setFirstResult($offset) + ->setMaxResults($limit); + + foreach ($this->tcaTableService->getJoins() as $join) { + $query->from($join->getTable()); + $query->andWhere($join->getCondition()); + } + + $records = $query->execute()->fetchAll(); + if ($records === null) { return null; } @@ -106,7 +116,6 @@ class TcaIndexer extends AbstractIndexer protected function getDatabaseConnection() { - return GeneralUtility::makeInstance(ConnectionPool::class) - ->getConnectionByName('Default'); + return GeneralUtility::makeInstance(ConnectionPool::class); } } diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index b09e483..93cfffb 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -42,38 +42,31 @@ class RelationResolver implements Singleton * @param TcaTableService $service * @param array $record */ - public function resolveRelationsForRecord(TcaTableService $service, array &$record) + public function resolveRelationsForRecord(TcaTableService $service, array &$record) : void { - $formData = GeneralUtility::makeInstance( - FormDataCompiler::class, - GeneralUtility::makeInstance(TcaDatabaseRecord::class) - )->compile([ - 'tableName' => $service->getTableName(), - 'vanillaUid' => (int)$record['uid'], - 'command' => 'edit', - ]); - $record = $formData['databaseRow']; - foreach (array_keys($record) as $column) { + // TODO: Define / configure fields to exclude?! + if ($column === 'pid') { + continue; + } + $record[$column] = BackendUtility::getProcessedValueExtra($service->getTableName(), $column, $record[$column], 0, $record['uid']); + try { $config = $service->getColumnConfig($column); + + if ($this->isRelation($config)) { + $record[$column] = $this->resolveValue($record[$column], $config); + } } catch (InvalidArgumentException $e) { // Column is not configured. - continue; } - - if (! $this->isRelation($config) || !is_array($formData['processedTca']['columns'][$column])) { - continue; - } - - $record[$column] = $this->resolveValue($record[$column], $formData['processedTca']['columns'][$column]); } } /** * Resolve the given value from TYPO3 API response. * - * @param string $value The value from FormEngine to resolve. + * @param mixed $value The value from FormEngine to resolve. * @param array $tcaColumn The tca config of the relation. * * @return array|string @@ -83,15 +76,10 @@ class RelationResolver implements Singleton if ($value === '' || $value === '0') { return ''; } - if ($tcaColumn['config']['type'] === 'select') { - return $this->resolveSelectValue($value, $tcaColumn); - } - if ($tcaColumn['config']['type'] === 'group' && strpos($value, '|') !== false) { + + if ($tcaColumn['type'] === 'select' || $tcaColumn['type'] === 'group') { return $this->resolveForeignDbValue($value); } - if ($tcaColumn['config']['type'] === 'inline') { - return $this->resolveInlineValue($tcaColumn); - } return ''; } @@ -100,66 +88,24 @@ class RelationResolver implements Singleton * @param array Column config. * @return bool */ - protected function isRelation(array &$config) + protected function isRelation(array &$config) : bool { return isset($config['foreign_table']) - || (isset($config['items']) && is_array($config['items'])) + || (isset($config['renderType']) && $config['renderType'] !== 'selectSingle') || (isset($config['internal_type']) && strtolower($config['internal_type']) === 'db') ; } - /** - * Resolves internal representation of select to array of labels. - * - * @param array $value - * @param array $tcaColumn - * @return array - */ - protected function resolveSelectValue(array $values, array $tcaColumn) - { - $resolvedValues = []; - - foreach ($tcaColumn['config']['items'] as $item) { - if (in_array($item[1], $values)) { - $resolvedValues[] = $item[0]; - } - } - - if ($tcaColumn['config']['renderType'] === 'selectSingle' || $tcaColumn['config']['maxitems'] === 1) { - return current($resolvedValues); - } - - return $resolvedValues; - } - /** * @param string $value * * @return array */ - protected function resolveForeignDbValue($value) + protected function resolveForeignDbValue(string $value) : array { - $titles = []; - - foreach (explode(',', urldecode($value)) as $title) { - $titles[] = explode('|', $title)[1]; + if ($value === 'N/A') { + return []; } - - return $titles; - } - - /** - * @param array $tcaColumn - * @return array - */ - protected function resolveInlineValue(array $tcaColumn) - { - $titles = []; - - foreach ($tcaColumn['children'] as $selected) { - $titles[] = $selected['recordTitle']; - } - - return $titles; + return array_map('trim', explode(';', $value)); } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 65adf70..77b7578 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Database\Doctrine\Join; +use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -92,7 +94,7 @@ class TcaTableService /** * @return string */ - public function getTableName() + public function getTableName() : string { return $this->tableName; } @@ -100,9 +102,9 @@ class TcaTableService /** * @return string */ - public function getTableClause() + public function getTableClause() : string { - return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid'; + return $this->tableName; } /** @@ -111,7 +113,7 @@ class TcaTableService * @param array &$records * @return void */ - public function filterRecordsByRootLineBlacklist(array &$records) + public function filterRecordsByRootLineBlacklist(array &$records) : void { $records = array_filter( $records, @@ -125,7 +127,7 @@ class TcaTableService * Adjust record accordingly to configuration. * @param array &$record */ - public function prepareRecord(array &$record) + public function prepareRecord(array &$record) : void { $this->relationResolver->resolveRelationsForRecord($this, $record); @@ -137,11 +139,9 @@ class TcaTableService } } - /** - * @return string - */ - public function getWhereClause() + public function getWhereClause() : Where { + $parameters = []; $whereClause = $this->getSystemWhereClause(); $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); if (is_string($userDefinedWhere)) { @@ -149,16 +149,41 @@ class TcaTableService } if ($this->isBlacklistedRootLineConfigured()) { - $whereClause .= ' AND pages.uid NOT IN (' - . implode(',', $this->getBlacklistedRootLine()) - . ')' - . ' AND pages.pid NOT IN (' - . implode(',', $this->getBlacklistedRootLine()) - . ')'; + $parameters[':blacklistedRootLine'] = $this->getBlacklistedRootLine(); + $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' + . ' AND pages.pid NOT IN (:blacklistedRootLine)'; } $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]); - return $whereClause; + return new Where($whereClause, $parameters); + } + + public function getFields() : array + { + $fields = array_merge( + ['uid','pid'], + array_filter( + array_keys($this->tca['columns']), + function ($columnName) { + return !$this->isSystemField($columnName); + } + ) + ); + + foreach ($fields as $key => $field) { + $fields[$key] = $this->tableName . '.' . $field; + } + + $this->logger->debug('Generated fields.', [$this->tableName, $fields]); + return $fields; + return implode(', ', $fields); + } + + public function getJoins() : array + { + return [ + new Join('pages', 'pages.uid = ' . $this->tableName . '.pid'), + ]; } /** @@ -177,34 +202,11 @@ class TcaTableService ; } - /** - * @return string - */ - public function getFields() - { - $fields = array_merge( - ['uid','pid'], - array_filter( - array_keys($this->tca['columns']), - function ($columnName) { - return !$this->isSystemField($columnName); - } - ) - ); - - foreach ($fields as $key => $field) { - $fields[$key] = $this->tableName . '.' . $field; - } - - $this->logger->debug('Generated fields.', [$this->tableName, $fields]); - return implode(',', $fields); - } - /** * @param string * @return bool */ - protected function isSystemField($columnName) + protected function isSystemField($columnName) : bool { $systemFields = [ // Versioning fields, @@ -228,7 +230,7 @@ class TcaTableService * @return array * @throws InvalidArgumentException */ - public function getColumnConfig($columnName) + public function getColumnConfig($columnName) : array { if (!isset($this->tca['columns'][$columnName])) { throw new InvalidArgumentException( @@ -250,7 +252,7 @@ class TcaTableService * @param array &$record * @return bool */ - protected function isRecordBlacklistedByRootline(array &$record) + protected function isRecordBlacklistedByRootline(array &$record) : bool { // If no rootline exists, the record is on a unreachable page and therefore blacklisted. $rootline = BackendUtility::BEgetRootLine($record['pid']); @@ -275,7 +277,7 @@ class TcaTableService * * @return bool */ - protected function isBlackListedRootLineConfigured() + protected function isBlackListedRootLineConfigured() : bool { return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); } @@ -285,7 +287,7 @@ class TcaTableService * * @return array */ - protected function getBlackListedRootLine() + protected function getBlackListedRootLine() : array { return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); } diff --git a/Makefile b/Makefile index 87bcdfc..b6bc72b 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,6 @@ functionalTests: typo3DatabaseHost=$(typo3DatabaseHost) \ TYPO3_PATH_WEB=$(TYPO3_WEB_DIR) \ .Build/bin/phpunit --colors --debug -v \ - --stop-on-error \ -c Tests/Functional/FunctionalTests.xml unitTests: diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index aa5a5dc..f0b8404 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -55,7 +55,7 @@ class FilterTest extends AbstractFunctionalTestCase $searchRequest->setFilter(['CType' => 'HTML']); $result = $searchService->search($searchRequest); - $this->assertSame('5', $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); + $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 9678d83..8a76c77 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -150,7 +150,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ['_source' => [ 'uid' => '9', 'CType' => 'Header', // Testing items - 'categories' => ['Category 1', 'Category 2'], // Testing mm (with sorting) + 'categories' => ['Category 2', 'Category 1'], // Testing mm ]], $response->getData()['hits']['hits'][0], false, diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 330c16e..58c03b4 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -64,9 +64,14 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); + $whereClause =$this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0', - $this->subject->getWhereClause() + $whereClause->getStatement() + ); + $this->assertSame( + [], + $whereClause->getParameters() ); } @@ -83,9 +88,14 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); + $whereClause = $this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0 AND table.field = "someValue"', - $this->subject->getWhereClause() + $whereClause->getStatement() + ); + $this->assertSame( + [], + $whereClause->getParameters() ); } } From d61a86f8fefd6f0eb36d9c3262d6f5fc534f7cab Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 7 Jul 2017 16:14:41 +0200 Subject: [PATCH 026/158] TASK: Finish TYPO3 CMS 8 update --- Classes/Database/Doctrine/Where.php | 3 --- Classes/Domain/Index/TcaIndexer/RelationResolver.php | 12 ++++++++++-- .../Hooks/DataHandler/AbstractDataHandlerTest.php | 3 +-- .../Hooks/DataHandler/ProcessesAllowedTablesTest.php | 11 ++++++----- .../Indexing/TcaIndexer/RelationResolverTest.php | 6 +----- ext_localconf.php | 4 ++-- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Classes/Database/Doctrine/Where.php b/Classes/Database/Doctrine/Where.php index 3398991..6586b8a 100644 --- a/Classes/Database/Doctrine/Where.php +++ b/Classes/Database/Doctrine/Where.php @@ -20,9 +20,6 @@ namespace Codappix\SearchCore\Database\Doctrine; * 02110-1301, USA. */ -/** - * - */ class Where { /** diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 93cfffb..bfc0cc4 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -73,13 +73,16 @@ class RelationResolver implements Singleton */ protected function resolveValue($value, array $tcaColumn) { - if ($value === '' || $value === '0') { + if ($value === '') { return ''; } - if ($tcaColumn['type'] === 'select' || $tcaColumn['type'] === 'group') { + if ($tcaColumn['type'] === 'select') { return $this->resolveForeignDbValue($value); } + if ($tcaColumn['type'] === 'inline' || $tcaColumn['type'] === 'group') { + return $this->resolveInlineValue($value); + } return ''; } @@ -108,4 +111,9 @@ class RelationResolver implements Singleton } return array_map('trim', explode(';', $value)); } + + protected function resolveInlineValue(string $value) : array + { + return array_map('trim', explode(',', $value)); + } } diff --git a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php index 40b64ea..0b0f128 100644 --- a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php +++ b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php @@ -50,7 +50,6 @@ abstract class AbstractDataHandlerTest extends AbstractFunctionalTestCase ->setMethods(['add', 'update', 'delete']) ->getMock(); - // This way TYPO3 will use our mock instead of a new instance. - $GLOBALS['T3_VAR']['getUserObj']['&' . DataHandlerHook::class] = new DataHandlerHook($this->subject); + GeneralUtility::setSingletonInstance(\Codappix\SearchCore\Hook\DataHandler::class, new DataHandlerHook($this->subject)); } } diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 715fe29..7fa1cc5 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -47,7 +47,8 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function deletionWillBeTriggeredForTtContent() { - $this->subject->expects($this->exactly(1))->method('delete') + $this->subject->expects($this->exactly(1)) + ->method('delete') ->with($this->equalTo('tt_content'), $this->equalTo('1')); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -71,9 +72,9 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '1' - && isset($record['pid']) && $record['pid'] === '1' - && isset($record['colPos']) && $record['colPos'] === '1' + return isset($record['uid']) && $record['uid'] === 1 + && isset($record['pid']) && $record['pid'] === 1 + && isset($record['colPos']) && $record['colPos'] === 1 ; }) ); @@ -99,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 2 + return isset($record['uid']) && $record['uid'] === '2' && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php b/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php index 5577ac0..a491677 100644 --- a/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php +++ b/Tests/Functional/Indexing/TcaIndexer/RelationResolverTest.php @@ -37,10 +37,6 @@ class RelationResolverTest extends AbstractFunctionalTestCase $objectManager = GeneralUtility::makeInstance(ObjectManager::class); $table = 'sys_file'; - // Only by adding the field to showitem, it will be processed by FormEngine. - // We use this field to test inline relations, as there is only one alternative. - $GLOBALS['TCA']['sys_file']['types'][1]['showitem'] .= ',metadata'; - $subject = $objectManager->get(TcaTableService::class, $table); $record = BackendUtility::getRecord($table, 1); $subject->prepareRecord($record); @@ -113,8 +109,8 @@ class RelationResolverTest extends AbstractFunctionalTestCase $this->assertEquals( [ - 'Category 1', 'Category 2', + 'Category 1', ], $record['categories'], 'Foreign mm select relation was not resolved as expected.' diff --git a/ext_localconf.php b/ext_localconf.php index 1c1f9cd..54bf43b 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -16,10 +16,10 @@ call_user_func( ], 't3lib/class.t3lib_tcemain.php' => [ 'processCmdmapClass' => [ - $extensionKey => '&' . \Codappix\SearchCore\Hook\DataHandler::class, + $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class, ], 'processDatamapClass' => [ - $extensionKey => '&' . \Codappix\SearchCore\Hook\DataHandler::class, + $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class, ], ], ], From a8fdb8a44d79beea2ba3f6af7d5dc4db982f1f6f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 7 Jul 2017 16:24:56 +0200 Subject: [PATCH 027/158] TASK: Update dependencies for CMS 8 --- ext_emconf.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext_emconf.php b/ext_emconf.php index 118e2df..4d342a5 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,8 +7,8 @@ $EM_CONF[$_EXTKEY] = [ 'clearCacheOnLoad' => 1, 'constraints' => [ 'depends' => [ - 'typo3' => '7.6.0-7.6.99', - 'php' => '5.6.0-7.99.99' + 'typo3' => '8.7.0-8.7.99', + 'php' => '7.1.0-7.99.99' ], 'conflicts' => [], ], From 2cd5debf977c8e1e3893cf645da9624ed73782ae Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 7 Jul 2017 16:44:57 +0200 Subject: [PATCH 028/158] BUGFIX: Fix broken getRecord method Also add test covering method. --- Classes/Domain/Index/TcaIndexer.php | 43 ++++++++++--------- .../Elasticsearch/IndexTcaTableTest.php | 23 ++++++++++ 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 18444ce..7923565 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Connection\ConnectionInterface; use TYPO3\CMS\Core\Database\ConnectionPool; +use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Utility\GeneralUtility; /** @@ -53,21 +54,11 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecords($offset, $limit) { - $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->tcaTableService->getTableName()); - $where = $this->tcaTableService->getWhereClause(); - $query = $queryBuilder->select(... $this->tcaTableService->getFields()) - ->from($this->tcaTableService->getTableClause()) - ->where($where->getStatement()) - ->setParameters($where->getParameters()) + $records = $this->getQuery() ->setFirstResult($offset) - ->setMaxResults($limit); - - foreach ($this->tcaTableService->getJoins() as $join) { - $query->from($join->getTable()); - $query->andWhere($join->getCondition()); - } - - $records = $query->execute()->fetchAll(); + ->setMaxResults($limit) + ->execute() + ->fetchAll(); if ($records === null) { return null; @@ -88,12 +79,7 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecord($identifier) { - $record = $this->getDatabaseConnection()->exec_SELECTgetSingleRow( - $this->tcaTableService->getFields(), - $this->tcaTableService->getTableClause(), - $this->tcaTableService->getWhereClause() - . ' AND ' . $this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier - ); + $record = $this->getQuery()->execute()->fetch(); if ($record === false || $record === null) { throw new NoRecordFoundException( @@ -114,6 +100,23 @@ class TcaIndexer extends AbstractIndexer return $this->tcaTableService->getTableName(); } + protected function getQuery() : QueryBuilder + { + $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->tcaTableService->getTableName()); + $where = $this->tcaTableService->getWhereClause(); + $query = $queryBuilder->select(... $this->tcaTableService->getFields()) + ->from($this->tcaTableService->getTableClause()) + ->where($where->getStatement()) + ->setParameters($where->getParameters()); + + foreach ($this->tcaTableService->getJoins() as $join) { + $query->from($join->getTable()); + $query->andWhere($join->getCondition()); + } + + return $query; + } + protected function getDatabaseConnection() { return GeneralUtility::makeInstance(ConnectionPool::class); diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 8a76c77..7210e01 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -59,6 +59,29 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase ); } + /** + * @test + */ + public function indexSingleBasicTtContent() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexDocument(6) + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + + $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); + $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.'); + $this->assertArraySubset( + ['_source' => ['header' => 'indexed content element']], + $response->getData()['hits']['hits'][0], + false, + 'Record was not indexed.' + ); + } + /** * @test * @expectedException \Codappix\SearchCore\Domain\Index\IndexingException From 9fff750e852343091fec047139b9fa2c2663ba5e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 7 Jul 2017 16:29:16 +0200 Subject: [PATCH 029/158] BUGFIX: Fix TypoScript warning Constants can't use multi line strings. --- Configuration/TypoScript/constants.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index f97c039..149d610 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -14,10 +14,7 @@ plugin { # should also be added, together with additionalWhereClause # based on doktypes tt_content { - additionalWhereClause ( - pages.doktype NOT IN (3, 199) - AND tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - ) + additionalWhereClause = pages.doktype NOT IN (3, 199) AND tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') } } } From b6ab05bac7053d4127c70abfccc4e5781b6b104b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 13 Jul 2017 12:51:36 +0200 Subject: [PATCH 030/158] FEATURE: Switch from ttcontent to pages Also provide search_abstract as new auto added field which is configurable. --- Classes/Domain/Index/AbstractIndexer.php | 54 ++++++++++- Classes/Domain/Index/IndexerFactory.php | 25 +++-- Classes/Domain/Index/IndexerInterface.php | 9 ++ Classes/Domain/Index/TcaIndexer.php | 6 +- .../Domain/Index/TcaIndexer/PagesIndexer.php | 93 +++++++++++++++++++ .../Index/TcaIndexer/TcaTableService.php | 13 ++- 6 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 Classes/Domain/Index/TcaIndexer/PagesIndexer.php diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index b780dc5..18e1702 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -20,7 +20,9 @@ namespace Codappix\SearchCore\Domain\Index; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; +use \TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface { @@ -29,6 +31,16 @@ abstract class AbstractIndexer implements IndexerInterface */ protected $connection; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var string + */ + protected $identifier; + /** * @var \TYPO3\CMS\Core\Log\Logger */ @@ -44,23 +56,34 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger = $logManager->getLogger(__CLASS__); } + public function setIdentifier($identifier) + { + $this->identifier = $identifier; + } + /** * @param ConnectionInterface $connection + * @param ConfigurationContainerInterface $configuration */ - public function __construct(ConnectionInterface $connection) + public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration) { $this->connection = $connection; + $this->configuration = $configuration; } public function indexAllDocuments() { $this->logger->info('Start indexing'); foreach ($this->getRecordGenerator() as $records) { - $this->logger->debug('Index records.', [$records]); if ($records === null) { break; } + foreach ($records as &$record) { + $this->prepareRecord($record); + } + + $this->logger->debug('Index records.', [$records]); $this->connection->addDocuments($this->getDocumentName(), $records); } $this->logger->info('Finish indexing'); @@ -70,7 +93,10 @@ abstract class AbstractIndexer implements IndexerInterface { $this->logger->info('Start indexing single record.', [$identifier]); try { - $this->connection->addDocument($this->getDocumentName(), $this->getRecord($identifier)); + $record = $this->getRecord($identifier); + $this->prepareRecord($record); + + $this->connection->addDocument($this->getDocumentName(), $record); } catch (NoRecordFoundException $e) { $this->logger->info('Could not index document.', [$e->getMessage()]); } @@ -92,6 +118,28 @@ abstract class AbstractIndexer implements IndexerInterface } } + /** + * @param array &$record + */ + protected function prepareRecord(array &$record) + { + $record['search_abstract'] = ''; + + $fieldsToUse = GeneralUtility::trimExplode( + ',', + $this->configuration->getIfExists('indexing.' . $this->identifier . '.abstractFields') + ); + if (!$fieldsToUse) { + return; + } + foreach ($fieldsToUse as $fieldToUse) { + if (isset($record[$fieldToUse]) && trim($record[$fieldToUse])) { + $record['search_abstract'] = trim($record[$fieldToUse]); + break; + } + } + } + /** * @param int $offset * @param int $limit diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 6618d01..dbae818 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -83,17 +83,30 @@ class IndexerFactory implements Singleton */ protected function buildIndexer($indexerClass, $identifier) { - if ($indexerClass === TcaIndexer::class) { - return $this->objectManager->get( - TcaIndexer::class, + $indexer = null; + if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class) + || $indexerClass === TcaIndexer\PagesIndexer::class + ) { + $indexer = $this->objectManager->get( + $indexerClass, + $this->objectManager->get(TcaTableService::class, $identifier), + $this->objectManager->get(TcaTableService::class, 'tt_content') + ); + } elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) { + $indexer = $this->objectManager->get( + $indexerClass, $this->objectManager->get(TcaTableService::class, $identifier) ); + } elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) { + $indexer = $this->objectManager->get($indexerClass); } - if (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) { - return $this->objectManager->get($indexerClass); + if ($indexer === null) { + throw new NoMatchingIndexerException('Could not find indexer: ' . $indexerClass, 1497341442); } - throw new NoMatchingIndexerException('Could not find indexer: ' . $indexerClass, 1497341442); + $indexer->setIdentifier($identifier); + + return $indexer; } } diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 5fef64f..5a4ca6c 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -40,4 +40,13 @@ interface IndexerInterface * @return void */ public function indexDocument($identifier); + + /** + * Recieves the identifier of the indexer itself. + * + * @param string $identifier + * + * @return void + */ + public function setIdentifier($identifier); } diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index c29cf60..44d7c46 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Index; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; /** @@ -35,13 +36,16 @@ class TcaIndexer extends AbstractIndexer /** * @param TcaIndexer\TcaTableService $tcaTableService * @param ConnectionInterface $connection + * @param ConfigurationContainerInterface $configuration */ public function __construct( TcaIndexer\TcaTableService $tcaTableService, - ConnectionInterface $connection + ConnectionInterface $connection, + ConfigurationContainerInterface $configuration ) { $this->tcaTableService = $tcaTableService; $this->connection = $connection; + $this->configuration = $configuration; } /** diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php new file mode 100644 index 0000000..d4ce0a6 --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -0,0 +1,93 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer; + +/** + * Specific indexer for Pages, will basically add content of page. + */ +class PagesIndexer extends TcaIndexer +{ + /** + * @var TcaTableService + */ + protected $contentTableService; + + /** + * @param TcaTableService $tcaTableService + * @param TcaTableService $tcaTableService + * @param ConnectionInterface $connection + * @param ConfigurationContainerInterface $configuration + */ + public function __construct( + TcaTableService $tcaTableService, + TcaTableService $contentTableService, + ConnectionInterface $connection, + ConfigurationContainerInterface $configuration + ) { + $this->tcaTableService = $tcaTableService; + $this->contentTableService = $contentTableService; + $this->connection = $connection; + $this->configuration = $configuration; + } + + /** + * @param array &$record + */ + protected function prepareRecord(array &$record) + { + parent::prepareRecord($record); + $record['content'] = $this->fetchContentForPage($record['uid']); + } + + /** + * @param int $uid + * @return string + */ + protected function fetchContentForPage($uid) + { + $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( + $this->contentTableService->getFields(), + $this->contentTableService->getTableClause(), + $this->contentTableService->getWhereClause() . + sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid) + ); + + if ($contentElements === null) { + $this->logger->debug('No content for page ' . $uid); + return ''; + } + + $this->logger->debug('Fetched content for page ' . $uid); + $content = []; + foreach ($contentElements as $contentElement) { + $content[] = $contentElement['bodytext']; + } + + // Remove Tags. + // Interpret escaped new lines and special chars. + // Trim, e.g. trailing or leading new lines. + return trim(stripcslashes(strip_tags(implode(' ', $content)))); + } +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 21e6374..b5f48ab 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -102,6 +102,10 @@ class TcaTableService */ public function getTableClause() { + if ($this->tableName === 'pages') { + return $this->tableName; + } + return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid'; } @@ -145,12 +149,15 @@ class TcaTableService $whereClause = '1=1' . BackendUtility::BEenableFields($this->tableName) . BackendUtility::deleteClause($this->tableName) - - . BackendUtility::BEenableFields('pages') - . BackendUtility::deleteClause('pages') . ' AND pages.no_search = 0' ; + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } + $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); if (is_string($userDefinedWhere)) { $whereClause .= ' AND ' . $userDefinedWhere; From d36d8e859403bd88ec227544adb807726db5f5ca Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 13 Jul 2017 13:58:16 +0200 Subject: [PATCH 031/158] TASK: Add documentation and config for new feature --- Configuration/TypoScript/constants.txt | 12 +++--------- Configuration/TypoScript/setup.txt | 7 ++++--- Documentation/source/configuration.rst | 24 ++++++++++++++++++++++-- Documentation/source/indexer.rst | 24 +++++++++++++++++++++++- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index f97c039..dff37e3 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -9,15 +9,9 @@ plugin { } indexing { - # Pages are not supported yet, see - # https://github.com/DanielSiepmann/search_core/issues/24 but - # should also be added, together with additionalWhereClause - # based on doktypes - tt_content { - additionalWhereClause ( - pages.doktype NOT IN (3, 199) - AND tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - ) + pages { + additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255, 199) + abstractFields = abstract, description, bodytext } } } diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 0efcf4d..e81368e 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -9,9 +9,10 @@ plugin { } indexing { - tt_content { - indexer = Codappix\SearchCore\Domain\Index\TcaIndexer - additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause} + pages { + indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer + additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} + abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} } } } diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 3d8db75..9db3c6e 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -125,7 +125,7 @@ The following settings are available. For each setting its documented which inde ``rootLineBlacklist`` """"""""""""""""""""" - Used by: :ref:`TcaIndexer`. + Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. Defines a blacklist of page uids. Records below any of these pages, or subpages, are not indexed. This allows you to define areas that should not be indexed. @@ -147,7 +147,7 @@ options are available: ``additionalWhereClause`` """"""""""""""""""""""""" - Used by: :ref:`TcaIndexer`. + Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. Add additional SQL to where clauses to determine indexable records from the table. This way you can exclude specific records like ``tt_content`` records with specific ``CType`` values or @@ -162,6 +162,26 @@ options are available: Make sure to prefix all fields with the corresponding table name. The selection from database will contain joins and can lead to SQL errors if a field exists in multiple tables. +.. _abstractFields: + +``abstractFields`` +""""""""""""""""""""""""" + + Used by: :ref:`PagesIndexer`. + + Define which field should be used to provide the auto generated field "search_abstract". + The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also + possible. + + Example:: + + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + + Default:: + + abstract, description, bodytext + .. _mapping: ``mapping`` diff --git a/Documentation/source/indexer.rst b/Documentation/source/indexer.rst index 01d7b18..ddc6772 100644 --- a/Documentation/source/indexer.rst +++ b/Documentation/source/indexer.rst @@ -27,7 +27,29 @@ The indexer is configurable through the following options: * :ref:`additionalWhereClause` +.. _PagesIndexer: + +PagesIndexer +------------ + +Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can +start indexing TYPO3. + +The indexer will use the TCA to fetch all necessary information like relations. Currently the +implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and +further stuff. Also all static content from each page will be concatenated into a single field to +improve search. + +The indexer is configurable through the following options: + +* :ref:`allowedTables` + +* :ref:`rootLineBlacklist` + +* :ref:`additionalWhereClause` + +* :ref:`abstractFields` + .. note:: Not all relations are resolved yet, see :issue:`17` and :pr:`20`. - Also the `pages`-Table is not available yet, see :issue:`24`. From 299ec3af5eedcb92ddb46528017256cd58e47ed6 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 13 Jul 2017 14:54:37 +0200 Subject: [PATCH 032/158] TASK: Add tests covering new pages indexing Including content and search abstract. --- Configuration/TypoScript/setup.txt | 9 +++ Tests/Functional/Fixtures/BasicSetup.ts | 16 +++++ Tests/Functional/Fixtures/BasicSetup.xml | 1 + .../Fixtures/Indexing/IndexTcaTable.xml | 39 +++++++++++ .../Functional/Indexing/PagesIndexerTest.php | 64 +++++++++++++++++++ Tests/Functional/Indexing/TcaIndexerTest.php | 16 ++--- 6 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 Tests/Functional/Indexing/PagesIndexerTest.php diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index e81368e..d77c42e 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -9,6 +9,15 @@ plugin { } indexing { + # Not for direct indexing therefore no indexer. + # Used to configure tt_content fetching while indexing pages + tt_content { + additionalWhereClause ( + tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') + AND tt_content.bodytext != '' + ) + } + pages { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.pages.additionalWhereClause} diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index e24aee8..1e2b3a9 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -12,6 +12,22 @@ plugin { tt_content { indexer = Codappix\SearchCore\Domain\Index\TcaIndexer + additionalWhereClause ( + tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') + AND tt_content.bodytext != '' + ) + + mapping { + CType { + type = keyword + } + } + } + + pages { + indexer = Codappix\SearchCore\Domain\Index\TcaIndexer\PagesIndexer + abstractFields = abstract, description, bodytext + mapping { CType { type = keyword diff --git a/Tests/Functional/Fixtures/BasicSetup.xml b/Tests/Functional/Fixtures/BasicSetup.xml index a85b72a..1a46f3b 100644 --- a/Tests/Functional/Fixtures/BasicSetup.xml +++ b/Tests/Functional/Fixtures/BasicSetup.xml @@ -4,5 +4,6 @@ 1 0 Root page containing necessary TypoScript + Used as abstract as no abstract is defined. diff --git a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml index 75a1f35..ec251ce 100644 --- a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml +++ b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml @@ -60,4 +60,43 @@ 0 + + 9 + 1 + 1480686370 + 1480686370 + 0 + 72 + list +
not indexed due to ctype
+ this is the content of header content element that should not get indexed + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + 10 + 1 + 1480686370 + 1480686370 + 0 + 72 + html +
Indexed without html tags
+ Some text in paragraph

]]> + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php new file mode 100644 index 0000000..d0440ba --- /dev/null +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -0,0 +1,64 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\Elasticsearch; +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class PagesIndexerTest extends AbstractFunctionalTestCase +{ + /** + * @test + */ + public function pagesContainAllAdditionalInformation() + { + $this->importDataSet('Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml'); + + $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); + $tableName = 'pages'; + + $connection = $this->getMockBuilder(Elasticsearch::class) + ->setMethods(['addDocuments']) + ->disableOriginalConstructor() + ->getMock(); + + $connection->expects($this->once()) + ->method('addDocuments') + ->with( + $this->stringContains($tableName), + $this->callback(function ($documents) { + return count($documents) === 1 + && isset($documents[0]['content']) && $documents[0]['content'] === + 'this is the content of header content element that should get indexed Some text in paragraph' + && isset($documents[0]['search_abstract']) && $documents[0]['search_abstract'] === + 'Used as abstract as no abstract is defined.' + ; + }) + ); + + $indexer = $objectManager->get(IndexerFactory::class)->getIndexer($tableName); + $this->inject($indexer, 'connection', $connection); + $indexer->indexAllDocuments(); + } +} diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 2b3f817..edf1a74 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -30,6 +30,14 @@ use TYPO3\CMS\Extbase\Object\ObjectManager; class TcaIndexerTest extends AbstractFunctionalTestCase { + protected function getTypoScriptFilesForFrontendRootPage() + { + return array_merge( + parent::getTypoScriptFilesForFrontendRootPage(), + ['EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts'] + ); + } + /** * @test */ @@ -69,12 +77,4 @@ class TcaIndexerTest extends AbstractFunctionalTestCase $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); } - - protected function getTypoScriptFilesForFrontendRootPage() - { - return array_merge( - parent::getTypoScriptFilesForFrontendRootPage(), - ['EXT:search_core/Tests/Functional/Fixtures/Indexing/TcaIndexer/RespectRootLineBlacklist.ts'] - ); - } } From eafed7fb11322d9ba1201db4290938224d1686ae Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 18 Jul 2017 09:27:46 +0200 Subject: [PATCH 033/158] TASK: Add keys for better access to Facets --- Classes/Connection/Elasticsearch/Facet.php | 2 +- Classes/Connection/Elasticsearch/SearchResult.php | 2 +- Tests/Functional/Connection/Elasticsearch/FilterTest.php | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index 7893f5f..e24a659 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -87,7 +87,7 @@ class Facet implements FacetInterface } foreach ($this->buckets as $bucket) { - $this->options[] = new FacetOption($bucket); + $this->options[$bucket['key']] = new FacetOption($bucket); } } } diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index f8ffdd8..486b6eb 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -140,7 +140,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->result->getAggregations() as $aggregationName => $aggregation) { - $this->facets[] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation); + $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation); } } } diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index aa5a5dc..b5771f4 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -78,16 +78,16 @@ class FilterTest extends AbstractFunctionalTestCase $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); - $facet = $result->getFacets()[0]; + $facet = current($result->getFacets()); $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.'); $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.'); $options = $facet->getOptions(); $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.'); - $option = $options[0]; + $option = $options['HTML']; $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.'); $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); - $option = $options[1]; + $option = $options['Header']; $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.'); $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); } From 1030e8d5cfb7af8b30cc8d902f84059104df716f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 18 Jul 2017 10:44:39 +0200 Subject: [PATCH 034/158] FEATURE: Make number of search results to fetch configurable --- Classes/Connection/SearchRequestInterface.php | 7 ++ Classes/Domain/Model/SearchRequest.php | 21 ++++ Classes/Domain/Search/QueryFactory.php | 12 +++ Classes/Domain/Search/SearchService.php | 12 +++ Documentation/source/configuration.rst | 15 +++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 16 ++++ .../Unit/Domain/Search/SearchServiceTest.php | 95 +++++++++++++++++++ 7 files changed, 178 insertions(+) create mode 100644 Tests/Unit/Domain/Search/SearchServiceTest.php diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index ee779f7..7ec34ff 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -41,4 +41,11 @@ interface SearchRequestInterface * @return array */ public function getFilter(); + + /** + * Defines how many results should be fetched. + * + * @return int + */ + public function getSize(); } diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 163f65a..7850b98 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -35,6 +35,11 @@ class SearchRequest implements SearchRequestInterface */ protected $query = ''; + /** + * @var int + */ + protected $size = 10; + /** * @var array */ @@ -112,4 +117,20 @@ class SearchRequest implements SearchRequestInterface { return $this->facets; } + + /** + * @return int + */ + public function getSize() + { + return $this->size; + } + + /** + * @param int $size + */ + public function setSize($size) + { + $this->size = (int) $size; + } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7809fb6..128e556 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -63,6 +63,7 @@ class QueryFactory */ protected function createElasticaQuery(SearchRequestInterface $searchRequest) { + $this->addSize($searchRequest); $this->addSearch($searchRequest); $this->addFilter($searchRequest); $this->addFacets($searchRequest); @@ -71,6 +72,17 @@ class QueryFactory return new \Elastica\Query($this->query); } + /** + * @param SearchRequestInterface $searchRequest + */ + protected function addSize(SearchRequestInterface $searchRequest) + { + $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + 'from' => 0, + 'size' => $searchRequest->getSize(), + ]); + } + /** * @param SearchRequestInterface $searchRequest */ diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 158d9d1..aaed496 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -69,11 +69,23 @@ class SearchService */ public function search(SearchRequestInterface $searchRequest) { + $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); return $this->connection->search($searchRequest); } + /** + * Add configured size of search result items to request. + * + * @param SearchRequestInterface $searchRequest + */ + protected function addSize(SearchRequestInterface $searchRequest) + { + $size = $this->configuration->getIfExists('searching.size') ?: 10; + $searchRequest->setSize($size); + } + /** * Add facets from configuration to request. * diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 3d8db75..4555cb1 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -189,6 +189,21 @@ options are available: Searching ^^^^^^^^^ +.. _size: + +``size`` +"""""""" + + Used by: Elasticsearch connection while building search query. + + Defined how many search results should be fetched to be available in search result. + + Example:: + + plugin.tx_searchcore.settings.searching.size = 50 + + Default if not configured is 10. + .. _facets: ``facets`` diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 32c8fbd..56b0b03 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -145,4 +145,20 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Facets were not added to query.' ); } + + /** + * @test + */ + public function sizeIsAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setSize(45); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + 45, + $query->toArray()['size'], + 'Size was not added to query.' + ); + } } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php new file mode 100644 index 0000000..6a90a3c --- /dev/null +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -0,0 +1,95 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; + +class SearchServiceTest extends AbstractUnitTestCase +{ + /** + * @var SearchService + */ + protected $subject; + + public function setUp() + { + parent::setUp(); + + $this->connection = $this->getMockBuilder(ConnectionInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + $this->objectManager = $this->getMockBuilder(ObjectManagerInterface::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->subject = new SearchService( + $this->connection, + $this->configuration, + $this->objectManager + ); + } + + /** + * @test + */ + public function sizeIsAddedFromConfiguration() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(45, null)); + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getSize() === 45; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $this->subject->search($searchRequest); + } + + /** + * @test + */ + public function defaultSizeIsAddedIfNothingIsConfigured() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getSize() === 10; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $this->subject->search($searchRequest); + } +} From e2c1846d6be1fd57da7db3879b36f83fd2dace81 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 18 Jul 2017 14:19:32 +0200 Subject: [PATCH 035/158] BUGFIX: Allow parent implementation to access content entry E.g. it's possible to configure search_abstract to contain the value of any record entry. Still content was not possible as it was set afterwards. --- Classes/Domain/Index/TcaIndexer/PagesIndexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index d4ce0a6..94a898b 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -57,8 +57,8 @@ class PagesIndexer extends TcaIndexer */ protected function prepareRecord(array &$record) { - parent::prepareRecord($record); $record['content'] = $this->fetchContentForPage($record['uid']); + parent::prepareRecord($record); } /** From a737501dac636eaf390b103e1ed2c7728004d685 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 18 Jul 2017 15:12:24 +0200 Subject: [PATCH 036/158] TASK: Update existing tests to work with new pages setup --- .../Elasticsearch/IndexTcaTableTest.php | 22 ++++++++++--------- .../Fixtures/Indexing/IndexTcaTable.xml | 2 +- .../Fixtures/Indexing/ResolveRelations.xml | 12 +++++----- .../Fixtures/Indexing/UserWhereClause.xml | 4 ++-- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 9678d83..21f084a 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -50,10 +50,10 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); $this->assertArraySubset( ['_source' => ['header' => 'indexed content element']], - $response->getData()['hits']['hits'][0], + $response->getData()['hits']['hits'][1], false, 'Record was not indexed.' ); @@ -90,7 +90,7 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document was indexed.'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); } /** @@ -113,16 +113,18 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( ['_source' => ['header' => 'Also indexable record']], $response->getData()['hits']['hits'][0], false, 'Record was not indexed.' ); + $response = $this->client->request('typo3content/_search?q=uid:6'); $this->assertArraySubset( ['_source' => ['header' => 'indexed content element']], - $response->getData()['hits']['hits'][1], + $response->getData()['hits']['hits'][0], false, 'Record was not indexed.' ); @@ -143,12 +145,12 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertTrue($response->isOK(), 'Elastica did not answer with ok code.'); - $this->assertSame($response->getData()['hits']['total'], 3, 'Not exactly 3 documents were indexed.'); + $this->assertSame($response->getData()['hits']['total'], 4, 'Not exactly 4 documents were indexed.'); - $response = $this->client->request('typo3content/_search?q=uid:9'); + $response = $this->client->request('typo3content/_search?q=uid:11'); $this->assertArraySubset( ['_source' => [ - 'uid' => '9', + 'uid' => '11', 'CType' => 'Header', // Testing items 'categories' => ['Category 1', 'Category 2'], // Testing mm (with sorting) ]], @@ -157,10 +159,10 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase 'Record was not indexed with resolved category relations to multiple values.' ); - $response = $this->client->request('typo3content/_search?q=uid:10'); + $response = $this->client->request('typo3content/_search?q=uid:12'); $this->assertArraySubset( ['_source' => [ - 'uid' => '10', + 'uid' => '12', 'CType' => 'Header', 'categories' => ['Category 2'], ]], diff --git a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml index ec251ce..dd7daaa 100644 --- a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml +++ b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml @@ -67,7 +67,7 @@ 1480686370 0 72 - list + div
not indexed due to ctype
this is the content of header content element that should not get indexed 0 diff --git a/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml b/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml index 8b41246..bb76b81 100644 --- a/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml +++ b/Tests/Functional/Fixtures/Indexing/ResolveRelations.xml @@ -1,7 +1,7 @@ - 9 + 11 1 1480686370 1480686370 @@ -22,7 +22,7 @@ - 10 + 12 1 1480686370 1480686370 @@ -92,7 +92,7 @@ 1 - 9 + 11 tt_content categories 2 @@ -100,7 +100,7 @@ 2 - 9 + 11 tt_content categories 1 @@ -108,7 +108,7 @@ 3 - 9 + 11 tt_content categories 3 @@ -117,7 +117,7 @@ 2 - 10 + 12 tt_content categories 1 diff --git a/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml b/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml index 71212b2..12347ef 100644 --- a/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml +++ b/Tests/Functional/Fixtures/Indexing/UserWhereClause.xml @@ -1,7 +1,7 @@ - 9 + 11 1 1480686370 1480686370 @@ -21,7 +21,7 @@ - 10 + 12 1 1480686370 1480686370 From b832a6e6b3387f67897fcd371ab1adc4acd7e6c1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 20 Jul 2017 09:48:44 +0200 Subject: [PATCH 037/158] TASK: Allow sub indexer to exchange limit As long as it's not configurable, allow concrete implementations to exchange. Necessary for one customer at the moment. --- Classes/Domain/Index/AbstractIndexer.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index b780dc5..475532b 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -83,8 +83,7 @@ abstract class AbstractIndexer implements IndexerInterface protected function getRecordGenerator() { $offset = 0; - // TODO: Make configurable. - $limit = 50; + $limit = $this->getLimit(); while (($records = $this->getRecords($offset, $limit)) !== []) { yield $records; @@ -92,6 +91,17 @@ abstract class AbstractIndexer implements IndexerInterface } } + /** + * Returns the limit to use to fetch records. + * + * @return int + */ + protected function getLimit() + { + // TODO: Make configurable. + return 50; + } + /** * @param int $offset * @param int $limit From e8a84a8ecc3e1c9a2dad2c61c7d980961bd99c59 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 20 Jul 2017 13:48:27 +0200 Subject: [PATCH 038/158] TASK: Remove temp variable --- Classes/Domain/Search/SearchService.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index aaed496..bb8b138 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -82,8 +82,9 @@ class SearchService */ protected function addSize(SearchRequestInterface $searchRequest) { - $size = $this->configuration->getIfExists('searching.size') ?: 10; - $searchRequest->setSize($size); + $searchRequest->setSize( + $this->configuration->getIfExists('searching.size') ?: 10 + ); } /** From 7722c37ea53f2060a7feb69ac87fac7d4cf882f9 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 20 Jul 2017 14:22:52 +0200 Subject: [PATCH 039/158] TASK: Check multiple fields for pages to be used as title --- Classes/Domain/Index/TcaIndexer/PagesIndexer.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 94a898b..b6b71be 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -57,6 +57,14 @@ class PagesIndexer extends TcaIndexer */ protected function prepareRecord(array &$record) { + $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title']; + foreach ($possibleTitleFields as $searchTitleField) { + if (isset($record[$searchTitleField]) && trim($record[$searchTitleField])) { + $record['search_title'] = trim($record[$searchTitleField]); + break; + } + } + $record['content'] = $this->fetchContentForPage($record['uid']); parent::prepareRecord($record); } From 6462052c9b050ed6906e3ca25ae501b71ad81563 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 25 Jul 2017 09:52:17 +0200 Subject: [PATCH 040/158] FEATURE: Add ngram Provide configuration for index. Provide minimum_should_match configuration. --- .../Connection/Elasticsearch/IndexFactory.php | 39 ++++-- .../Elasticsearch/MappingFactory.php | 8 +- Classes/Domain/Search/QueryFactory.php | 41 ++++-- Documentation/source/configuration.rst | 50 +++++++ .../Elasticsearch/IndexFactoryTest.php | 132 ++++++++++++++++++ .../Elasticsearch/MappingFactoryTest.php | 86 ++++++++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 88 +++++++++--- 7 files changed, 397 insertions(+), 47 deletions(-) create mode 100644 Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php create mode 100644 Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 019b091..f9fac51 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -20,6 +20,8 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Elastica\Exception\ResponseException; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; @@ -31,6 +33,19 @@ use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; */ class IndexFactory implements Singleton { + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @param ConfigurationContainerInterface $configuration + */ + public function __construct(ConfigurationContainerInterface $configuration) + { + $this->configuration = $configuration; + } + /** * Get an index bases on TYPO3 table name. * @@ -41,19 +56,25 @@ class IndexFactory implements Singleton */ public function getIndex(Connection $connection, $documentType) { - // TODO: Fetch index name from configuration, based on $documentType. $index = $connection->getClient()->getIndex('typo3content'); - try { - // TODO: Provide configuration?! - // http://elastica.io/getting-started/storing-and-indexing-documents.html#section-analysis - $index->create(); - } catch (ResponseException $exception) { - if (stripos($exception->getMessage(), 'already exists') === false) { - throw $exception; - } + if ($index->exists() === false) { + $index->create($this->getConfigurationFor($documentType)); } return $index; } + + /** + * @param string $documentType + * @return array + */ + protected function getConfigurationFor($documentType) + { + try { + return $this->configuration->get('indexing.' . $documentType . '.index'); + } catch (InvalidArgumentException $e) { + return []; + } + } } diff --git a/Classes/Connection/Elasticsearch/MappingFactory.php b/Classes/Connection/Elasticsearch/MappingFactory.php index d4ac037..44a05c1 100644 --- a/Classes/Connection/Elasticsearch/MappingFactory.php +++ b/Classes/Connection/Elasticsearch/MappingFactory.php @@ -53,7 +53,13 @@ class MappingFactory implements Singleton { $mapping = new \Elastica\Type\Mapping(); $mapping->setType($type); - $mapping->setProperties($this->getConfiguration($type->getName())); + + $configuration = $this->getConfiguration($type->getName()); + if (isset($configuration['_all'])) { + $mapping->setAllField($configuration['_all']); + unset($configuration['_all']); + } + $mapping->setProperties($configuration); return $mapping; } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7809fb6..a64bcf1 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Search; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -32,6 +33,11 @@ class QueryFactory */ protected $logger; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + /** * @var array */ @@ -39,10 +45,14 @@ class QueryFactory /** * @param \TYPO3\CMS\Core\Log\LogManager $logManager + * @param ConfigurationContainerInterface $configuration */ - public function __construct(\TYPO3\CMS\Core\Log\LogManager $logManager) - { + public function __construct( + \TYPO3\CMS\Core\Log\LogManager $logManager, + ConfigurationContainerInterface $configuration + ) { $this->logger = $logManager->getLogger(__CLASS__); + $this->configuration = $configuration; } /** @@ -76,19 +86,20 @@ class QueryFactory */ protected function addSearch(SearchRequestInterface $searchRequest) { - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ - 'query' => [ - 'bool' => [ - 'must' => [ - [ - 'match' => [ - '_all' => $searchRequest->getSearchTerm() - ], - ], - ], - ], - ], - ]); + $this->query = ArrayUtility::setValueByPath( + $this->query, + 'query.bool.must.0.match._all.query', + $searchRequest->getSearchTerm() + ); + + $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); + if ($minimumShouldMatch) { + $this->query = ArrayUtility::setValueByPath( + $this->query, + 'query.bool.must.0.match._all.minimum_should_match', + $minimumShouldMatch + ); + } } /** diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 3d8db75..e7b8a7f 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -184,6 +184,43 @@ options are available: makes building a facet possible. +.. _index: + +``index`` +""""""""" + + Used by: Elasticsearch connection while indexing. + + Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter { + html_strip + } + + filter { + lowercase + } + } + } + + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 + } + } + } + } + .. _configuration_options_search: Searching @@ -209,3 +246,16 @@ Searching The above example will provide a facet with options for all found ``CType`` results together with a count. + +.. _minimumShouldMatch: + +``minimumShouldMatch`` +"""""""""""""""""""""" + + Used by: Elasticsearch connection while building search query. + + Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html + + Example:: + + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% diff --git a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php new file mode 100644 index 0000000..ee55156 --- /dev/null +++ b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php @@ -0,0 +1,132 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\Elasticsearch\Connection; +use Codappix\SearchCore\Connection\Elasticsearch\IndexFactory; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class IndexFactoryTest extends AbstractUnitTestCase +{ + /** + * @var IndexFactory + */ + protected $subject; + + public function setUp() + { + parent::setUp(); + + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->subject = new IndexFactory($this->configuration); + } + + /** + * @test + */ + public function indexIsNotCreatedIfAlreadyExisting() + { + $indexMock = $this->getMockBuilder(\Elastica\Index::class) + ->disableOriginalConstructor() + ->getMock(); + $indexMock->expects($this->once()) + ->method('exists') + ->willReturn(true); + $indexMock->expects($this->never()) + ->method('create'); + $clientMock = $this->getMockBuilder(\Elastica\Client::class) + ->disableOriginalConstructor() + ->getMock(); + $clientMock->expects($this->once()) + ->method('getIndex') + ->with('typo3content') + ->willReturn($indexMock); + $connection = $this->getMockBuilder(Connection::class) + ->disableOriginalConstructor() + ->getMock(); + $connection->expects($this->once()) + ->method('getClient') + ->willReturn($clientMock); + + $this->subject->getIndex($connection, 'someIndex'); + } + + /** + * @test + */ + public function typoScriptConfigurationIsProvidedToIndex() + { + $configuration = [ + 'analysis' => [ + 'analyzer' => [ + 'ngram4' => [ + 'type' => 'custom', + 'tokenizer' => 'ngram4', + 'char_filter' => [ + 'html_strip', + ], + 'filter' => [ + 'lowercase', + ], + ], + ], + 'tokenizer' => [ + 'ngram4' => [ + 'type' => 'ngram', + 'min_gram' => 4, + 'max_gram' => 4, + ], + ], + ], + ]; + + $indexMock = $this->getMockBuilder(\Elastica\Index::class) + ->disableOriginalConstructor() + ->getMock(); + $indexMock->expects($this->once()) + ->method('exists') + ->willReturn(false); + $indexMock->expects($this->once()) + ->method('create') + ->with($configuration); + $clientMock = $this->getMockBuilder(\Elastica\Client::class) + ->disableOriginalConstructor() + ->getMock(); + $clientMock->expects($this->once()) + ->method('getIndex') + ->with('typo3content') + ->willReturn($indexMock); + $connection = $this->getMockBuilder(Connection::class) + ->disableOriginalConstructor() + ->getMock(); + $connection->expects($this->once()) + ->method('getClient') + ->willReturn($clientMock); + + $this->configuration->expects($this->once()) + ->method('get') + ->with('indexing.someIndex.index') + ->willReturn($configuration); + + $this->subject->getIndex($connection, 'someIndex'); + } +} diff --git a/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php new file mode 100644 index 0000000..8dde1cb --- /dev/null +++ b/Tests/Unit/Connection/Elasticsearch/MappingFactoryTest.php @@ -0,0 +1,86 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Connection\Elasticsearch\MappingFactory; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class MappingFactoryTest extends AbstractUnitTestCase +{ + /** + * @var MappingFactory + */ + protected $subject; + + public function setUp() + { + parent::setUp(); + + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->subject = new MappingFactory($this->configuration); + } + + /** + * @test + */ + public function typoScriptConfigurationIsProvidedToIndex() + { + $indexName = 'someIndex'; + $configuration = [ + '_all' => [ + 'type' => 'text', + 'analyzer' => 'ngram4', + ], + 'channel' => [ + 'type' => 'keyword', + ], + ]; + $type = $this->getMockBuilder(\Elastica\Type::class) + ->disableOriginalConstructor() + ->getMock(); + $type->expects($this->any()) + ->method('getName') + ->willReturn($indexName); + $this->configuration->expects($this->once()) + ->method('get') + ->with('indexing.' . $indexName . '.mapping') + ->willReturn($configuration); + + $mapping = $this->subject->getMapping($type)->toArray()[$indexName]; + $this->assertArraySubset( + [ + '_all' => $configuration['_all'] + ], + $mapping, + true, + 'Configuration of _all field was not set for mapping.' + ); + $this->assertArraySubset( + [ + 'channel' => $configuration['channel'] + ], + $mapping['properties'], + true, + 'Configuration for properties was not set for mapping.' + ); + } +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 32c8fbd..8dcf218 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\QueryFactory; @@ -32,11 +33,17 @@ class QueryFactoryTest extends AbstractUnitTestCase */ protected $subject; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + public function setUp() { parent::setUp(); - $this->subject = new QueryFactory($this->getMockedLogger()); + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); } /** @@ -97,27 +104,6 @@ class QueryFactoryTest extends AbstractUnitTestCase ); } - /** - * @test - */ - public function userInputIsAlwaysString() - { - $searchRequest = new SearchRequest(10); - $searchRequest->setFilter(['field' => 20]); - - $query = $this->subject->create($searchRequest); - $this->assertSame( - '10', - $query->toArray()['query']['bool']['must'][0]['match']['_all'], - 'Search word was not escaped as expected.' - ); - $this->assertSame( - '20', - $query->toArray()['query']['bool']['filter'][0]['term']['field'], - 'Search word was not escaped as expected.' - ); - } - /** * @test */ @@ -145,4 +131,62 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Facets were not added to query.' ); } + + /** + * @test + */ + public function searchTermIsAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $query = $this->subject->create($searchRequest); + + $this->assertSame( + [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => [ + 'query' => 'SearchWord', + ], + ], + ], + ], + ], + ], + $query->toArray()['query'], + 'Search term was not added to query as expected.' + ); + } + + /** + * @test + */ + public function minimumShouldMatchIsAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $this->configuration->expects($this->once()) + ->method('getIfExists') + ->with('searching.minimumShouldMatch') + ->willReturn('50%'); + $query = $this->subject->create($searchRequest); + + $this->assertArraySubset( + [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => [ + 'minimum_should_match' => '50%', + ], + ], + ], + ], + ], + ], + $query->toArray()['query'], + 'minimum_should_match was not added to query as configured.' + ); + } } From c1cc16efa59154cb8a6516a37a86e5ad79510465 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 25 Jul 2017 15:00:25 +0200 Subject: [PATCH 041/158] BUGFIX: Fix nun working options due to miss match of ts and es As TypoScript does not provide a way to configure key less options, we use a comma separated list and explode them to stay compatible with elasticsearch. --- .../Connection/Elasticsearch/IndexFactory.php | 30 ++++++++++++++++++- Documentation/source/configuration.rst | 11 +++---- .../Elasticsearch/IndexFactoryTest.php | 14 ++++----- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index f9fac51..66b448b 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -24,6 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Elastica\Exception\ResponseException; use TYPO3\CMS\Core\SingletonInterface as Singleton; +use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; /** @@ -67,14 +68,41 @@ class IndexFactory implements Singleton /** * @param string $documentType + * * @return array */ protected function getConfigurationFor($documentType) { try { - return $this->configuration->get('indexing.' . $documentType . '.index'); + $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); + + if (isset($configuration['analysis']['analyzer'])) { + foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) { + $configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer); + } + } + + return $configuration; } catch (InvalidArgumentException $e) { return []; } } + + /** + * @param array $analyzer + * + * @return array + */ + protected function prepareAnalyzerConfiguration(array $analyzer) + { + $fieldsToExplode = ['char_filter', 'filter']; + + foreach ($fieldsToExplode as $fieldToExplode) { + if (isset($analyzer[$fieldToExplode])) { + $analyzer[$fieldToExplode] = GeneralUtility::trimExplode(',', $analyzer[$fieldToExplode], true); + } + } + + return $analyzer; + } } diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index e7b8a7f..68994d7 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -201,13 +201,8 @@ options are available: ngram4 { type = custom tokenizer = ngram4 - char_filter { - html_strip - } - - filter { - lowercase - } + char_filter = html_strip + filter = lowercase, asciifolding } } @@ -221,6 +216,8 @@ options are available: } } + ``char_filter`` and ``filter`` are a comma separated list of options. + .. _configuration_options_search: Searching diff --git a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php index ee55156..c73fe36 100644 --- a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php +++ b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php @@ -81,12 +81,8 @@ class IndexFactoryTest extends AbstractUnitTestCase 'ngram4' => [ 'type' => 'custom', 'tokenizer' => 'ngram4', - 'char_filter' => [ - 'html_strip', - ], - 'filter' => [ - 'lowercase', - ], + 'char_filter' => 'html_strip', + 'filter' => 'lowercase, , asciifolding', ], ], 'tokenizer' => [ @@ -99,6 +95,10 @@ class IndexFactoryTest extends AbstractUnitTestCase ], ]; + $expectedConfiguration = $configuration; + $expectedConfiguration['analysis']['analyzer']['ngram4']['char_filter'] = ['html_strip']; + $expectedConfiguration['analysis']['analyzer']['ngram4']['filter'] = ['lowercase', 'asciifolding']; + $indexMock = $this->getMockBuilder(\Elastica\Index::class) ->disableOriginalConstructor() ->getMock(); @@ -107,7 +107,7 @@ class IndexFactoryTest extends AbstractUnitTestCase ->willReturn(false); $indexMock->expects($this->once()) ->method('create') - ->with($configuration); + ->with($expectedConfiguration); $clientMock = $this->getMockBuilder(\Elastica\Client::class) ->disableOriginalConstructor() ->getMock(); From f138cd9034ea6f6beb151878acba1884cd104c57 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 25 Jul 2017 15:38:40 +0200 Subject: [PATCH 042/158] FEATURE: Add possibility to boost certain fields Allow configuration via TS to boost certain fields during searching. --- Classes/Domain/Search/QueryFactory.php | 56 ++++++++++++++++++- Documentation/source/configuration.rst | 19 +++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 49 +++++++++++++++- 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7809fb6..7f4df73 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Search; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -32,6 +33,11 @@ class QueryFactory */ protected $logger; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + /** * @var array */ @@ -39,13 +45,21 @@ class QueryFactory /** * @param \TYPO3\CMS\Core\Log\LogManager $logManager + * @param ConfigurationContainerInterface $configuration */ - public function __construct(\TYPO3\CMS\Core\Log\LogManager $logManager) - { + public function __construct( + \TYPO3\CMS\Core\Log\LogManager $logManager, + ConfigurationContainerInterface $configuration + ) { $this->logger = $logManager->getLogger(__CLASS__); + $this->configuration = $configuration; } /** + * TODO: This is not in scope Elasticsearch, therefore it should not return + * \Elastica\Query, but decide to use a more specific QueryFactory like + * ElasticaQueryFactory, once the second query is added? + * * @param SearchRequestInterface $searchRequest * * @return \Elastica\Query @@ -58,12 +72,12 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest * - * TODO: This is not in scope Elasticsearch, therefore should not return elastica. * @return \Elastica\Query */ protected function createElasticaQuery(SearchRequestInterface $searchRequest) { $this->addSearch($searchRequest); + $this->addBoosts($searchRequest); $this->addFilter($searchRequest); $this->addFacets($searchRequest); @@ -91,6 +105,42 @@ class QueryFactory ]); } + /** + * @param SearchRequestInterface $searchRequest + */ + protected function addBoosts(SearchRequestInterface $searchRequest) + { + try { + $fields = $this->configuration->get('searching.boost'); + if (!$fields) { + return; + } + } catch (InvalidArgumentException $e) { + return; + } + + $boostQueryParts = []; + + foreach ($fields as $fieldName => $boostValue) { + $boostQueryParts[] = [ + 'match' => [ + $fieldName => [ + 'query' => $searchRequest->getSearchTerm(), + 'boost' => $boostValue, + ], + ], + ]; + } + + $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + 'query' => [ + 'bool' => [ + 'should' => $boostQueryParts, + ], + ], + ]); + } + /** * @param SearchRequestInterface $searchRequest */ diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 3d8db75..14197fe 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -209,3 +209,22 @@ Searching The above example will provide a facet with options for all found ``CType`` results together with a count. + +.. _boost: + +``boost`` +""""""""" + + Used by: Elasticsearch connection while building search query. + + Define fields that should boost the score for results. + + Example:: + + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } + + For further information take a look at + https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 32c8fbd..6c6b834 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\QueryFactory; @@ -32,11 +33,17 @@ class QueryFactoryTest extends AbstractUnitTestCase */ protected $subject; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + public function setUp() { parent::setUp(); - $this->subject = new QueryFactory($this->getMockedLogger()); + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); } /** @@ -145,4 +152,44 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Facets were not added to query.' ); } + + /** + * @test + */ + public function boostsAreAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + + $this->configuration->expects($this->once()) + ->method('get') + ->with('searching.boost') + ->willReturn([ + 'search_title' => 3, + 'search_abstract' => 1.5, + ]); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + [ + 'match' => [ + 'search_title' => [ + 'query' => 'SearchWord', + 'boost' => 3, + ], + ], + ], + [ + 'match' => [ + 'search_abstract' => [ + 'query' => 'SearchWord', + 'boost' => 1.5, + ], + ], + ], + ], + $query->toArray()['query']['bool']['should'], + 'Boosts were not added to query.' + ); + } } From 334bb34625b0ff579329b906495cc51cb8b0e1ca Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 27 Jul 2017 13:00:51 +0200 Subject: [PATCH 043/158] TASK: Cleanup code Use get and use an try catch. Receiving null through ifExists will result in an php error. --- Classes/Domain/Index/AbstractIndexer.php | 27 ++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 18e1702..51acc12 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Index; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use \TYPO3\CMS\Core\Utility\GeneralUtility; @@ -125,18 +126,22 @@ abstract class AbstractIndexer implements IndexerInterface { $record['search_abstract'] = ''; - $fieldsToUse = GeneralUtility::trimExplode( - ',', - $this->configuration->getIfExists('indexing.' . $this->identifier . '.abstractFields') - ); - if (!$fieldsToUse) { - return; - } - foreach ($fieldsToUse as $fieldToUse) { - if (isset($record[$fieldToUse]) && trim($record[$fieldToUse])) { - $record['search_abstract'] = trim($record[$fieldToUse]); - break; + try { + $fieldsToUse = GeneralUtility::trimExplode( + ',', + $this->configuration->get('indexing.' . $this->identifier . '.abstractFields') + ); + if (!$fieldsToUse) { + return; } + foreach ($fieldsToUse as $fieldToUse) { + if (isset($record[$fieldToUse]) && trim($record[$fieldToUse])) { + $record['search_abstract'] = trim($record[$fieldToUse]); + break; + } + } + } catch (InvalidArgumentException $e) { + return; } } From f436a02f5568960da77c681013229c309e5daf0b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 27 Jul 2017 14:20:37 +0200 Subject: [PATCH 044/158] FEATURE: Add field_value_factor support through configuration --- Classes/Domain/Search/QueryFactory.php | 22 +++++- Documentation/source/configuration.rst | 19 +++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 78 +++++++++++++++++-- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7f4df73..d8acbf2 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -81,6 +82,10 @@ class QueryFactory $this->addFilter($searchRequest); $this->addFacets($searchRequest); + // Use last, as it might change structure of query. + // Better approach would be something like DQL to generate query and build result in the end. + $this->addFactorBoost(); + $this->logger->debug('Generated elasticsearch query.', [$this->query]); return new \Elastica\Query($this->query); } @@ -112,9 +117,6 @@ class QueryFactory { try { $fields = $this->configuration->get('searching.boost'); - if (!$fields) { - return; - } } catch (InvalidArgumentException $e) { return; } @@ -141,6 +143,20 @@ class QueryFactory ]); } + protected function addFactorBoost() + { + try { + $this->query['query'] = [ + 'function_score' => [ + 'query' => $this->query['query'], + 'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'), + ], + ]; + } catch (InvalidArgumentException $e) { + return; + } + } + /** * @param SearchRequestInterface $searchRequest */ diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 14197fe..4b215cb 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -228,3 +228,22 @@ Searching For further information take a look at https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html + +.. _fieldValueFactor: + +``fieldValueFactor`` +"""""""""""""""""""" + + Used by: Elasticsearch connection while building search query. + + Define a field to use as a factor for scoring. The configuration is passed through to elastic + search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor + + Example:: + + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 6c6b834..2b27af2 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\QueryFactory; @@ -53,6 +54,10 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $query = $this->subject->create($searchRequest); $this->assertInstanceOf( \Elastica\Query::class, @@ -66,6 +71,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); @@ -84,6 +93,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptyFilterIsNotAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ 'field' => '', @@ -109,6 +122,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function userInputIsAlwaysString() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest(10); $searchRequest->setFilter(['field' => 20]); @@ -130,6 +147,9 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function facetsAreAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); @@ -160,13 +180,16 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->once()) + $this->configuration->expects($this->exactly(2)) ->method('get') - ->with('searching.boost') - ->willReturn([ - 'search_title' => 3, - 'search_abstract' => 1.5, - ]); + ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->will($this->onConsecutiveCalls( + [ + 'search_title' => 3, + 'search_abstract' => 1.5, + ], + $this->throwException(new InvalidArgumentException) + )); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -192,4 +215,47 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Boosts were not added to query.' ); } + + /** + * @test + */ + public function factorBoostIsAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $fieldConfig = [ + 'field' => 'rootlineLevel', + 'modifier' => 'reciprocal', + 'factor' => '2', + 'missing' => '1', + ]; + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $fieldConfig + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'function_score' => [ + 'query' => [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => 'SearchWord', + ], + ], + ], + ], + ], + 'field_value_factor' => $fieldConfig, + ], + ], + $query->toArray()['query'], + 'Boosts were not added to query.' + ); + } } From da7692b5022025b5d68084f02e6c9d3baa8e8ac6 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 27 Jul 2017 16:31:37 +0200 Subject: [PATCH 045/158] TASK: Adjust code accordingly to pr --- Classes/Domain/Index/AbstractIndexer.php | 2 +- Configuration/TypoScript/constants.txt | 2 +- Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index f4be6db..143a219 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -40,7 +40,7 @@ abstract class AbstractIndexer implements IndexerInterface /** * @var string */ - protected $identifier; + protected $identifier = ''; /** * @var \TYPO3\CMS\Core\Log\Logger diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index dff37e3..bcf191e 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -10,7 +10,7 @@ plugin { indexing { pages { - additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255, 199) + additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255) abstractFields = abstract, description, bodytext } } diff --git a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml index dd7daaa..c236f07 100644 --- a/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml +++ b/Tests/Functional/Fixtures/Indexing/IndexTcaTable.xml @@ -69,7 +69,7 @@ 72 div
not indexed due to ctype
- this is the content of header content element that should not get indexed + this is the content of div content element that should not get indexed 0 0 0 From 31937d2d6f671bbe5ac5bc6eeb2792a5afeb84a9 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 28 Jul 2017 11:58:24 +0200 Subject: [PATCH 046/158] BUGFIX: Fix broken test --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 33ef852..9ff690a 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -153,6 +153,9 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function sizeIsAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setSize(45); From 56ce88b0055edf1fba62a827bb73346c5a66a912 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 4 Aug 2017 13:39:18 +0200 Subject: [PATCH 047/158] TASK: Resolve relations always to array Do not resolve to empty string but array to allow same handling for all relations. --- Classes/Domain/Index/TcaIndexer/RelationResolver.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index b09e483..483fa4b 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -20,13 +20,10 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ -use TYPO3\CMS\Backend\Utility\BackendUtility; -use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Utility\LocalizationUtility; - use TYPO3\CMS\Backend\Form\FormDataCompiler; use TYPO3\CMS\Backend\Form\FormDataGroup\TcaDatabaseRecord; +use TYPO3\CMS\Core\SingletonInterface as Singleton; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Resolves relations from TCA using TCA. @@ -81,7 +78,7 @@ class RelationResolver implements Singleton protected function resolveValue($value, array $tcaColumn) { if ($value === '' || $value === '0') { - return ''; + return []; } if ($tcaColumn['config']['type'] === 'select') { return $this->resolveSelectValue($value, $tcaColumn); @@ -93,7 +90,7 @@ class RelationResolver implements Singleton return $this->resolveInlineValue($tcaColumn); } - return ''; + return []; } /** From ac78464c030ab66a94a2b49c1fb186ef6e8e0de4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 4 Aug 2017 13:39:48 +0200 Subject: [PATCH 048/158] TASK: Provide more helpful logging --- Classes/Connection/Elasticsearch/DocumentFactory.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 99d29e5..390c592 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -61,7 +61,10 @@ class DocumentFactory implements Singleton $identifier = $document['search_identifier']; unset($document['search_identifier']); - $this->logger->debug('Convert document to document', [$identifier, $document]); + $this->logger->debug( + sprintf('Convert %s %u to document.', $documentType, $identifier), + [$identifier, $document] + ); return new \Elastica\Document($identifier, $document); } From 6595d7129abebb8e810d525bf5db27f1152fd4f5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 7 Aug 2017 18:04:07 +0200 Subject: [PATCH 049/158] TASK: Move additionalWhereClause to constants To keep things the same, move additionalWhereClause of tt_content also to constants. --- Configuration/TypoScript/constants.txt | 4 ++++ Configuration/TypoScript/setup.txt | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index bcf191e..1a39851 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -9,6 +9,10 @@ plugin { } indexing { + tt_content { + additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != '' + } + pages { additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255) abstractFields = abstract, description, bodytext diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index d77c42e..67612e1 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -12,10 +12,7 @@ plugin { # Not for direct indexing therefore no indexer. # Used to configure tt_content fetching while indexing pages tt_content { - additionalWhereClause ( - tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - AND tt_content.bodytext != '' - ) + additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause} } pages { From baca4824d5f6b38ef572840072cc00265d7d1fb5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 09:47:12 +0200 Subject: [PATCH 050/158] BUGFIX: Do not test php 5.6 As TYPO3 7.6.x introduced breaking change to not support PHP 5.6 any longer. --- .travis.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3253c3c..28c45aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ before_install: language: php php: - - 5.6 - 7.0 - 7.1 @@ -31,12 +30,6 @@ env: matrix: fast_finish: true - exclude: - # TYPO3 no longer supports 5.6 - - env: TYPO3_VERSION="~8" - php: 5.6 - - env: TYPO3_VERSION="dev-master" - php: 5.6 allow_failures: - env: TYPO3_VERSION="~8" php: 7.0 From 9e64252a6a69396671fa2eb69c7c668df0133c70 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 09:57:56 +0200 Subject: [PATCH 051/158] BUGFIX: Grant necessary mysql permission to run ft In order to run functional tests, we need to grand necessary permissions on travis. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3253c3c..190a765 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ addons: - oracle-java8-set-default before_install: - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.2.0.deb && sudo service elasticsearch start + - mysql -u root -e 'GRANT ALL ON `typo3_ci_ft%`.* TO travis@127.0.0.1;' language: php From 4c3cca98557cfd474296b3e391b198ddf4c82713 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 10:13:24 +0200 Subject: [PATCH 052/158] BUGFIX: Do not use PHP 5.6 any longer for TYPO3 7.6.x As TYPO3 7.6.x introduced a breaking change which no longer works with PHP 5.6, we need to remove the support. --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 190a765..6d3f68e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ before_install: language: php php: - - 5.6 - 7.0 - 7.1 @@ -33,11 +32,6 @@ env: matrix: fast_finish: true exclude: - # TYPO3 no longer supports 5.6 - - env: TYPO3_VERSION="~8" - php: 5.6 - - env: TYPO3_VERSION="dev-master" - php: 5.6 allow_failures: - env: TYPO3_VERSION="~8" php: 7.0 From f3f554f0a6a44708b231e98d0d956eaced809eb4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 10:23:36 +0200 Subject: [PATCH 053/158] TASK: Only test working TYPO3 Version As we have support branches for 6.2 and feature branch for 8 support, we only test 7 for this branch as only 7 is supported. --- .travis.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d3f68e..f22bb48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,23 +24,7 @@ env: - typo3DatabaseHost="127.0.0.1" - typo3DatabaseUsername="travis" - typo3DatabasePassword="" - matrix: - TYPO3_VERSION="~7.6" - - TYPO3_VERSION="~8" - - TYPO3_VERSION="dev-master" - -matrix: - fast_finish: true - exclude: - allow_failures: - - env: TYPO3_VERSION="~8" - php: 7.0 - - env: TYPO3_VERSION="~8" - php: 7.1 - - env: TYPO3_VERSION="dev-master" - php: 7.0 - - env: TYPO3_VERSION="dev-master" - php: 7.1 services: - mysql From 67d1180ee18b9d4dcc57d2fad7b10f976da12527 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 10:36:50 +0200 Subject: [PATCH 054/158] TASK: Remove php 7.0 testing As we do not support PHP 7.0. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea6d947..21d058f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ before_install: language: php php: - - 7.0 - 7.1 env: @@ -31,8 +30,6 @@ env: matrix: fast_finish: true allow_failures: - - env: TYPO3_VERSION="dev-master" - php: 7.0 - env: TYPO3_VERSION="dev-master" php: 7.1 From 49a56496c6cd301d01451ab86772b573d5c6b2e1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 11:54:32 +0200 Subject: [PATCH 055/158] TASK: Fix codacy issues Break line to not exceed maximum line length. Use imported namespace to shorten line. --- Classes/Domain/Index/TcaIndexer/RelationResolver.php | 8 +++++++- .../Hooks/DataHandler/AbstractDataHandlerTest.php | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 5d5a256..88aa982 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -39,7 +39,13 @@ class RelationResolver implements Singleton if ($column === 'pid') { continue; } - $record[$column] = BackendUtility::getProcessedValueExtra($service->getTableName(), $column, $record[$column], 0, $record['uid']); + $record[$column] = BackendUtility::getProcessedValueExtra( + $service->getTableName(), + $column, + $record[$column], + 0, + $record['uid'] + ); try { $config = $service->getColumnConfig($column); diff --git a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php index 0b0f128..84ac0bd 100644 --- a/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php +++ b/Tests/Functional/Hooks/DataHandler/AbstractDataHandlerTest.php @@ -50,6 +50,6 @@ abstract class AbstractDataHandlerTest extends AbstractFunctionalTestCase ->setMethods(['add', 'update', 'delete']) ->getMock(); - GeneralUtility::setSingletonInstance(\Codappix\SearchCore\Hook\DataHandler::class, new DataHandlerHook($this->subject)); + GeneralUtility::setSingletonInstance(DataHandlerHook::class, new DataHandlerHook($this->subject)); } } From 17eb35a92bf152a5d571f2e97b44b22bb791cf2f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 12:58:01 +0200 Subject: [PATCH 056/158] FEATURE: Respect inherited start- and endtime for pages Do not index records below tables that extend their start- or endtime to their subpages are not accessible due to timing now. --- .../Index/TcaIndexer/TcaTableService.php | 44 ++++++++++++++----- .../Indexing/PagesIndexer/InheritedTiming.xml | 34 ++++++++++++++ .../Functional/Indexing/PagesIndexerTest.php | 29 ++++++++++++ 3 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index b5f48ab..488f949 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -24,6 +24,8 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Index\IndexingException; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Core\Utility\RootlineUtility; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** * Encapsulate logik related to TCA configuration. @@ -47,15 +49,20 @@ class TcaTableService */ protected $configuration; + /** + * @var RelationResolver + */ + protected $relationResolver; + /** * @var \TYPO3\CMS\Core\Log\Logger */ protected $logger; /** - * @var RelationResolver + * @var ObjectManagerInterface */ - protected $relationResolver; + protected $objectManager; /** * Inject log manager to get concrete logger from it. @@ -67,6 +74,14 @@ class TcaTableService $this->logger = $logManager->getLogger(__CLASS__); } + /** + * @param ObjectManagerInterface $objectManager + */ + public function injectObjectManager(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + /** * @param string $tableName * @param ConfigurationContainerInterface $configuration @@ -243,26 +258,31 @@ class TcaTableService * Checks whether the given record was blacklisted by root line. * This can be configured by typoscript as whole root lines can be black listed. * - * NOTE: Does not support pages yet. We have to add a switch once we - * support them to use uid instead. - * * @param array &$record * @return bool */ protected function isRecordBlacklistedByRootline(array &$record) { // If no rootline exists, the record is on a unreachable page and therefore blacklisted. - $rootline = BackendUtility::BEgetRootLine($record['pid']); + if ($record['pid'] == 0) { + return false; + } + + $rootline = $this->objectManager->get(RootlineUtility::class, $record['pid'])->get(); if (!isset($rootline[0])) { return true; } - // Check configured black list if present. - if ($this->isBlackListedRootLineConfigured()) { - foreach ($rootline as $pageInRootLine) { - if (in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())) { - return true; - } + foreach ($rootline as $pageInRootLine) { + // Check configured black list if present. + if ($this->isBlackListedRootLineConfigured() && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())) { + return true; + } + if ($pageInRootLine['extendToSubpages'] && ( + ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) + || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) + )) { + return true; } } diff --git a/Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml b/Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml new file mode 100644 index 0000000..552ba74 --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml @@ -0,0 +1,34 @@ + + + + + 2 + 1 + Some disabled page due to timing + 1502186635 + 1 + + + 3 + 2 + Some disabled page due to inherited timing + + + 4 + 1 + Some disabled page due to timing + 2147483647 + 1 + + + 5 + 4 + Some disabled page due to inherited timing + + + + 6 + 1 + Some enabled page due to no be below inherited disabled timing + + diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php index d0440ba..2f00d2f 100644 --- a/Tests/Functional/Indexing/PagesIndexerTest.php +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -61,4 +61,33 @@ class PagesIndexerTest extends AbstractFunctionalTestCase $this->inject($indexer, 'connection', $connection); $indexer->indexAllDocuments(); } + + /** + * @test + */ + public function inheritedTimingIsRespectedDuringIndexing() + { + $this->importDataSet('Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml'); + + $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); + $tableName = 'pages'; + + $connection = $this->getMockBuilder(Elasticsearch::class) + ->setMethods(['addDocuments']) + ->disableOriginalConstructor() + ->getMock(); + + $connection->expects($this->once()) + ->method('addDocuments') + ->with( + $this->stringContains($tableName), + $this->callback(function ($documents) { + return count($documents) === 2; + }) + ); + + $indexer = $objectManager->get(IndexerFactory::class)->getIndexer($tableName); + $this->inject($indexer, 'connection', $connection); + $indexer->indexAllDocuments(); + } } From f7e1bd1cdfa8f42e4610596185270a520cad6eb1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 17:07:23 +0200 Subject: [PATCH 057/158] FEATURE: Implement necessary logic to support PaginateViewHelper --- Classes/Connection/Elasticsearch.php | 2 +- .../Connection/Elasticsearch/SearchResult.php | 98 +++++++--- Classes/Connection/SearchRequestInterface.php | 14 +- Classes/Connection/SearchResultInterface.php | 17 +- Classes/Domain/Model/SearchRequest.php | 183 ++++++++++++++++-- Classes/Domain/Search/QueryFactory.php | 10 +- Classes/Domain/Search/SearchService.php | 3 +- Tests/Unit/Domain/Search/QueryFactoryTest.php | 10 +- .../Unit/Domain/Search/SearchServiceTest.php | 4 +- 9 files changed, 264 insertions(+), 77 deletions(-) diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index f6321bc..4c66b6a 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -189,7 +189,7 @@ class Elasticsearch implements Singleton, ConnectionInterface $search->addIndex('typo3content'); $search->setQuery($this->queryFactory->create($searchRequest)); - return $this->objectManager->get(SearchResult::class, $search->search()); + return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search()); } /** diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 486b6eb..46ffb14 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -22,11 +22,17 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Connection\FacetInterface; use Codappix\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; class SearchResult implements SearchResultInterface { + /** + * @var SearchRequestInterface + */ + protected $searchRequest; + /** * @var \Elastica\ResultSet */ @@ -47,8 +53,12 @@ class SearchResult implements SearchResultInterface */ protected $objectManager; - public function __construct(\Elastica\ResultSet $result, ObjectManagerInterface $objectManager) - { + public function __construct( + SearchRequestInterface $searchRequest, + \Elastica\ResultSet $result, + ObjectManagerInterface $objectManager + ) { + $this->searchRequest = $searchRequest; $this->result = $result; $this->objectManager = $objectManager; } @@ -75,25 +85,37 @@ class SearchResult implements SearchResultInterface return $this->facets; } - /** - * Returns the total sum of matching results. - * - * @return int - */ - public function getTotalCount() + public function getCurrentCount() { - return $this->result->getTotalHits(); + return $this->result->count(); + } + + protected function initResults() + { + if ($this->results !== []) { + return; + } + + foreach ($this->result->getResults() as $result) { + $this->results[] = new ResultItem($result); + } + } + + protected function initFacets() + { + if ($this->facets !== [] || !$this->result->hasAggregations()) { + return; + } + + foreach ($this->result->getAggregations() as $aggregationName => $aggregation) { + $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation); + } } // Countable - Interface - /** - * Returns the total sum of results contained in this result. - * - * @return int - */ public function count() { - return $this->result->count(); + return $this->result->getTotalHits(); } // Iterator - Interface @@ -122,25 +144,41 @@ class SearchResult implements SearchResultInterface $this->result->rewind(); } - protected function initResults() - { - if ($this->results !== []) { - return; - } + // Extbase QueryResultInterface - Implemented to support Pagination of Fluid. - foreach ($this->result->getResults() as $result) { - $this->results[] = new ResultItem($result); - } + public function getQuery() + { + return $this->searchRequest; } - protected function initFacets() + public function getFirst() { - if ($this->facets !== [] || !$this->result->hasAggregations()) { - return; - } + throw new \BadMethodCallException('Method is not implemented yet.', 1502195121); + } - foreach ($this->result->getAggregations() as $aggregationName => $aggregation) { - $this->facets[$aggregationName] = $this->objectManager->get(Facet::class, $aggregationName, $aggregation); - } + public function toArray() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502195135); + } + + public function offsetExists($offset) + { + // Return false to allow Fluid to use appropriate getter methods. + return false; + } + + public function offsetGet($offset) + { + throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933); + } + + public function offsetSet($offset, $value) + { + throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934); + } + + public function offsetUnset($offset) + { + throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936); } } diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index 7ec34ff..7c7956e 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -20,10 +20,9 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ -/** - * - */ -interface SearchRequestInterface +use TYPO3\CMS\Extbase\Persistence\QueryInterface; + +interface SearchRequestInterface extends QueryInterface { /** * Returns the actual string the user searched for. @@ -41,11 +40,4 @@ interface SearchRequestInterface * @return array */ public function getFilter(); - - /** - * Defines how many results should be fetched. - * - * @return int - */ - public function getSize(); } diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index 5c45bcd..d52a8ba 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -20,10 +20,12 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ +use TYPO3\CMS\Extbase\Persistence\QueryResultInterface; + /** * A search result. */ -interface SearchResultInterface extends \Iterator, \Countable +interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterface { /** * @return array @@ -38,18 +40,9 @@ interface SearchResultInterface extends \Iterator, \Countable public function getFacets(); /** - * Returns the total sum of matching results. + * Returns the number of results in current result * * @return int */ - public function getTotalCount(); - - // Countable - Interface - - /** - * Returns the total sum of results contained in this result. - * - * @return int - */ - public function count(); + public function getCurrentCount(); } diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 7850b98..7d6436c 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Model; * 02110-1301, USA. */ +use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\FacetRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -35,11 +36,6 @@ class SearchRequest implements SearchRequestInterface */ protected $query = ''; - /** - * @var int - */ - protected $size = 10; - /** * @var array */ @@ -50,6 +46,23 @@ class SearchRequest implements SearchRequestInterface */ protected $facets = []; + /** + * @var int + */ + protected $offset = 0; + + /** + * @var int + */ + protected $limit = 10; + + /** + * Used for QueryInterface implementation to allow execute method to work. + * + * @var ConnectionInterface + */ + protected $connection = null; + /** * @param string $query */ @@ -119,18 +132,162 @@ class SearchRequest implements SearchRequestInterface } /** - * @return int + * Define connection to use for this request. + * Necessary to allow implementation of execute for interface. + * + * @param ConnectionInterface $connection */ - public function getSize() + public function setConnection(ConnectionInterface $connection) { - return $this->size; + $this->connection = $connection; } - /** - * @param int $size - */ - public function setSize($size) + // Extbase QueryInterface + // Current implementation covers only paginate widget support. + public function execute($returnRawQueryResult = false) { - $this->size = (int) $size; + if ($this->connection instanceof ConnectionInterface) { + return $this->connection->search($this); + } + + throw new \InvalidArgumentException( + 'Connection was not set before, therefore execute can not work. Use `setConnection` before.', + 1502197732 + ); + } + + public function setLimit($limit) + { + $this->limit = (int) $limit; + } + + public function setOffset($offset) + { + $this->offset = (int) $offset; + } + + public function getLimit() + { + return $this->limit; + } + + public function getOffset() + { + return $this->offset; + } + + public function getSource() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196146); + } + + public function setOrderings(array $orderings) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196163); + } + + public function matching($constraint) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196197); + } + + public function logicalAnd($constraint1) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196166); + } + + public function logicalOr($constraint1) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196198); + } + + public function logicalNot(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\ConstraintInterface $constraint) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196166); + } + + public function equals($propertyName, $operand, $caseSensitive = true) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196199); + } + + public function like($propertyName, $operand, $caseSensitive = true) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196167); + } + + public function contains($propertyName, $operand) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196200); + } + + public function in($propertyName, $operand) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196167); + } + + public function lessThan($propertyName, $operand) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196201); + } + + public function lessThanOrEqual($propertyName, $operand) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196168); + } + + public function greaterThan($propertyName, $operand) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196202); + } + + public function greaterThanOrEqual($propertyName, $operand) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196168); + } + + public function getType() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196203); + } + + public function setQuerySettings(\TYPO3\CMS\Extbase\Persistence\Generic\QuerySettingsInterface $querySettings) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196168); + } + + public function getQuerySettings() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196205); + } + + public function count() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196169); + } + + public function getOrderings() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196206); + } + + public function getConstraint() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196171); + } + + public function isEmpty($propertyName) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196207); + } + + public function setSource(\TYPO3\CMS\Extbase\Persistence\Generic\Qom\SourceInterface $source) + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196172); + } + + public function getStatement() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502196208); } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index ce702f9..4b3c775 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -97,8 +97,8 @@ class QueryFactory protected function addSize(SearchRequestInterface $searchRequest) { $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ - 'from' => 0, - 'size' => $searchRequest->getSize(), + 'from' => $searchRequest->getOffset(), + 'size' => $searchRequest->getLimit(), ]); } @@ -151,8 +151,8 @@ class QueryFactory 'query' => [ 'bool' => [ 'should' => $boostQueryParts, - ], - ], + ], + ], ]); } @@ -163,7 +163,7 @@ class QueryFactory 'function_score' => [ 'query' => $this->query['query'], 'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'), - ], + ], ]; } catch (InvalidArgumentException $e) { return; diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index bb8b138..114ebfe 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -69,6 +69,7 @@ class SearchService */ public function search(SearchRequestInterface $searchRequest) { + $searchRequest->setConnection($this->connection); $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); @@ -82,7 +83,7 @@ class SearchService */ protected function addSize(SearchRequestInterface $searchRequest) { - $searchRequest->setSize( + $searchRequest->setLimit( $this->configuration->getIfExists('searching.size') ?: 10 ); } diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 9ff690a..de82ebf 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -157,13 +157,19 @@ class QueryFactoryTest extends AbstractUnitTestCase ->method('get') ->will($this->throwException(new InvalidArgumentException)); $searchRequest = new SearchRequest('SearchWord'); - $searchRequest->setSize(45); + $searchRequest->setLimit(45); + $searchRequest->setOffset(35); $query = $this->subject->create($searchRequest); $this->assertSame( 45, $query->toArray()['size'], - 'Size was not added to query.' + 'Limit was not added to query.' + ); + $this->assertSame( + 35, + $query->toArray()['from'], + 'From was not added to query.' ); } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 6a90a3c..046f751 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -67,7 +67,7 @@ class SearchServiceTest extends AbstractUnitTestCase $this->connection->expects($this->once()) ->method('search') ->with($this->callback(function ($searchRequest) { - return $searchRequest->getSize() === 45; + return $searchRequest->getLimit() === 45; })); $searchRequest = new SearchRequest('SearchWord'); @@ -86,7 +86,7 @@ class SearchServiceTest extends AbstractUnitTestCase $this->connection->expects($this->once()) ->method('search') ->with($this->callback(function ($searchRequest) { - return $searchRequest->getSize() === 10; + return $searchRequest->getLimit() === 10; })); $searchRequest = new SearchRequest('SearchWord'); From c5766f5b1269c77140cb8d5fa117546381837706 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 10 Aug 2017 08:54:36 +0200 Subject: [PATCH 058/158] BUGFIX: Use fresh query for each creation This prevents issues with modifying an build query. --- Classes/Domain/Search/QueryFactory.php | 62 ++++++++++++++------------ 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index ce702f9..4edc7b0 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -39,11 +39,6 @@ class QueryFactory */ protected $configuration; - /** - * @var array - */ - protected $query = []; - /** * @param \TYPO3\CMS\Core\Log\LogManager $logManager * @param ConfigurationContainerInterface $configuration @@ -77,26 +72,28 @@ class QueryFactory */ protected function createElasticaQuery(SearchRequestInterface $searchRequest) { - $this->addSize($searchRequest); - $this->addSearch($searchRequest); - $this->addBoosts($searchRequest); - $this->addFilter($searchRequest); - $this->addFacets($searchRequest); + $query = []; + $this->addSize($searchRequest, $query); + $this->addSearch($searchRequest, $query); + $this->addBoosts($searchRequest, $query); + $this->addFilter($searchRequest, $query); + $this->addFacets($searchRequest, $query); // Use last, as it might change structure of query. // Better approach would be something like DQL to generate query and build result in the end. - $this->addFactorBoost(); + $this->addFactorBoost($query); - $this->logger->debug('Generated elasticsearch query.', [$this->query]); - return new \Elastica\Query($this->query); + $this->logger->debug('Generated elasticsearch query.', [$query]); + return new \Elastica\Query($query); } /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addSize(SearchRequestInterface $searchRequest) + protected function addSize(SearchRequestInterface $searchRequest, array &$query) { - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'from' => 0, 'size' => $searchRequest->getSize(), ]); @@ -104,19 +101,20 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addSearch(SearchRequestInterface $searchRequest) + protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { - $this->query = ArrayUtility::setValueByPath( - $this->query, + $query = ArrayUtility::setValueByPath( + $query, 'query.bool.must.0.match._all.query', $searchRequest->getSearchTerm() ); $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); if ($minimumShouldMatch) { - $this->query = ArrayUtility::setValueByPath( - $this->query, + $query = ArrayUtility::setValueByPath( + $query, 'query.bool.must.0.match._all.minimum_should_match', $minimumShouldMatch ); @@ -125,8 +123,9 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addBoosts(SearchRequestInterface $searchRequest) + protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { try { $fields = $this->configuration->get('searching.boost'); @@ -147,7 +146,7 @@ class QueryFactory ]; } - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ 'should' => $boostQueryParts, @@ -156,12 +155,15 @@ class QueryFactory ]); } - protected function addFactorBoost() + /** + * @param array &$query + */ + protected function addFactorBoost(array &$query) { try { - $this->query['query'] = [ + $query['query'] = [ 'function_score' => [ - 'query' => $this->query['query'], + 'query' => $query['query'], 'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'), ], ]; @@ -172,8 +174,9 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addFilter(SearchRequestInterface $searchRequest) + protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { return; @@ -188,7 +191,7 @@ class QueryFactory ]; } - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ 'filter' => $terms, @@ -199,11 +202,12 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addFacets(SearchRequestInterface $searchRequest) + protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'aggs' => [ $facet->getIdentifier() => [ 'terms' => [ From 51863c9e5daacfe662bbc7676a9383f9ae11eca2 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 10 Aug 2017 08:59:48 +0200 Subject: [PATCH 059/158] TASK: Cleanup PR issues --- Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php | 2 +- composer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 58c03b4..7fb0280 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -64,7 +64,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause =$this->subject->getWhereClause(); + $whereClause = $this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0', $whereClause->getStatement() diff --git a/composer.json b/composer.json index f56b60a..0061751 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,7 @@ }, "require" : { "php": ">=7.1.0", - "typo3/cms": "~8.2", + "typo3/cms": "~8.7", "ruflin/elastica": "~3.2" }, "require-dev": { From 416e49026ea2500fa32848183528c4e066f73f51 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 10 Aug 2017 09:05:20 +0200 Subject: [PATCH 060/158] TASK: Break line exceeding max line length --- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 488f949..dd3df5f 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -275,7 +275,9 @@ class TcaTableService foreach ($rootline as $pageInRootLine) { // Check configured black list if present. - if ($this->isBlackListedRootLineConfigured() && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine())) { + if ($this->isBlackListedRootLineConfigured() + && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine()) + ) { return true; } if ($pageInRootLine['extendToSubpages'] && ( From f311357d0eea0bc8eb8c8e89d58dc07a45a006ad Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 15 Aug 2017 08:30:23 +0200 Subject: [PATCH 061/158] TASK: Fix indentation --- Classes/Domain/Search/QueryFactory.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index ba4bc40..bdcd6f6 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -150,8 +150,8 @@ class QueryFactory 'query' => [ 'bool' => [ 'should' => $boostQueryParts, - ], - ], + ], + ], ]); } @@ -165,7 +165,7 @@ class QueryFactory 'function_score' => [ 'query' => $query['query'], 'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'), - ], + ], ]; } catch (InvalidArgumentException $e) { return; From 040206c95d90e97dbd23334dc48f85de1a6bd16d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 15 Aug 2017 09:21:04 +0200 Subject: [PATCH 062/158] FEATURE: Respect further root line cases Respect the following situations during indexing: - Page is not reachable due to broken root line. - Page is not reachable due to being below a recycler. --- .../Index/TcaIndexer/TcaTableService.php | 36 ++++++++++++++++--- .../Indexing/PagesIndexer/BrokenRootLine.xml | 20 +++++++++++ .../Indexing/PagesIndexer/Recycler.xml | 21 +++++++++++ .../Functional/Indexing/PagesIndexerTest.php | 15 ++++++-- 4 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml create mode 100644 Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dd3df5f..70bb52d 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -258,18 +258,27 @@ class TcaTableService * Checks whether the given record was blacklisted by root line. * This can be configured by typoscript as whole root lines can be black listed. * + * Also further TYPO3 mechanics are taken into account. Does a valid root + * line exist, is page inside a recycler, is inherited start- endtime + * excluded, etc. + * * @param array &$record * @return bool */ protected function isRecordBlacklistedByRootline(array &$record) { - // If no rootline exists, the record is on a unreachable page and therefore blacklisted. - if ($record['pid'] == 0) { - return false; + $pageUid = $record['pid']; + if ($this->tableName === 'pages') { + $pageUid = $record['uid']; } - $rootline = $this->objectManager->get(RootlineUtility::class, $record['pid'])->get(); - if (!isset($rootline[0])) { + try { + $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get(); + } catch (\RuntimeException $e) { + $this->logger->notice( + sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()), + [$record, $e] + ); return true; } @@ -278,12 +287,29 @@ class TcaTableService if ($this->isBlackListedRootLineConfigured() && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine()) ) { + $this->logger->info( + sprintf( + 'Record %u is black listed due to configured root line configuration of page %u.', + $record['uid'], + $pageInRootLine['uid'] + ), + [$record, $pageInRootLine] + ); return true; } + if ($pageInRootLine['extendToSubpages'] && ( ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) )) { + $this->logger->info( + sprintf( + 'Record %u is black listed due to configured timing of parent page %u.', + $record['uid'], + $pageInRootLine['uid'] + ), + [$record, $pageInRootLine] + ); return true; } } diff --git a/Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml b/Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml new file mode 100644 index 0000000..81f2316 --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml @@ -0,0 +1,20 @@ + + + + + 3 + 2 + Some disabled page due broken root line + + + 4 + 3 + Some disabled page due to parent pages root line being broken + + + + 6 + 1 + Some enabled page due valid root line + + diff --git a/Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml b/Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml new file mode 100644 index 0000000..1421ad5 --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml @@ -0,0 +1,21 @@ + + + + + 2 + 1 + Some disabled page due being recycler + 255 + + + 3 + 2 + Some disabled page due to parent page being recycler + + + + 6 + 1 + Some enabled page due to no be below recycler + + diff --git a/Tests/Functional/Indexing/PagesIndexerTest.php b/Tests/Functional/Indexing/PagesIndexerTest.php index 2f00d2f..244413d 100644 --- a/Tests/Functional/Indexing/PagesIndexerTest.php +++ b/Tests/Functional/Indexing/PagesIndexerTest.php @@ -64,10 +64,12 @@ class PagesIndexerTest extends AbstractFunctionalTestCase /** * @test + * @dataProvider rootLineDataSets + * @param string $dataSetPath */ - public function inheritedTimingIsRespectedDuringIndexing() + public function rootLineIsRespectedDuringIndexing($dataSetPath) { - $this->importDataSet('Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml'); + $this->importDataSet($dataSetPath); $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); $tableName = 'pages'; @@ -90,4 +92,13 @@ class PagesIndexerTest extends AbstractFunctionalTestCase $this->inject($indexer, 'connection', $connection); $indexer->indexAllDocuments(); } + + public function rootLineDataSets() + { + return [ + 'Broken root line' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/BrokenRootLine.xml'], + 'Recycler doktype' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/Recycler.xml'], + 'Extended timing to sub pages' => ['Tests/Functional/Fixtures/Indexing/PagesIndexer/InheritedTiming.xml'], + ]; + } } From 9617733826155086d6cb8694ab9f5fe317efc023 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 15 Aug 2017 09:27:27 +0200 Subject: [PATCH 063/158] BUGFIX: Fix accessing non existing property --- Classes/Domain/Search/QueryFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index bdcd6f6..60f22e2 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -93,7 +93,7 @@ class QueryFactory */ protected function addSize(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'from' => $searchRequest->getOffset(), 'size' => $searchRequest->getLimit(), ]); From 4b26799e7f86e7fcf0fdf91eb347eeb5f42afc70 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 15 Aug 2017 09:42:18 +0200 Subject: [PATCH 064/158] BUGFIX: Do not test with master of TYPO3 This version only supports V8 of TYPO3 CMS. --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d60175..4490198 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,15 +23,6 @@ env: - typo3DatabaseHost="127.0.0.1" - typo3DatabaseUsername="travis" - typo3DatabasePassword="" - matrix: - - TYPO3_VERSION="~8" - - TYPO3_VERSION="dev-master" - -matrix: - fast_finish: true - allow_failures: - - env: TYPO3_VERSION="dev-master" - php: 7.1 matrix: fast_finish: true From fe754964fe0319d9f3d9a51022882b4108a98d28 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 21 Aug 2017 12:10:34 +0200 Subject: [PATCH 065/158] BUGFIX: Fetch record to update --- Classes/Domain/Index/TcaIndexer.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 25bef53..b5cb766 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -83,7 +83,10 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecord($identifier) { - $record = $this->getQuery()->execute()->fetch(); + $query = $this->getQuery(); + $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier); + $record = $query->execute()->fetch(); + if ($record === false || $record === null) { throw new NoRecordFoundException( From efc2fb7da60448aa760671fab71d4887db84bb10 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 25 Aug 2017 11:46:46 +0200 Subject: [PATCH 066/158] BUGFIX: Remove pr issue --- Classes/Domain/Index/TcaIndexer.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index b5cb766..c35b9ea 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -87,7 +87,6 @@ class TcaIndexer extends AbstractIndexer $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier); $record = $query->execute()->fetch(); - if ($record === false || $record === null) { throw new NoRecordFoundException( 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.', From b31f315ec46176ca5d95dd0300f3bdf344c98d8f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 6 Sep 2017 22:38:46 +0200 Subject: [PATCH 067/158] BUGFIX: Allow iteration / pagination of result items Implement necessary logic based on mapped result items, not elastica result items. --- .../Connection/Elasticsearch/SearchResult.php | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 46ffb14..f45f02f 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -48,6 +48,13 @@ class SearchResult implements SearchResultInterface */ protected $results = []; + /** + * For Iterator interface. + * + * @var int + */ + protected $position = 0; + /** * @var ObjectManagerInterface */ @@ -121,27 +128,29 @@ class SearchResult implements SearchResultInterface // Iterator - Interface public function current() { - return $this->result->current(); + return $this->getResults()[$this->position]; } public function next() { - return $this->result->next(); + ++$this->position; + + return $this->current(); } public function key() { - return $this->result->key(); + return $this->position; } public function valid() { - return $this->result->valid(); + return isset($this->getResults()[$this->position]); } public function rewind() { - $this->result->rewind(); + $this->position = 0; } // Extbase QueryResultInterface - Implemented to support Pagination of Fluid. From be752485176938ac302dd9d88367b6c6fe026fce Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 15 Sep 2017 21:35:52 +0200 Subject: [PATCH 068/158] FEATURE: Allow configured filters Add new feature to allow pre configured filters. The filters will be applied to all requests. --- Classes/Domain/Search/SearchService.php | 18 ++++ Documentation/source/configuration.rst | 17 ++++ .../Unit/Domain/Search/SearchServiceTest.php | 86 +++++++++++++++++++ 3 files changed, 121 insertions(+) diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 114ebfe..384f6aa 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -72,6 +72,7 @@ class SearchService $searchRequest->setConnection($this->connection); $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); + $this->addConfiguredFilters($searchRequest); return $this->connection->search($searchRequest); } @@ -113,4 +114,21 @@ class SearchService )); } } + + /** + * Add filters from configuration, e.g. flexform or TypoScript. + * + * @param SearchRequestInterface $searchRequest + */ + protected function addConfiguredFilters(SearchRequestInterface $searchRequest) + { + try { + $searchRequest->setFilter(array_merge( + $searchRequest->getFilter(), + $this->configuration->get('searching.filter') + )); + } catch (InvalidArgumentException $e) { + // Nothing todo, no filter configured. + } + } } diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 8ea8e29..c4677e1 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -279,6 +279,23 @@ Searching The above example will provide a facet with options for all found ``CType`` results together with a count. +.. _filter: + +``filter`` +""""""""""" + + Used by: While building search request. + + Define filter that should be set for all requests. + + Example:: + + plugin.tx_searchcore.settings.searching.filter { + property = value + } + + For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. + .. _minimumShouldMatch: ``minimumShouldMatch`` diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 046f751..9149e51 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -21,6 +21,7 @@ namespace Copyright\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\SearchService; @@ -64,6 +65,10 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(45, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) ->method('search') ->with($this->callback(function ($searchRequest) { @@ -83,6 +88,10 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) ->method('search') ->with($this->callback(function ($searchRequest) { @@ -92,4 +101,81 @@ class SearchServiceTest extends AbstractUnitTestCase $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); } + + /** + * @test + */ + public function configuredFilterAreAddedToRequestWithoutAnyFilter() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->willReturn(['property' => 'something']); + + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getFilter() === ['property' => 'something']; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $this->subject->search($searchRequest); + } + + /** + * @test + */ + public function configuredFilterAreAddedToRequestWithExistingFilter() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->willReturn(['property' => 'something']); + + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getFilter() === [ + 'anotherProperty' => 'anything', + 'property' => 'something', + ]; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter(['anotherProperty' => 'anything']); + $this->subject->search($searchRequest); + } + + /** + * @test + */ + public function nonConfiguredFilterIsNotChangingRequestWithExistingFilter() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->will($this->throwException(new InvalidArgumentException)); + + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter(['anotherProperty' => 'anything']); + $this->subject->search($searchRequest); + } } From 4de18289052beca9bf1ac9acfb77d464df1b6208 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 15 Sep 2017 21:36:52 +0200 Subject: [PATCH 069/158] FIX: Fix phpcs issues with annotations --- Classes/Domain/Search/QueryFactory.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 60f22e2..f3efd68 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -89,7 +89,7 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest - * @param array &$query + * @param array $query */ protected function addSize(SearchRequestInterface $searchRequest, array &$query) { @@ -101,7 +101,7 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest - * @param array &$query + * @param array $query */ protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { @@ -123,7 +123,7 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest - * @param array &$query + * @param array $query */ protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { @@ -156,7 +156,7 @@ class QueryFactory } /** - * @param array &$query + * @param array $query */ protected function addFactorBoost(array &$query) { @@ -174,7 +174,7 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest - * @param array &$query + * @param array $query */ protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { @@ -202,7 +202,7 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest - * @param array &$query + * @param array $query */ protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { From 13004e86f2694aa576cb99c08bbe3f9c3279d6a3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 15 Sep 2017 21:54:47 +0200 Subject: [PATCH 070/158] FEATURE: Allow filter mode by not forcing a search term --- Classes/Domain/Model/SearchRequest.php | 2 +- Classes/Domain/Search/QueryFactory.php | 4 ++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 7d6436c..39e477c 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -66,7 +66,7 @@ class SearchRequest implements SearchRequestInterface /** * @param string $query */ - public function __construct($query) + public function __construct($query = '') { $this->query = (string) $query; } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index f3efd68..9455f80 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -105,6 +105,10 @@ class QueryFactory */ protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { + if (trim($searchRequest->getSearchTerm()) === '') { + return; + } + $query = ArrayUtility::setValueByPath( $query, 'query.bool.must.0.match._all.query', diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index de82ebf..54fdc2a 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -324,4 +324,23 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Boosts were not added to query.' ); } + + /** + * @test + */ + public function emptySearchStringWillNotAddSearchToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertInstanceOf( + stdClass, + $query->toArray()['query']['match_all'], + 'Empty search request does not create expected query.' + ); + } } From 9a0d73f1c784ea09dd57ac3a9a0bcacce10a31ac Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 15 Sep 2017 22:26:52 +0200 Subject: [PATCH 071/158] FEATURE: Allow to switch from search to filter mode --- Classes/Controller/SearchController.php | 11 +++ Documentation/source/configuration.rst | 17 ++++ .../Unit/Controller/SearchControllerTest.php | 99 +++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 Tests/Unit/Controller/SearchControllerTest.php diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index 068f9a1..a02cd80 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -44,6 +44,17 @@ class SearchController extends ActionController parent::__construct(); } + public function initializeSearchAction() + { + if (isset($this->settings['searching']['mode']) && $this->settings['searching']['mode'] === 'filter' + && $this->request->hasArgument('searchRequest') === false + ) { + $this->request->setArguments([ + 'searchRequest' => $this->objectManager->get(SearchRequest::class), + ]); + } + } + /** * Process a search and deliver original request and result to view. * diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index c4677e1..549ca68 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -346,3 +346,20 @@ Searching factor = 2 missing = 1 } + +.. _mode: + +``mode`` +"""""""" + + Used by: Controller while preparing action. + + Define to switch from search to filter mode. + + Example:: + + plugin.tx_searchcore.settings.searching { + mode = filter + } + + Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. diff --git a/Tests/Unit/Controller/SearchControllerTest.php b/Tests/Unit/Controller/SearchControllerTest.php new file mode 100644 index 0000000..fa38665 --- /dev/null +++ b/Tests/Unit/Controller/SearchControllerTest.php @@ -0,0 +1,99 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Controller\SearchController; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Extbase\Mvc\Web\Request; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class SearchControllerTest extends AbstractUnitTestCase +{ + /** + * @var SearchController + */ + protected $subject; + + /** + * @var Request + */ + protected $request; + + public function setUp() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( + \TYPO3\CMS\Core\Cache\CacheManager::class + )->setCacheConfigurations([ + 'extbase_object' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + 'extbase_datamapfactory_datamap' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + ]); + + parent::setUp(); + + $searchService = $this->getMockBuilder(SearchService::class) + ->disableOriginalConstructor() + ->getMock(); + $this->request = new Request(); + + $this->subject = new SearchController($searchService); + $this->inject($this->subject, 'request', $this->request); + $this->inject($this->subject, 'objectManager', new ObjectManager()); + } + + /** + * @test + */ + public function searchRequestArgumentIsAddedIfModeIsFilterAndArgumentDoesNotExist() + { + $this->inject($this->subject, 'settings', [ + 'searching' => [ + 'mode' => 'filter', + ] + ]); + + $this->subject->initializeSearchAction(); + $this->assertInstanceOf( + SearchRequest::class, + $this->request->getArgument('searchRequest'), + 'Search request was not created.' + ); + } + + /** + * @test + */ + public function searchRequestArgumentIsNotAddedIfModeIsNotFilter() + { + $this->inject($this->subject, 'settings', ['searching' => []]); + + $this->subject->initializeSearchAction(); + $this->assertFalse( + $this->request->hasArgument('searchRequest'), + 'Search request should not exist.' + ); + } +} From a47b1c3a97751b477c0319867f89911e2985620f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 15 Sep 2017 22:29:20 +0200 Subject: [PATCH 072/158] TASK: Remove unused fields for plugin content element As we do not make use of recursion or pages, we hide the inputs. --- Configuration/TCA/Overrides/tt_content.php | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Configuration/TCA/Overrides/tt_content.php diff --git a/Configuration/TCA/Overrides/tt_content.php b/Configuration/TCA/Overrides/tt_content.php new file mode 100644 index 0000000..6a976c6 --- /dev/null +++ b/Configuration/TCA/Overrides/tt_content.php @@ -0,0 +1,3 @@ + Date: Fri, 15 Sep 2017 23:47:34 +0200 Subject: [PATCH 073/158] BUGFIX: Keep existing arguments in filter mode E.g. to support paginate widget arguments. --- Classes/Controller/SearchController.php | 9 ++++-- .../Unit/Controller/SearchControllerTest.php | 29 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index a02cd80..0fa7f73 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -49,9 +49,12 @@ class SearchController extends ActionController if (isset($this->settings['searching']['mode']) && $this->settings['searching']['mode'] === 'filter' && $this->request->hasArgument('searchRequest') === false ) { - $this->request->setArguments([ - 'searchRequest' => $this->objectManager->get(SearchRequest::class), - ]); + $this->request->setArguments(array_merge( + $this->request->getArguments(), + [ + 'searchRequest' => $this->objectManager->get(SearchRequest::class), + ] + )); } } diff --git a/Tests/Unit/Controller/SearchControllerTest.php b/Tests/Unit/Controller/SearchControllerTest.php index fa38665..67c6d98 100644 --- a/Tests/Unit/Controller/SearchControllerTest.php +++ b/Tests/Unit/Controller/SearchControllerTest.php @@ -83,6 +83,35 @@ class SearchControllerTest extends AbstractUnitTestCase ); } + /** + * @test + */ + public function searchRequestArgumentIsAddedToExistingArguments() + { + $this->request->setArguments([ + '@widget_0' => [ + 'currentPage' => '7', + ] + ]); + $this->inject($this->subject, 'settings', [ + 'searching' => [ + 'mode' => 'filter', + ] + ]); + + $this->subject->initializeSearchAction(); + $this->assertInstanceOf( + SearchRequest::class, + $this->request->getArgument('searchRequest'), + 'Search request was not created.' + ); + $this->assertSame( + ['currentPage' => '7'], + $this->request->getArgument('@widget_0'), + 'Existing arguments were not kept.' + ); + } + /** * @test */ From fafa919f37155aa1a679a9154bcc07fea9c6a99f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 16 Sep 2017 20:50:03 +0200 Subject: [PATCH 074/158] WIP|FEATURE: Basic hardcoded implementation --- Classes/DataProcessing/GeoPointProcessing.php | 34 +++++++++++++++++++ Classes/DataProcessing/ProcessorInterface.php | 34 +++++++++++++++++++ .../Index/TcaIndexer/TcaTableService.php | 20 ++++++++++- Classes/Domain/Model/SearchRequest.php | 17 +++++++++- Classes/Domain/Search/QueryFactory.php | 34 +++++++++++++++---- 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 Classes/DataProcessing/GeoPointProcessing.php create mode 100644 Classes/DataProcessing/ProcessorInterface.php diff --git a/Classes/DataProcessing/GeoPointProcessing.php b/Classes/DataProcessing/GeoPointProcessing.php new file mode 100644 index 0000000..cbf6589 --- /dev/null +++ b/Classes/DataProcessing/GeoPointProcessing.php @@ -0,0 +1,34 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class GeoPointProcessing implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + $record[$configuration['to']] = [ + 'lat' => (float) $record[$configuration['fields']['lat']], + 'lon' => (float) $record[$configuration['fields']['lon']], + ]; + + return $record; + } +} diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php new file mode 100644 index 0000000..4dc0794 --- /dev/null +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -0,0 +1,34 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * All DataProcessing Processors should implement this interface, otherwise they + * will not be executed. + */ +interface ProcessorInterface +{ + /** + * Processes the given record. + * Also retrieves the configuration for this processor instance. + */ + public function processRecord(array $record, array $configuration) : array; +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..6b9024f 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -146,6 +147,17 @@ class TcaTableService { $this->relationResolver->resolveRelationsForRecord($this, $record); + try { + foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { + $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidConfigurationArgumentException $e) { + // Nothing to do. + } + if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } @@ -181,7 +193,7 @@ class TcaTableService array_filter( array_keys($this->tca['columns']), function ($columnName) { - return !$this->isSystemField($columnName); + return !$this->isSystemField($columnName) && !$this->isUserField($columnName); } ) ); @@ -249,6 +261,12 @@ class TcaTableService return in_array($columnName, $systemFields); } + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + /** * @param string $columnName * @return array diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 39e477c..fd76062 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,22 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter(array_map('strval', $filter)); + $this->filter = array_filter($filter); + + $this->mapGeoDistanceFilter(); + } + + public function mapGeoDistanceFilter() + { + if (isset($this->filter['geo_distance'])) { + foreach (array_keys($this->filter['geo_distance']) as $config) { + if (strpos($config, '_') !== false) { + $newKey = str_replace('_', '.', $config); + $this->filter['geo_distance'][$newKey] = $this->filter['geo_distance'][$config]; + unset($this->filter['geo_distance'][$config]); + } + } + } } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9455f80..9f2f68a 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -164,6 +164,17 @@ class QueryFactory */ protected function addFactorBoost(array &$query) { + $query['sort'] = [ + '_geo_distance' => [ + 'location' => [ + 'lat' => 51.2014392, + 'lon' => 6.4302962, + ], + 'order' => 'asc', + 'unit' => 'km', + 'distance_type' => 'plane', + ] + ]; try { $query['query'] = [ 'function_score' => [ @@ -186,24 +197,33 @@ class QueryFactory return; } - $terms = []; + $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $terms[] = [ - 'term' => [ - $name => $value, - ], - ]; + $filter[] = $this->buildFilter($name, $value); } $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ - 'filter' => $terms, + 'filter' => $filter, ], ], ]); } + protected function buildFilter(string $name, $value) : array + { + if ($name === 'geo_distance') { + return [$name => $value]; + } + + return [ + 'term' => [ + $name => $value, + ], + ]; + } + /** * @param SearchRequestInterface $searchRequest * @param array $query From efeb5d1e07bdc17884c670382e757b65192b7d2d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 13:02:48 +0200 Subject: [PATCH 075/158] FEATURE: Add data processing to extension Allow integrators / developer to apply data processing concept known from FLUIDTEMPLATE to indexing. --- Classes/DataProcessing/CopyToProcessor.php | 54 +++ Classes/DataProcessing/ProcessorInterface.php | 39 ++ .../Index/TcaIndexer/TcaTableService.php | 13 + Documentation/source/concepts.rst | 10 + Documentation/source/conf.py | 1 + Documentation/source/configuration.rst | 334 +----------------- .../source/configuration/connections.rst | 55 +++ .../source/configuration/indexing.rst | 202 +++++++++++ .../source/configuration/searching.rst | 125 +++++++ Documentation/source/features.rst | 2 +- Documentation/source/indexer.rst | 18 +- .../ProcessesAllowedTablesTest.php | 2 +- .../DataProcessing/CopyToProcessorTest.php | 85 +++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 43 +++ 14 files changed, 652 insertions(+), 331 deletions(-) create mode 100644 Classes/DataProcessing/CopyToProcessor.php create mode 100644 Classes/DataProcessing/ProcessorInterface.php create mode 100644 Documentation/source/configuration/connections.rst create mode 100644 Documentation/source/configuration/indexing.rst create mode 100644 Documentation/source/configuration/searching.rst create mode 100644 Tests/Unit/DataProcessing/CopyToProcessorTest.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php new file mode 100644 index 0000000..a252a9a --- /dev/null +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -0,0 +1,54 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Copies values from one field to another one. + */ +class CopyToProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) + { + $all = []; + + $this->addArray($all, $record); + $all = array_filter($all); + $record[$configuration['to']] = implode(PHP_EOL, $all); + + return $record; + } + + /** + * @param array &$target + * @param array $from + */ + protected function addArray(array &$target, array $from) + { + foreach ($from as $value) { + if (is_array($value)) { + $this->addArray($target, $value); + continue; + } + + $target[] = (string) $value; + } + } +} diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php new file mode 100644 index 0000000..1c78be3 --- /dev/null +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -0,0 +1,39 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * All DataProcessing Processors should implement this interface, otherwise they + * will not be executed. + */ +interface ProcessorInterface +{ + /** + * Processes the given record. + * Also retrieves the configuration for this processor instance. + * + * @param array $record + * @param array $configuration + * + * @return array + */ + public function processRecord(array $record, array $configuration); +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..a0a9182 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -146,6 +148,17 @@ class TcaTableService { $this->relationResolver->resolveRelationsForRecord($this, $record); + try { + foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { + $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidConfigurationArgumentException $e) { + // Nothing to do. + } + if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index 0ce3a0e..0fe6c5d 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -28,3 +28,13 @@ The indexing is done by one of the available indexer. For each identifier it's p the indexer to use. Also it's possible to write custom indexer to use. Currently only the :ref:`TcaIndexer` is provided. + +.. _concepts_indexing_dataprocessing: + +DataProcessing +^^^^^^^^^^^^^^ + +Before data is transfered to search service, it can be processed by "DataProcessors" like already +known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`. + +Configuration is done through TypoScript, see :ref:`dataProcessing`. diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index a708209..1419689 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -304,6 +304,7 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { 't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None), + 't3tsref': ('https://docs.typo3.org/typo3cms/TyposcriptReference/', None), } extlinks = { 'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '), diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 549ca68..eca5ba4 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -36,330 +36,14 @@ Here is the example default configuration that's provided through static include Options ------- -The following section contains the different options, e.g. for :ref:`connections` and -:ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or -``plugin.tx_searchcore.settings.indexing``. +The following sections contains the different options grouped by their applied area, e.g. for +:ref:`connections` and :ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or +``plugin.tx_searchcore.settings.indexing``: -.. _configuration_options_connection: +.. toctree:: + :maxdepth: 1 + :glob: -connections -^^^^^^^^^^^ - -Holds settings regarding the different possible connections for search services like Elasticsearch -or Solr. - -Configured as:: - - plugin { - tx_searchcore { - settings { - connections { - connectionName { - // the settings - } - } - } - } - } - -Where ``connectionName`` is one of the available :ref:`connections`. - -The following settings are available. For each setting its documented which connection consumes it. - -.. _host: - -``host`` -"""""""" - - Used by: :ref:`Elasticsearch`. - - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. - - Example:: - - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost - -.. _port: - -``port`` -"""""""" - - Used by: :ref:`Elasticsearch`. - - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - - Example:: - - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 - - -.. _configuration_options_index: - -Indexing -^^^^^^^^ - -Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. - -Configured as:: - - plugin { - tx_searchcore { - settings { - indexing { - identifier { - indexer = FullyQualifiedClassname - // the settings - } - } - } - } - } - -Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. - -The following settings are available. For each setting its documented which indexer consumes it. - -.. _rootLineBlacklist: - -``rootLineBlacklist`` -""""""""""""""""""""" - - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. - - Contains a comma separated list of page uids. Spaces are trimmed. - - Example:: - - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 - -Also it's possible to define some behaviour for the different document types. In context of TYPO3 -tables are used as document types 1:1. It's possible to configure different tables. The following -options are available: - -.. _additionalWhereClause: - -``additionalWhereClause`` -""""""""""""""""""""""""" - - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. - - Example:: - - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - - .. attention:: - - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. - -.. _abstractFields: - -``abstractFields`` -""""""""""""""""""""""""" - - Used by: :ref:`PagesIndexer`. - - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. - - Example:: - - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - - Default:: - - abstract, description, bodytext - -.. _mapping: - -``mapping`` -""""""""""" - - Used by: Elasticsearch connection while indexing. - - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. - - Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } - } - - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. - - -.. _index: - -``index`` -""""""""" - - Used by: Elasticsearch connection while indexing. - - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - - Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } - } - - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } - } - } - } - - ``char_filter`` and ``filter`` are a comma separated list of options. - -.. _configuration_options_search: - -Searching -^^^^^^^^^ - -.. _size: - -``size`` -"""""""" - - Used by: Elasticsearch connection while building search query. - - Defined how many search results should be fetched to be available in search result. - - Example:: - - plugin.tx_searchcore.settings.searching.size = 50 - - Default if not configured is 10. - -.. _facets: - -``facets`` -""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. - - Example:: - - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } - } - - The above example will provide a facet with options for all found ``CType`` results together - with a count. - -.. _filter: - -``filter`` -""""""""""" - - Used by: While building search request. - - Define filter that should be set for all requests. - - Example:: - - plugin.tx_searchcore.settings.searching.filter { - property = value - } - - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. - -.. _minimumShouldMatch: - -``minimumShouldMatch`` -"""""""""""""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - - Example:: - - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% - -.. _boost: - -``boost`` -""""""""" - - Used by: Elasticsearch connection while building search query. - - Define fields that should boost the score for results. - - Example:: - - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } - - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html - -.. _fieldValueFactor: - -``fieldValueFactor`` -"""""""""""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - - Example:: - - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 - } - -.. _mode: - -``mode`` -"""""""" - - Used by: Controller while preparing action. - - Define to switch from search to filter mode. - - Example:: - - plugin.tx_searchcore.settings.searching { - mode = filter - } - - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. + configuration/connections + configuration/indexing + configuration/searching diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst new file mode 100644 index 0000000..841d878 --- /dev/null +++ b/Documentation/source/configuration/connections.rst @@ -0,0 +1,55 @@ +.. _configuration_options_connection: + +Connections +=========== + +Holds settings regarding the different possible connections for search services like Elasticsearch +or Solr. + +Configured as:: + + plugin { + tx_searchcore { + settings { + connections { + connectionName { + // the settings + } + } + } + } + } + +Where ``connectionName`` is one of the available :ref:`connections`. + +The following settings are available. For each setting its documented which connection consumes it. + +.. _host: + +``host`` +-------- + + Used by: :ref:`Elasticsearch`. + + The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 + installation. + + Example:: + + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + +.. _port: + +``port`` +-------- + + Used by: :ref:`Elasticsearch`. + + The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. + + Example:: + + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + + + diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst new file mode 100644 index 0000000..24c037f --- /dev/null +++ b/Documentation/source/configuration/indexing.rst @@ -0,0 +1,202 @@ +.. _configuration_options_index: + +Indexing +======== + +Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. + +Configured as:: + + plugin { + tx_searchcore { + settings { + indexing { + identifier { + indexer = FullyQualifiedClassname + // the settings + } + } + } + } + } + +Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. + +The following settings are available. For each setting its documented which indexer consumes it. + +.. _rootLineBlacklist: + +rootLineBlacklist +----------------- + + Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. + + Defines a blacklist of page uids. Records below any of these pages, or subpages, are not + indexed. This allows you to define areas that should not be indexed. + The page attribute *No Search* is also taken into account to prevent indexing records from only one + page without recursion. + + Contains a comma separated list of page uids. Spaces are trimmed. + + Example:: + + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + +Also it's possible to define some behaviour for the different document types. In context of TYPO3 +tables are used as document types 1:1. It's possible to configure different tables. The following +options are available: + +.. _additionalWhereClause: + +additionalWhereClause +--------------------- + + Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. + + Add additional SQL to where clauses to determine indexable records from the table. This way you + can exclude specific records like ``tt_content`` records with specific ``CType`` values or + something else. E.g. you can add a new field to the table to exclude records from indexing. + + Example:: + + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + + .. attention:: + + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. + +.. _abstractFields: + +abstractFields +-------------- + + Used by: :ref:`PagesIndexer`. + + Define which field should be used to provide the auto generated field "search_abstract". + The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also + possible. + + Example:: + + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + + Default:: + + abstract, description, bodytext + +.. _mapping: + +mapping +------- + + Used by: Elasticsearch connection while indexing. + + Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html + You are able to define the mapping for each property / columns. + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword + } + } + + The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This + makes building a facet possible. + +.. _index: + +index +----- + + Used by: Elasticsearch connection while indexing. + + Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding + } + } + + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 + } + } + } + } + + ``char_filter`` and ``filter`` are a comma separated list of options. + +.. _dataProcessing: + +dataProcessing +-------------- + + Used by: All connections while indexing. + + Configure modifications on each document before sending it to the configured connection. Same as + provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through + :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + + All processors are applied in configured order. Allowing to work with already processed data. + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + + The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards + all fields, including ``search_spellcheck`` will be copied to ``search_all``. + E.g. used to index all information into a field for :ref:`spellchecking` or searching with + different :ref:`mapping`. + + The following Processor are available: + + ``Codappix\SearchCore\DataProcessing\CopyToProcessor`` + Will copy contents of fields to other fields + + The following Processor are planned: + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + + Of course you are able to provide further processors. Just implement + ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified + class name) as done in the examples above. + + By implementing also the same interface as necessary for TYPO3 + :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code + also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst new file mode 100644 index 0000000..2143364 --- /dev/null +++ b/Documentation/source/configuration/searching.rst @@ -0,0 +1,125 @@ +.. _configuration_options_search: + +Searching +========= + +.. _size: + +size +---- + + Used by: Elasticsearch connection while building search query. + + Defined how many search results should be fetched to be available in search result. + + Example:: + + plugin.tx_searchcore.settings.searching.size = 50 + + Default if not configured is 10. + +.. _facets: + +facets +------ + + Used by: Elasticsearch connection while building search query. + + Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html + Currently only the term facet is provided. + + Example:: + + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType + } + } + + The above example will provide a facet with options for all found ``CType`` results together + with a count. + +.. _filter: + +``filter`` +""""""""""" + + Used by: While building search request. + + Define filter that should be set for all requests. + + Example:: + + plugin.tx_searchcore.settings.searching.filter { + property = value + } + + For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. + +.. _minimumShouldMatch: + +minimumShouldMatch +------------------ + + Used by: Elasticsearch connection while building search query. + + Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html + + Example:: + + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + +.. _boost: + +boost +----- + + Used by: Elasticsearch connection while building search query. + + Define fields that should boost the score for results. + + Example:: + + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } + + For further information take a look at + https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html + +.. _fieldValueFactor: + +fieldValueFactor +---------------- + + Used by: Elasticsearch connection while building search query. + + Define a field to use as a factor for scoring. The configuration is passed through to elastic + search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor + + Example:: + + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mode: + +``mode`` +"""""""" + + Used by: Controller while preparing action. + + Define to switch from search to filter mode. + + Example:: + + plugin.tx_searchcore.settings.searching { + mode = filter + } + + Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 669e632..199192a 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -24,7 +24,7 @@ Currently all fields are searched for a single search input. Also multiple filter are supported. Filtering results by fields for string contents. -Even facets / aggregates are now possible. Therefore a mapping has to be defined in TypoScript for +Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. .. _features_planned: diff --git a/Documentation/source/indexer.rst b/Documentation/source/indexer.rst index ddc6772..3bb10bc 100644 --- a/Documentation/source/indexer.rst +++ b/Documentation/source/indexer.rst @@ -21,12 +21,18 @@ further stuff. The indexer is configurable through the following options: -* :ref:`allowedTables` - * :ref:`rootLineBlacklist` * :ref:`additionalWhereClause` +* :ref:`abstractFields` + +* :ref:`mapping` + +* :ref:`index` + +* :ref:`dataProcessing` + .. _PagesIndexer: PagesIndexer @@ -42,14 +48,18 @@ improve search. The indexer is configurable through the following options: -* :ref:`allowedTables` - * :ref:`rootLineBlacklist` * :ref:`additionalWhereClause` * :ref:`abstractFields` +* :ref:`mapping` + +* :ref:`index` + +* :ref:`dataProcessing` + .. note:: Not all relations are resolved yet, see :issue:`17` and :pr:`20`. diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 7fa1cc5..6a027cf 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -100,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php new file mode 100644 index 0000000..2f9e498 --- /dev/null +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -0,0 +1,85 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class CopyToProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new CopyToProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Copy all fields to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field 2' => 'Some more content like ipsum', + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field 2' => 'Some more content like ipsum', + 'new_field' => 'Some content like lorem' . PHP_EOL . 'Some more content like ipsum', + ], + ], + 'Copy all fields with sub array to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2', + ], + ], + ]; + } +} diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 7fb0280..3088db6 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -98,4 +100,45 @@ class TcaTableServiceTest extends AbstractUnitTestCase $whereClause->getParameters() ); } + + /** + * @test + */ + public function executesConfiguredDataProcessing() + { + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('indexing.testTable.dataProcessing') + ->will($this->returnValue([ + '1' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + '2' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + ])); + + $subject = $this->getMockBuilder(TcaTableService::class) + ->disableOriginalConstructor() + ->setMethodsExcept(['prepareRecord']) + ->getMock(); + $this->inject($subject, 'configuration', $this->configuration); + $this->inject($subject, 'tableName', 'testTable'); + $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); + + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['new_test_field'] = 'test'; + $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; + + $subject->prepareRecord($record); + + $this->assertSame( + $expectedRecord, + $record, + 'Dataprocessing is not executed by TcaTableService as expected.' + ); + } } From 6c01abe5a546d77a96600cda62c6e484d94b7dfa Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 13:35:44 +0200 Subject: [PATCH 076/158] BUGFIX: Also handle data processor without configuration --- .../Index/TcaIndexer/TcaTableService.php | 9 ++++- .../Index/TcaIndexer/TcaTableServiceTest.php | 34 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index a0a9182..c46b0e8 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -150,7 +150,14 @@ class TcaTableService try { foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { - $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + $className = ''; + if (is_string($configuration)) { + $className = $configuration; + $configuration = []; + } else { + $className = $configuration['_typoScriptNodeValue']; + } + $dataProcessor = GeneralUtility::makeInstance($className); if ($dataProcessor instanceof ProcessorInterface) { $record = $dataProcessor->processRecord($record, $configuration); } diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 3088db6..e12a4af 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -104,7 +104,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase /** * @test */ - public function executesConfiguredDataProcessing() + public function executesConfiguredDataProcessingWithConfiguration() { $this->configuration->expects($this->exactly(1)) ->method('get') @@ -141,4 +141,36 @@ class TcaTableServiceTest extends AbstractUnitTestCase 'Dataprocessing is not executed by TcaTableService as expected.' ); } + + /** + * @test + */ + public function executesConfiguredDataProcessingWithoutConfiguration() + { + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('indexing.testTable.dataProcessing') + ->will($this->returnValue([CopyToProcessor::class])); + + $subject = $this->getMockBuilder(TcaTableService::class) + ->disableOriginalConstructor() + ->setMethodsExcept(['prepareRecord']) + ->getMock(); + $this->inject($subject, 'configuration', $this->configuration); + $this->inject($subject, 'tableName', 'testTable'); + $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); + + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord[''] = 'test'; + $expectedRecord['search_title'] = 'test'; + + $subject->prepareRecord($record); + + $this->assertSame( + $expectedRecord, + $record, + 'Dataprocessing is not executed by TcaTableService as expected.' + ); + } } From 2c466854b2b76b9f2a02091925f754f570159c2e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 12:27:48 +0200 Subject: [PATCH 077/158] BUGFIX: Do not add non existing db columns to fields array As TCA might contain columns which do not exist in DB, filter them out. --- .../Index/TcaIndexer/TcaTableService.php | 17 ++++++- .../ProcessesAllowedTablesTest.php | 2 +- .../Index/TcaIndexer/TcaTableServiceTest.php | 49 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..c00172d 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -181,7 +181,10 @@ class TcaTableService array_filter( array_keys($this->tca['columns']), function ($columnName) { - return !$this->isSystemField($columnName); + return !$this->isSystemField($columnName) + && !$this->isUserField($columnName) + && !$this->isPassthroughField($columnName) + ; } ) ); @@ -249,6 +252,18 @@ class TcaTableService return in_array($columnName, $systemFields); } + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + + protected function isPassthroughField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'passthrough'; + } + /** * @param string $columnName * @return array diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 7fa1cc5..6a027cf 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -100,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 7fb0280..27a53ae 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -98,4 +99,52 @@ class TcaTableServiceTest extends AbstractUnitTestCase $whereClause->getParameters() ); } + + /** + * @test + */ + public function allConfiguredAndAllowedTcaColumnsAreReturnedAsFields() + { + $GLOBALS['TCA']['test_table'] = [ + 'ctrl' => [ + 'languageField' => 'sys_language', + ], + 'columns' => [ + 'sys_language' => [], + 't3ver_oid' => [], + 'available_column' => [ + 'config' => [ + 'type' => 'input', + ], + ], + 'user_column' => [ + 'config' => [ + 'type' => 'user', + ], + ], + 'passthrough_column' => [ + 'config' => [ + 'type' => 'passthrough', + ], + ], + ], + ]; + $subject = new TcaTableService( + 'test_table', + $this->getMockBuilder(RelationResolver::class)->getMock(), + $this->configuration + ); + $this->inject($subject, 'logger', $this->getMockedLogger()); + + $this->assertSame( + [ + 'test_table.uid', + 'test_table.pid', + 'test_table.available_column', + ], + $subject->getFields(), + '' + ); + unset($GLOBALS['TCA']['test_table']); + } } From 67a43e64a5b892344fa62a66ab9626238d979c8e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 13:06:22 +0200 Subject: [PATCH 078/158] BUGFIX: Fix typos in method name and php doc --- Classes/Connection/Elasticsearch/SearchResult.php | 2 +- Classes/Connection/SearchResultInterface.php | 2 +- .../Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php | 2 +- Tests/Unit/Domain/Search/QueryFactoryTest.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index f45f02f..3919722 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -83,7 +83,7 @@ class SearchResult implements SearchResultInterface /** * Return all facets, if any. * - * @return array + * @return array */ public function getFacets() { diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index d52a8ba..60718cd 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -35,7 +35,7 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf /** * Return all facets, if any. * - * @return array + * @return array */ public function getFacets(); diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 7fa1cc5..6a027cf 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -100,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 54fdc2a..f7d0dd2 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -50,7 +50,7 @@ class QueryFactoryTest extends AbstractUnitTestCase /** * @test */ - public function creatonOfQueryWorksInGeneral() + public function creationOfQueryWorksInGeneral() { $searchRequest = new SearchRequest('SearchWord'); From e1764dca133fb9c10ee7f98d045163dbe83bcc67 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 20 Oct 2017 16:36:26 +0200 Subject: [PATCH 079/158] FEATURE: Add GeoPoint Processor Document data processors. Add test for new data processor. --- Classes/DataProcessing/CopyToProcessor.php | 6 +- ...ntProcessing.php => GeoPointProcessor.php} | 31 ++++- .../dataProcessing/CopyToProcessor.rst | 23 ++++ .../dataProcessing/GeoPointProcessor.rst | 27 +++++ .../source/configuration/indexing.rst | 8 +- .../DataProcessing/GeoPointProcessorTest.php | 113 ++++++++++++++++++ 6 files changed, 198 insertions(+), 10 deletions(-) rename Classes/DataProcessing/{GeoPointProcessing.php => GeoPointProcessor.php} (53%) create mode 100644 Documentation/source/configuration/dataProcessing/CopyToProcessor.rst create mode 100644 Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst create mode 100644 Tests/Unit/DataProcessing/GeoPointProcessorTest.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index a252a9a..eea555f 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class CopyToProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) + public function processRecord(array $record, array $configuration) : array { $all = []; @@ -36,10 +36,6 @@ class CopyToProcessor implements ProcessorInterface return $record; } - /** - * @param array &$target - * @param array $from - */ protected function addArray(array &$target, array $from) { foreach ($from as $value) { diff --git a/Classes/DataProcessing/GeoPointProcessing.php b/Classes/DataProcessing/GeoPointProcessor.php similarity index 53% rename from Classes/DataProcessing/GeoPointProcessing.php rename to Classes/DataProcessing/GeoPointProcessor.php index cbf6589..f9256b3 100644 --- a/Classes/DataProcessing/GeoPointProcessing.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -20,15 +20,40 @@ namespace Codappix\SearchCore\DataProcessing; * 02110-1301, USA. */ -class GeoPointProcessing implements ProcessorInterface +/** + * Adds a new fields, ready to use as GeoPoint field for Elasticsearch. + */ +class GeoPointProcessor implements ProcessorInterface { public function processRecord(array $record, array $configuration) : array { + if (! $this->canApply($record, $configuration)) { + return $record; + } + $record[$configuration['to']] = [ - 'lat' => (float) $record[$configuration['fields']['lat']], - 'lon' => (float) $record[$configuration['fields']['lon']], + 'lat' => (float) $record[$configuration['lat']], + 'lon' => (float) $record[$configuration['lon']], ]; return $record; } + + protected function canApply(array $record, array $configuration) : bool + { + if (!isset($record[$configuration['lat']]) + || !is_numeric($record[$configuration['lat']]) + || trim($record[$configuration['lat']]) === '' + ) { + return false; + } + if (!isset($record[$configuration['lon']]) + || !is_numeric($record[$configuration['lon']]) + || trim($record[$configuration['lon']]) === '' + ) { + return false; + } + + return true; + } } diff --git a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst new file mode 100644 index 0000000..6d83d70 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\CopyToProcessor`` +====================================================== + +Will copy contents of fields to other fields. + +Possible Options: + +``to`` + Defines the field to copy the values into. All values not false will be copied at the moment. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = all + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = spellcheck + } + } + diff --git a/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst new file mode 100644 index 0000000..a19928f --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst @@ -0,0 +1,27 @@ +``Codappix\SearchCore\DataProcessing\GeoPointProcessor`` +======================================================== + +Will create a new field, ready to use as GeoPoint field for Elasticsearch. + +Possible Options: + +``to`` + Defines the field to create as GeoPoint. +``lat`` + Defines the field containing the latitude. +``lon`` + Defines the field containing the longitude. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor + 1 { + to = location + lat = lat + lon = lng + } + } + +The above example will create a new field ``location`` as GeoPoint with latitude fetched from field +``lat`` and longitude fetched from field ``lng``. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 24c037f..413da3d 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -175,8 +175,12 @@ dataProcessing The following Processor are available: - ``Codappix\SearchCore\DataProcessing\CopyToProcessor`` - Will copy contents of fields to other fields + .. toctree:: + :maxdepth: 1 + :glob: + + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor The following Processor are planned: diff --git a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php new file mode 100644 index 0000000..99b8eb3 --- /dev/null +++ b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\GeoPointProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class GeoPointProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new GeoPointProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Create new field with existing lat and lng' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + 'location' => [ + 'lat' => 23.232, + 'lon' => 45.43, + ], + ], + ], + 'Do not create new field due to missing configuration' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + ], + 'Do not create new field due to missing lat and lon' => [ + 'record' => [ + 'lat' => '', + 'lng' => '', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => '', + 'lng' => '', + ], + ], + 'Do not create new field due to invalid lat and lon' => [ + 'record' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + ], + ]; + } +} From 636ef78a143e34461d8c43fb74e2ccefbea32598 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 11:23:28 +0200 Subject: [PATCH 080/158] WIP|TASK: Remove sort and add filter As we filter for distance and do not sort. --- Classes/Domain/Model/SearchRequest.php | 15 ---- Classes/Domain/Search/QueryFactory.php | 98 +++++++++----------------- 2 files changed, 35 insertions(+), 78 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index fd76062..1e3e652 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -93,21 +93,6 @@ class SearchRequest implements SearchRequestInterface public function setFilter(array $filter) { $this->filter = array_filter($filter); - - $this->mapGeoDistanceFilter(); - } - - public function mapGeoDistanceFilter() - { - if (isset($this->filter['geo_distance'])) { - foreach (array_keys($this->filter['geo_distance']) as $config) { - if (strpos($config, '_') !== false) { - $newKey = str_replace('_', '.', $config); - $this->filter['geo_distance'][$newKey] = $this->filter['geo_distance'][$config]; - unset($this->filter['geo_distance'][$config]); - } - } - } } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9f2f68a..6511cdd 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -39,10 +39,6 @@ class QueryFactory */ protected $configuration; - /** - * @param \TYPO3\CMS\Core\Log\LogManager $logManager - * @param ConfigurationContainerInterface $configuration - */ public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, ConfigurationContainerInterface $configuration @@ -55,22 +51,13 @@ class QueryFactory * TODO: This is not in scope Elasticsearch, therefore it should not return * \Elastica\Query, but decide to use a more specific QueryFactory like * ElasticaQueryFactory, once the second query is added? - * - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query */ - public function create(SearchRequestInterface $searchRequest) + public function create(SearchRequestInterface $searchRequest) : \Elastica\Query { return $this->createElasticaQuery($searchRequest); } - /** - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query - */ - protected function createElasticaQuery(SearchRequestInterface $searchRequest) + protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query { $query = []; $this->addSize($searchRequest, $query); @@ -83,14 +70,12 @@ class QueryFactory // Better approach would be something like DQL to generate query and build result in the end. $this->addFactorBoost($query); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($query, '$query', 8, false);die; + $this->logger->debug('Generated elasticsearch query.', [$query]); return new \Elastica\Query($query); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSize(SearchRequestInterface $searchRequest, array &$query) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -99,10 +84,6 @@ class QueryFactory ]); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { if (trim($searchRequest->getSearchTerm()) === '') { @@ -125,10 +106,6 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { try { @@ -159,22 +136,8 @@ class QueryFactory ]); } - /** - * @param array $query - */ protected function addFactorBoost(array &$query) { - $query['sort'] = [ - '_geo_distance' => [ - 'location' => [ - 'lat' => 51.2014392, - 'lon' => 6.4302962, - ], - 'order' => 'asc', - 'unit' => 'km', - 'distance_type' => 'plane', - ] - ]; try { $query['query'] = [ 'function_score' => [ @@ -187,10 +150,6 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { @@ -199,7 +158,11 @@ class QueryFactory $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $filter[] = $this->buildFilter($name, $value); + $filter[] = $this->buildFilter( + $name, + $value, + $this->configuration->getIfExists('searching.filter.' . $name) ?: [] + ); } $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -211,23 +174,6 @@ class QueryFactory ]); } - protected function buildFilter(string $name, $value) : array - { - if ($name === 'geo_distance') { - return [$name => $value]; - } - - return [ - 'term' => [ - $name => $value, - ], - ]; - } - - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { @@ -242,4 +188,30 @@ class QueryFactory ]); } } + + protected function buildFilter(string $name, $value, array $config) : array + { + if ($config === []) { + return [ + 'term' => [ + $name => $value, + ], + ]; + } + + $filter = []; + + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($name, '$name', 8, false); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($config, '$config', 8, false);die; + + if (isset($config['fields'])) { + foreach ($config['fields'] as $elasticField => $inputField) { + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false);die; + $filter[$elasticField] = $value[$inputField]; + } + } + + return [$config['field'] => $filter]; + } } From 8d343ee97fd178db5f549bc35345e8ab63bcefd5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 17:34:48 +0200 Subject: [PATCH 081/158] FEATURE: Finish filter configuration for geo_search --- Classes/Domain/Search/QueryFactory.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 6511cdd..2042775 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -70,8 +70,6 @@ class QueryFactory // Better approach would be something like DQL to generate query and build result in the end. $this->addFactorBoost($query); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($query, '$query', 8, false);die; - $this->logger->debug('Generated elasticsearch query.', [$query]); return new \Elastica\Query($query); } @@ -161,7 +159,7 @@ class QueryFactory $filter[] = $this->buildFilter( $name, $value, - $this->configuration->getIfExists('searching.filter.' . $name) ?: [] + $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: [] ); } @@ -201,13 +199,8 @@ class QueryFactory $filter = []; - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($name, '$name', 8, false); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($config, '$config', 8, false);die; - if (isset($config['fields'])) { foreach ($config['fields'] as $elasticField => $inputField) { - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false);die; $filter[$elasticField] = $value[$inputField]; } } From b1f81c0d3be781fbdc6d5fb0050b3dcd4a60a21d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 17:51:04 +0200 Subject: [PATCH 082/158] WIP|FEATURE: Add field and sorting Sort result by distance and provide distance to result items. --- Classes/Domain/Search/QueryFactory.php | 70 ++++++++++++++++++++------ 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 2042775..7fa3cd5 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -65,6 +65,8 @@ class QueryFactory $this->addBoosts($searchRequest, $query); $this->addFilter($searchRequest, $query); $this->addFacets($searchRequest, $query); + $this->addFields($searchRequest, $query); + $this->addSort($searchRequest, $query); // Use last, as it might change structure of query. // Better approach would be something like DQL to generate query and build result in the end. @@ -148,6 +150,44 @@ class QueryFactory } } + protected function addFields(SearchRequestInterface $searchRequest, array &$query) + { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => [ '_source' ], + ]); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'script_fields' => [ + 'distance' => [ + 'script' => [ + 'params' => [ + 'lat' => 51.168098, + 'lon' => 6.381384, + ], + 'lang' => 'painless', + 'inline' => 'doc["location"].arcDistance(params.lat,params.lon) * 0.001', + ], + ], + ], + ]); + } + + protected function addSort(SearchRequestInterface $searchRequest, array &$query) + { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'sort' => [ + '_geo_distance' => [ + 'location' => [ + 'lat' => 51.168098, + 'lon' => 6.381384, + ], + 'order' => 'asc', + 'unit' => 'km', + 'distance_type' => 'plane', + ], + ], + ]); + } + protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { @@ -172,21 +212,6 @@ class QueryFactory ]); } - protected function addFacets(SearchRequestInterface $searchRequest, array &$query) - { - foreach ($searchRequest->getFacets() as $facet) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'aggs' => [ - $facet->getIdentifier() => [ - 'terms' => [ - 'field' => $facet->getField(), - ], - ], - ], - ]); - } - } - protected function buildFilter(string $name, $value, array $config) : array { if ($config === []) { @@ -207,4 +232,19 @@ class QueryFactory return [$config['field'] => $filter]; } + + protected function addFacets(SearchRequestInterface $searchRequest, array &$query) + { + foreach ($searchRequest->getFacets() as $facet) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'aggs' => [ + $facet->getIdentifier() => [ + 'terms' => [ + 'field' => $facet->getField(), + ], + ], + ], + ]); + } + } } From 07a4fec622795ed959b4056c43944ad2ff01647d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 23 Oct 2017 16:35:38 +0200 Subject: [PATCH 083/158] WIP|FEATURE: Allow fields and sorting to be configurable --- Classes/Domain/Model/SearchRequest.php | 3 +- Classes/Domain/Search/QueryFactory.php | 67 ++++++++++++++------------ 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 1e3e652..d751ce8 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,8 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter($filter); + $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); + $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7fa3cd5..56d4dd3 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; +use TYPO3\CMS\Fluid\View\StandaloneView; class QueryFactory { @@ -152,40 +154,28 @@ class QueryFactory protected function addFields(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'stored_fields' => [ '_source' ], - ]); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'script_fields' => [ - 'distance' => [ - 'script' => [ - 'params' => [ - 'lat' => 51.168098, - 'lon' => 6.381384, - ], - 'lang' => 'painless', - 'inline' => 'doc["location"].arcDistance(params.lat,params.lon) * 0.001', - ], - ], - ], - ]); + try { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), + ]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } + + try { + $scriptFields = $this->configuration->get('searching.fields.script_fields'); + $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } } protected function addSort(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'sort' => [ - '_geo_distance' => [ - 'location' => [ - 'lat' => 51.168098, - 'lon' => 6.381384, - ], - 'order' => 'asc', - 'unit' => 'km', - 'distance_type' => 'plane', - ], - ], - ]); + $sorting = $this->configuration->getIfExists('searching.sort') ?: []; + $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -247,4 +237,21 @@ class QueryFactory ]); } } + + protected function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be end as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } } From 86d02f7b8d6930e32affd1058398d33756f91e45 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 26 Oct 2017 10:05:32 +0200 Subject: [PATCH 084/158] TASK: Allow integrators to use GET with minimal overhead Allow to map search request even if no trusted properties exist. Also cache initial call to plugin. This allows to use GET as submit for forms with minimal arguments in URL. --- Classes/Controller/SearchController.php | 6 ++++++ ext_localconf.php | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index 0fa7f73..e484fcd 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -56,6 +56,12 @@ class SearchController extends ActionController ] )); } + + if ($this->arguments->hasArgument('searchRequest')) { + $this->arguments->getArgument('searchRequest')->getPropertyMappingConfiguration() + ->allowAllProperties() + ; + } } /** diff --git a/ext_localconf.php b/ext_localconf.php index 54bf43b..5c42c9c 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -31,9 +31,6 @@ call_user_func( 'search', [ 'Search' => 'search' - ], - [ - 'Search' => 'search' // TODO: Enable caching. But submitting form results in previous result?! ] ); From 769bdd1003a94b06ad97579b51126d6f67109352 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 11:32:21 +0100 Subject: [PATCH 085/158] BUGFIX: Do not cache search action --- ext_localconf.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext_localconf.php b/ext_localconf.php index 5c42c9c..c52d05d 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -29,6 +29,9 @@ call_user_func( TYPO3\CMS\Extbase\Utility\ExtensionUtility::configurePlugin( 'Codappix.' . $extensionKey, 'search', + [ + 'Search' => 'search' + ], [ 'Search' => 'search' ] From bf91c4a5ba2005bf78d5497661e001f47e26af4a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 12:25:25 +0100 Subject: [PATCH 086/158] TASK: Allow fields and sorting to contain a condition This way integrators can configure when the sorting and fields should be added. --- Classes/Domain/Search/QueryFactory.php | 25 ++- .../source/configuration/connections.rst | 18 +- .../source/configuration/indexing.rst | 194 +++++++++--------- .../source/configuration/searching.rst | 192 ++++++++++++----- 4 files changed, 266 insertions(+), 163 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 56d4dd3..d5f3669 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -165,7 +165,10 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + $scriptFields = $this->filterByCondition($scriptFields); + if ($scriptFields !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } } catch (InvalidArgumentException $e) { // Nothing configured } @@ -175,7 +178,10 @@ class QueryFactory { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + $sorting = $this->filterByCondition($sorting); + if ($sorting !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + } } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -254,4 +260,19 @@ class QueryFactory return $array; } + + protected function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; + }); + + foreach ($entries as $key => $entry) { + if (array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } } diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst index 841d878..5819730 100644 --- a/Documentation/source/configuration/connections.rst +++ b/Documentation/source/configuration/connections.rst @@ -29,27 +29,27 @@ The following settings are available. For each setting its documented which conn ``host`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. +The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 +installation. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost .. _port: ``port`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. +The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 413da3d..93eff19 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -29,18 +29,18 @@ The following settings are available. For each setting its documented which inde rootLineBlacklist ----------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. +Defines a blacklist of page uids. Records below any of these pages, or subpages, are not +indexed. This allows you to define areas that should not be indexed. +The page attribute *No Search* is also taken into account to prevent indexing records from only one +page without recursion. - Contains a comma separated list of page uids. Spaces are trimmed. +Contains a comma separated list of page uids. Spaces are trimmed. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 Also it's possible to define some behaviour for the different document types. In context of TYPO3 tables are used as document types 1:1. It's possible to configure different tables. The following @@ -51,156 +51,156 @@ options are available: additionalWhereClause --------------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. +Add additional SQL to where clauses to determine indexable records from the table. This way you +can exclude specific records like ``tt_content`` records with specific ``CType`` values or +something else. E.g. you can add a new field to the table to exclude records from indexing. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - .. attention:: +.. attention:: - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. .. _abstractFields: abstractFields -------------- - Used by: :ref:`PagesIndexer`. +Used by: :ref:`PagesIndexer`. - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. +Define which field should be used to provide the auto generated field "search_abstract". +The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also +possible. - Example:: +Example:: - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - Default:: +Default:: - abstract, description, bodytext + abstract, description, bodytext .. _mapping: mapping ------- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. +Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html +You are able to define the mapping for each property / columns. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword } + } - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. +The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This +makes building a facet possible. .. _index: index ----- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html +Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding } + } - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 } } } + } - ``char_filter`` and ``filter`` are a comma separated list of options. +``char_filter`` and ``filter`` are a comma separated list of options. .. _dataProcessing: dataProcessing -------------- - Used by: All connections while indexing. +Used by: All connections while indexing. - Configure modifications on each document before sending it to the configured connection. Same as - provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. +Configure modifications on each document before sending it to the configured connection. Same as +provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - All processors are applied in configured order. Allowing to work with already processed data. +All processors are applied in configured order. Allowing to work with already processed data. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck } - The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards - all fields, including ``search_spellcheck`` will be copied to ``search_all``. - E.g. used to index all information into a field for :ref:`spellchecking` or searching with - different :ref:`mapping`. + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } - The following Processor are available: +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. +E.g. used to index all information into a field for :ref:`spellchecking` or searching with +different :ref:`mapping`. - .. toctree:: - :maxdepth: 1 - :glob: +The following Processor are available: - dataProcessing/CopyToProcessor - dataProcessing/GeoPointProcessor +.. toctree:: + :maxdepth: 1 + :glob: - The following Processor are planned: + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. +The following Processor are planned: - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". - Of course you are able to provide further processors. Just implement - ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified - class name) as done in the examples above. + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. - By implementing also the same interface as necessary for TYPO3 - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code - also for Fluid to prepare the same record fetched from DB for your fluid. +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index 2143364..e953dcb 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -8,118 +8,200 @@ Searching size ---- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Defined how many search results should be fetched to be available in search result. +Defined how many search results should be fetched to be available in search result. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.size = 50 + plugin.tx_searchcore.settings.searching.size = 50 - Default if not configured is 10. +Default if not configured is 10. .. _facets: facets ------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. +Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html +Currently only the term facet is provided. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType } + } - The above example will provide a facet with options for all found ``CType`` results together - with a count. +The above example will provide a facet with options for all found ``CType`` results together +with a count. .. _filter: -``filter`` -""""""""""" +filter +------ - Used by: While building search request. +Used by: While building search request. - Define filter that should be set for all requests. +Define filter that should be set for all requests. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.filter { - property = value - } + plugin.tx_searchcore.settings.searching.filter { + property = value + } - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. +For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. .. _minimumShouldMatch: minimumShouldMatch ------------------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html +Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - Example:: +Example:: - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% .. _boost: boost ----- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define fields that should boost the score for results. +Define fields that should boost the score for results. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html +For further information take a look at +https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html .. _fieldValueFactor: fieldValueFactor ---------------- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor +Define a field to use as a factor for scoring. The configuration is passed through to elastic +search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - Example:: +Example:: - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mapping.filter: + +mapping.filter +-------------- + +Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will +be added. + +E.g. you submit a filter in form of: + +.. code-block:: html + + + + + +This will create a ``distance`` filter with subproperties. To make this filter actually work, you +can add the following TypoScript, which will be added to the filter:: + + mapping { + filter { + distance { + field = geo_distance + fields { + distance = distance + location = location + } + } } + } + +``fields`` has a special meaning here. This will actually map the properties of the filter to fields +in elasticsearch. In above example they do match, but you can also use different names in your form. +On the left hand side is the elasticsearch field name, on the right side the one submitted as a +filter. + +The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. + +.. _fields: + +fields +------ + +Defines the fields to fetch from elasticsearch. Two sub entries exist: + +First ``stored_fields`` which is a list of comma separated fields which actually exist and will be +added. Typically you will use ``_source`` to fetch the whole indexed fields. + +Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +An example might look like the following:: + + fields { + script_fields { + distance { + condition = {request.filter.distance.location} + script { + params { + lat = {request.filter.distance.location.lat -> f:format.number()} + lon = {request.filter.distance.location.lon -> f:format.number()} + } + lang = painless + inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001 + } + } + } + } + +In above example we add a single ``script_field`` called ``distance``. We add a condition when this +field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. +If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the +query. The ``condition`` will be removed and everything else is submitted one to one to +elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +from search request, e.g. to insert latitude and longitude from a filter, like in the above example. + +.. _sort: + +sort +---- + +Sort is handled like :ref:`fields`. .. _mode: -``mode`` -"""""""" +mode +---- - Used by: Controller while preparing action. +Used by: Controller while preparing action. - Define to switch from search to filter mode. +Define to switch from search to filter mode. - Example:: +Example:: - plugin.tx_searchcore.settings.searching { - mode = filter - } + plugin.tx_searchcore.settings.searching { + mode = filter + } - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. +Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. From 7a5bea687e32f3f247beb9bac80b874564a517ee Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:03:42 +0100 Subject: [PATCH 087/158] TASK: Add new tests for filter setting on model --- Tests/Unit/Domain/Model/SearchRequestTest.php | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Tests/Unit/Domain/Model/SearchRequestTest.php diff --git a/Tests/Unit/Domain/Model/SearchRequestTest.php b/Tests/Unit/Domain/Model/SearchRequestTest.php new file mode 100644 index 0000000..127baf2 --- /dev/null +++ b/Tests/Unit/Domain/Model/SearchRequestTest.php @@ -0,0 +1,80 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class SearchRequestTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleEmptyFilter + */ + public function emptyFilterWillNotBeSet(array $filter) + { + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + [], + $searchRequest->getFilter(), + 'Empty filter were set, even if they should not.' + ); + } + + public function possibleEmptyFilter() + { + return [ + 'Complete empty Filter' => [ + 'filter' => [], + ], + 'Single filter with empty value' => [ + 'filter' => [ + 'someFilter' => '', + ], + ], + 'Single filter with empty recursive values' => [ + 'filter' => [ + 'someFilter' => [ + 'someKey' => '', + ], + ], + ], + ]; + } + + /** + * @test + */ + public function filterIsSet() + { + $filter = ['someField' => 'someValue']; + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + $filter, + $searchRequest->getFilter(), + 'Filter was not set.' + ); + } +} From 85bfb86f5fc7ac37b8766a467426151687901e72 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:03:58 +0100 Subject: [PATCH 088/158] TASK: Fix broken tests for query factory --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index f7d0dd2..baf239f 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -100,8 +100,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ 'field' => '', - 'field1' => 0, - 'field2' => false, ]); $this->assertFalse( @@ -209,10 +207,16 @@ class QueryFactoryTest extends AbstractUnitTestCase public function minimumShouldMatchIsAddedToQuery() { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->once()) + $this->configuration->expects($this->any()) ->method('getIfExists') - ->with('searching.minimumShouldMatch') - ->willReturn('50%'); + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + '50%', + null + )); $this->configuration->expects($this->any()) ->method('get') ->will($this->throwException(new InvalidArgumentException)); @@ -244,14 +248,21 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( [ 'search_title' => 3, 'search_abstract' => 1.5, ], + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException) )); @@ -292,10 +303,17 @@ class QueryFactoryTest extends AbstractUnitTestCase 'factor' => '2', 'missing' => '1', ]; - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $fieldConfig )); From c38f7b9d6ab6b4bcbd6dd915bf2515cb52813db1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:44:17 +0100 Subject: [PATCH 089/158] TASK: Move configuration logic into own class Also add tests for new code. --- .../Configuration/ConfigurationUtility.php | 69 +++++++++ Classes/Domain/Search/QueryFactory.php | 51 ++----- Tests/Unit/AbstractUnitTestCase.php | 15 ++ .../ConfigurationUtilityTest.php | 143 ++++++++++++++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 4 +- 5 files changed, 243 insertions(+), 39 deletions(-) create mode 100644 Classes/Configuration/ConfigurationUtility.php create mode 100644 Tests/Unit/Configuration/ConfigurationUtilityTest.php diff --git a/Classes/Configuration/ConfigurationUtility.php b/Classes/Configuration/ConfigurationUtility.php new file mode 100644 index 0000000..29cb721 --- /dev/null +++ b/Classes/Configuration/ConfigurationUtility.php @@ -0,0 +1,69 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Fluid\View\StandaloneView; + +class ConfigurationUtility +{ + /** + * Will parse all entries, recursive as fluid template, with request variable set to $searchRequest. + */ + public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be send as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } + + /** + * Will check all entries, whether they have a condition and filter entries out, where condition is false. + * Also will remove condition in the end. + */ + public function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !is_array($entry) + || !array_key_exists('condition', $entry) + || (bool) $entry['condition'] === true + ; + }); + + foreach ($entries as $key => $entry) { + if (is_array($entry) && array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } +} diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index d5f3669..f73372d 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -21,13 +21,13 @@ namespace Codappix\SearchCore\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; -use TYPO3\CMS\Fluid\View\StandaloneView; class QueryFactory { @@ -41,12 +41,19 @@ class QueryFactory */ protected $configuration; + /** + * @var ConfigurationUtility + */ + protected $configurationUtility; + public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, - ConfigurationContainerInterface $configuration + ConfigurationContainerInterface $configuration, + ConfigurationUtility $configurationUtility ) { $this->logger = $logManager->getLogger(__CLASS__); $this->configuration = $configuration; + $this->configurationUtility = $configurationUtility; } /** @@ -164,8 +171,8 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); - $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $scriptFields = $this->filterByCondition($scriptFields); + $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); if ($scriptFields !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); } @@ -177,8 +184,8 @@ class QueryFactory protected function addSort(SearchRequestInterface $searchRequest, array &$query) { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; - $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $sorting = $this->filterByCondition($sorting); + $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $sorting = $this->configurationUtility->filterByCondition($sorting); if ($sorting !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); } @@ -243,36 +250,4 @@ class QueryFactory ]); } } - - protected function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array - { - array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { - $template = new StandaloneView(); - $template->assign('request', $searchRequest); - $template->setTemplateSource($value); - $value = $template->render(); - - // As elasticsearch does need some doubles to be end as doubles. - if (is_numeric($value)) { - $value = (float) $value; - } - }, $searchRequest); - - return $array; - } - - protected function filterByCondition(array $entries) : array - { - $entries = array_filter($entries, function ($entry) { - return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; - }); - - foreach ($entries as $key => $entry) { - if (array_key_exists('condition', $entry)) { - unset($entries[$key]['condition']); - } - } - - return $entries; - } } diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index c11e74e..a8ac167 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -24,6 +24,21 @@ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; abstract class AbstractUnitTestCase extends CoreTestCase { + public function setUp() + { + parent::setUp(); + + // Disable caching backends to make TYPO3 parts work in unit test mode. + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( + \TYPO3\CMS\Core\Cache\CacheManager::class + )->setCacheConfigurations([ + 'extbase_object' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + ]); + } + /** * @return \TYPO3\CMS\Core\Log\LogManager */ diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php new file mode 100644 index 0000000..b2dc5f8 --- /dev/null +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -0,0 +1,143 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationUtility; +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class ConfigurationUtilityTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleRequestAndConfigurationForFluidtemplate + */ + public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->replaceArrayValuesWithRequestContent($searchRequest, $array), + 'Entries in array were not parsed as fluid template with search request.' + ); + } + + public function possibleRequestAndConfigurationForFluidtemplate() : array + { + return [ + 'Nothing in array' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [], + 'expected' => [], + ], + 'Small array with nothing to replace' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Rescursive array with replacements' => [ + 'searchRequest' => call_user_func(function () { + $request = new SearchRequest(); + $request->setFilter([ + 'distance' => [ + 'location' => '10', + ], + ]); + return $request; + }), + 'array' => [ + 'sub1' => [ + 'sub1.1' => '{request.filter.distance.location}', + 'sub1.2' => '{request.nonExisting}', + ], + ], + 'expected' => [ + 'sub1' => [ + // Numberics are casted to double + 'sub1.1' => 10.0, + 'sub1.2' => null, + ], + ], + ], + ]; + } + + /** + * @test + * @dataProvider possibleConditionEntries + */ + public function conditionsAreHandledAsExpected(array $entries, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->filterByCondition($entries), + 'Conditions were not processed as expected.' + ); + } + + public function possibleConditionEntries() : array + { + return [ + 'Nothing in array' => [ + 'entries' => [], + 'expected' => [], + ], + 'Entries without condition' => [ + 'entries' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Entry with matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => true, + 'sub1.2' => 'something', + ], + ], + 'expected' => [ + 'sub1' => [ + 'sub1.2' => 'something', + ], + ], + ], + 'Entry with non matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => false, + 'sub1.2' => 'something', + ], + ], + 'expected' => [], + ], + ]; + } +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index baf239f..3b70679 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; @@ -44,7 +45,8 @@ class QueryFactoryTest extends AbstractUnitTestCase parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); - $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); + $configurationUtility = new ConfigurationUtility(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility); } /** From b5225b943c3efd21d90c319677226eaebfb0bd07 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 14:16:16 +0100 Subject: [PATCH 090/158] TASK: Add tests for new query factory code --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 3b70679..6301cdd 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -363,4 +363,210 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Empty search request does not create expected query.' ); } + + /** + * @test + */ + public function storedFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + '_source, something,nothing', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + ['_source', 'something', 'nothing'], + $query->toArray()['stored_fields'], + 'Stored fields were not added to query as expected.' + ); + } + + /** + * @test + */ + public function storedFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertFalse( + isset($query->toArray()['stored_fields']), + 'Stored fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function scriptFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ], + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['script_fields'], + 'Script fields were not added to query as expected.' + ); + + } + + /** + * @test + */ + public function scriptFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['script_fields']), + 'Script fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function sortIsAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ] + )); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['sort'], + 'Sort was not added to query as expected.' + ); + } + + /** + * @test + */ + public function sortIsNotAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + null + )); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['sort']), + 'Sort was added to query even if not configured.' + ); + } } From b0eccc241d6d5bb89446b16f4701607f4051ae43 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 5 Sep 2017 20:30:34 +0200 Subject: [PATCH 091/158] TASK: Allow dev to be required as 1.0 Until we release first stable version. --- composer.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/composer.json b/composer.json index 0061751..985be01 100644 --- a/composer.json +++ b/composer.json @@ -36,6 +36,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "1.0.x-dev" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" From 8206a1ec59272d54f3d68f14d3323c7a4bfe541c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 17:08:33 +0100 Subject: [PATCH 092/158] BUGFIX: Do not remove submitted filter if configured one is empty This will be the case if you add a flexform to the plugin with no value. Then an empty filter is configured and you will not be able to submit a value for this filter. --- Classes/Domain/Search/SearchService.php | 15 ++++++++--- .../Unit/Domain/Search/SearchServiceTest.php | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 384f6aa..120ab3b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Domain\Model\FacetRequest; +use TYPO3\CMS\Core\Utility\ArrayUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** @@ -123,10 +124,16 @@ class SearchService protected function addConfiguredFilters(SearchRequestInterface $searchRequest) { try { - $searchRequest->setFilter(array_merge( - $searchRequest->getFilter(), - $this->configuration->get('searching.filter') - )); + $filter = $searchRequest->getFilter(); + + ArrayUtility::mergeRecursiveWithOverrule( + $filter, + $this->configuration->get('searching.filter'), + true, + false + ); + + $searchRequest->setFilter($filter); } catch (InvalidArgumentException $e) { // Nothing todo, no filter configured. } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 9149e51..c63e29e 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -178,4 +178,29 @@ class SearchServiceTest extends AbstractUnitTestCase $searchRequest->setFilter(['anotherProperty' => 'anything']); $this->subject->search($searchRequest); } + + /** + * @test + */ + public function emptyConfiguredFilterIsNotChangingRequestWithExistingFilter() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->willReturn(['anotherProperty' => '']); + + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter(['anotherProperty' => 'anything']); + $this->subject->search($searchRequest); + } } From 015931518380b2b3b7a429a3b99ddcd453f75f61 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:20:37 +0100 Subject: [PATCH 093/158] FEATURE: Add data processor to remove fields for indexing Add a new processor, with docs and tests, to allow removal of fields before sending them to search service like elasticsearch. E.g. remove sensitive information that should not be available. --- Classes/DataProcessing/RemoveProcessor.php | 44 +++++++ .../dataProcessing/RemoveProcessor.rst | 23 ++++ .../source/configuration/indexing.rst | 1 + .../DataProcessing/RemoveProcessorTest.php | 122 ++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 Classes/DataProcessing/RemoveProcessor.php create mode 100644 Documentation/source/configuration/dataProcessing/RemoveProcessor.rst create mode 100644 Tests/Unit/DataProcessing/RemoveProcessorTest.php diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php new file mode 100644 index 0000000..b5ab788 --- /dev/null +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -0,0 +1,44 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; + +/** + * Removes fields from record. + */ +class RemoveProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + if (!isset($configuration['fields'])) { + return $record; + } + + foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { + if (isset($record[$field])) { + unset($record[$field]); + } + } + + return $record; + } +} diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst new file mode 100644 index 0000000..197f3c1 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\RemoveProcessor`` +====================================================== + +Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. + +Possible Options: + +``fields`` + Comma separated list of fields to remove from record. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\RemoveProcessor + 1 { + fields = description + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + fields = description, another_field + } + } + diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 93eff19..b605008 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -180,6 +180,7 @@ The following Processor are available: :glob: dataProcessing/CopyToProcessor + dataProcessing/RemoveProcessor dataProcessing/GeoPointProcessor The following Processor are planned: diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php new file mode 100644 index 0000000..57600f0 --- /dev/null +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -0,0 +1,122 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\RemoveProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class RemoveProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new RemoveProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Nothing configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Single field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + 'Non existing field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'non existing', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Multiple fields configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'field 3' => 'Some more like lorem', + ], + 'configuration' => [ + 'fields' => 'field 3, field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + ]; + } +} From 379dddf8ac6f7e4bcb526f9f6627524398ecd5ff Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:36:04 +0100 Subject: [PATCH 094/158] BUGFIX: Also remove fields containing "null" --- Classes/DataProcessing/RemoveProcessor.php | 2 +- Tests/Unit/DataProcessing/RemoveProcessorTest.php | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php index b5ab788..6e74237 100644 --- a/Classes/DataProcessing/RemoveProcessor.php +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -34,7 +34,7 @@ class RemoveProcessor implements ProcessorInterface } foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { - if (isset($record[$field])) { + if (array_key_exists($field, $record)) { unset($record[$field]); } } diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php index 57600f0..dc55f73 100644 --- a/Tests/Unit/DataProcessing/RemoveProcessorTest.php +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -117,6 +117,17 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'field 1' => 'Some content like lorem', ], ], + 'Fields with "null" san be removed' => [ + 'record' => [ + 'field 1' => null, + ], + 'configuration' => [ + 'fields' => 'field 1', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + ], + ], ]; } } From ddb95e8c9111bcec5679b130af4b2db8c5a7a24b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:37:28 +0100 Subject: [PATCH 095/158] BUGFIX: Fix documentation for remove processor --- .../source/configuration/dataProcessing/RemoveProcessor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst index 197f3c1..c8653a5 100644 --- a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -15,7 +15,7 @@ Example:: 1 { fields = description } - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 = Codappix\SearchCore\DataProcessing\RemoveProcessor 2 { fields = description, another_field } From e1a14b2f049f016f915d3417021c7a81b15d2a0f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 21:05:53 +0100 Subject: [PATCH 096/158] !!!|FEATURE: Make data processing available to all indexer Before data processing was applied for TCA only, through tca table service. Now it's applied much later in process and in abstract indexer. Therefore all indexer will run data processing if configured. --- Classes/Domain/Index/AbstractIndexer.php | 29 ++++- .../Index/TcaIndexer/TcaTableService.php | 20 --- .../Unit/Domain/Index/AbstractIndexerTest.php | 121 ++++++++++++++++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 73 ----------- 4 files changed, 149 insertions(+), 94 deletions(-) create mode 100644 Tests/Unit/Domain/Index/AbstractIndexerTest.php diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..3b5d077 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -23,7 +23,8 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; -use \TYPO3\CMS\Core\Utility\GeneralUtility; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface { @@ -122,6 +123,32 @@ abstract class AbstractIndexer implements IndexerInterface * @param array &$record */ protected function prepareRecord(array &$record) + { + try { + foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { + $className = ''; + if (is_string($configuration)) { + $className = $configuration; + $configuration = []; + } else { + $className = $configuration['_typoScriptNodeValue']; + } + $dataProcessor = GeneralUtility::makeInstance($className); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidArgumentException $e) { + // Nothing to do. + } + + $this->handleAbstract($record); + } + + /** + * @param array &$record + */ + protected function handleAbstract(array &$record) { $record['search_abstract'] = ''; diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index f4fbd96..7a322a0 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; -use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -141,31 +140,12 @@ class TcaTableService } /** - * Adjust record accordingly to configuration. * @param array &$record */ public function prepareRecord(array &$record) : void { $this->relationResolver->resolveRelationsForRecord($this, $record); - try { - foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { - $className = ''; - if (is_string($configuration)) { - $className = $configuration; - $configuration = []; - } else { - $className = $configuration['_typoScriptNodeValue']; - } - $dataProcessor = GeneralUtility::makeInstance($className); - if ($dataProcessor instanceof ProcessorInterface) { - $record = $dataProcessor->processRecord($record, $configuration); - } - } - } catch (InvalidConfigurationArgumentException $e) { - // Nothing to do. - } - if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php new file mode 100644 index 0000000..3bbc97d --- /dev/null +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -0,0 +1,121 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Domain\Index\AbstractIndexer; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class AbstractIndexerTest extends AbstractUnitTestCase +{ + /** + * @var TcaTableService + */ + protected $subject; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var ConnectionInterface + */ + protected $connection; + + public function setUp() + { + parent::setUp(); + + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + + $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ + $this->connection, + $this->configuration + ]); + $this->subject->injectLogger($this->getMockedLogger()); + $this->subject->setIdentifier('testTable'); + $this->subject->expects($this->any()) + ->method('getDocumentName') + ->willReturn('testTable'); + } + + /** + * @test + */ + public function executesConfiguredDataProcessingWithConfiguration() + { + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['new_test_field'] = 'test'; + $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; + $expectedRecord['search_abstract'] = ''; + + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) + ->will($this->onConsecutiveCalls([ + '1' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + '2' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + ], $this->throwException(new InvalidArgumentException))); + $this->subject->expects($this->once()) + ->method('getRecord') + ->with(1) + ->willReturn($record) + ; + + $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); + $this->subject->indexDocument(1); + } + + /** + * @test + */ + public function executesNoDataProcessingForMissingConfiguration() + { + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['search_abstract'] = ''; + + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) + ->will($this->throwException(new InvalidArgumentException)); + $this->subject->expects($this->once()) + ->method('getRecord') + ->with(1) + ->willReturn($record) + ; + + $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); + $this->subject->indexDocument(1); + } +} diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index dd0a389..edad84a 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -148,77 +148,4 @@ class TcaTableServiceTest extends AbstractUnitTestCase ); unset($GLOBALS['TCA']['test_table']); } - - /** - * @test - */ - public function executesConfiguredDataProcessingWithConfiguration() - { - $this->configuration->expects($this->exactly(1)) - ->method('get') - ->with('indexing.testTable.dataProcessing') - ->will($this->returnValue([ - '1' => [ - '_typoScriptNodeValue' => CopyToProcessor::class, - 'to' => 'new_test_field', - ], - '2' => [ - '_typoScriptNodeValue' => CopyToProcessor::class, - 'to' => 'new_test_field2', - ], - ])); - - $subject = $this->getMockBuilder(TcaTableService::class) - ->disableOriginalConstructor() - ->setMethodsExcept(['prepareRecord']) - ->getMock(); - $this->inject($subject, 'configuration', $this->configuration); - $this->inject($subject, 'tableName', 'testTable'); - $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); - - $record = ['field 1' => 'test']; - $expectedRecord = $record; - $expectedRecord['new_test_field'] = 'test'; - $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; - - $subject->prepareRecord($record); - - $this->assertSame( - $expectedRecord, - $record, - 'Dataprocessing is not executed by TcaTableService as expected.' - ); - } - - /** - * @test - */ - public function executesConfiguredDataProcessingWithoutConfiguration() - { - $this->configuration->expects($this->exactly(1)) - ->method('get') - ->with('indexing.testTable.dataProcessing') - ->will($this->returnValue([CopyToProcessor::class])); - - $subject = $this->getMockBuilder(TcaTableService::class) - ->disableOriginalConstructor() - ->setMethodsExcept(['prepareRecord']) - ->getMock(); - $this->inject($subject, 'configuration', $this->configuration); - $this->inject($subject, 'tableName', 'testTable'); - $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); - - $record = ['field 1' => 'test']; - $expectedRecord = $record; - $expectedRecord[''] = 'test'; - $expectedRecord['search_title'] = 'test'; - - $subject->prepareRecord($record); - - $this->assertSame( - $expectedRecord, - $record, - 'Dataprocessing is not executed by TcaTableService as expected.' - ); - } } From 31202f88821d1691d8886d50461ad55ad188edee Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 10 Nov 2017 12:31:06 +0100 Subject: [PATCH 097/158] FEATURE: Provide form finisher for integration into form extension Provide a finisher, working as a proxy, to internal data handler, which is already used for Hooks in TYPO3 backend. --- .../Form/Finisher/DataHandlerFinisher.php | 71 +++++++++ Documentation/source/features.rst | 2 + Documentation/source/usage.rst | 28 ++++ Tests/Unit/AbstractUnitTestCase.php | 37 +++++ .../Form/Finisher/DataHandlerFinisherTest.php | 140 ++++++++++++++++++ 5 files changed, 278 insertions(+) create mode 100644 Classes/Integration/Form/Finisher/DataHandlerFinisher.php create mode 100644 Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php new file mode 100644 index 0000000..02d6bde --- /dev/null +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -0,0 +1,71 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher; +use TYPO3\CMS\Form\Domain\Finishers\Exception\FinisherException; + +/** + * Integrates search_core indexing into TYPO3 Form extension. + * + * Add this finisher AFTER all database operations, as search_core will fetch + * information from database. + */ +class DataHandlerFinisher extends AbstractFinisher +{ + /** + * @var \Codappix\SearchCore\Domain\Service\DataHandler + * @inject + */ + protected $dataHandler; + + /** + * @var array + */ + protected $defaultOptions = [ + 'indexIdentifier' => null, + 'recordUid' => null, + 'action' => '', + ]; + + protected function executeInternal() + { + $action = $this->parseOption('action'); + $record = ['uid' => (int) $this->parseOption('recordUid')]; + $tableName = $this->parseOption('indexIdentifier'); + + if ($action === '' || $tableName === '' || !is_string($tableName) || $record['uid'] === 0) { + throw new FinisherException('Not all necessary options were set.', 1510313095); + } + + switch ($action) { + case 'update': + $this->dataHandler->update($tableName, $record); + break; + case 'add': + $this->dataHandler->add($tableName, $record); + break; + case 'delete': + $this->dataHandler->delete($tableName, $record['uid']); + break; + } + } +} diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 199192a..8baf453 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -15,6 +15,8 @@ configuration needs. Still it's possible to configure the indexer. Also custom classes can be used as indexers. +Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing. + .. _features_search: Searching diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 1ca50d2..fd94615 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -30,6 +30,34 @@ The tables have to be configured via :ref:`configuration_options_index`. Not all hook operations are supported yet, see :issue:`27`. +.. _usage_form_finisher: + +Form finisher +------------- + +A form finisher is provided to integrate indexing into form extension. + +Add form finisher to your available finishers and configure it like: + +.. code-block:: yaml + :linenos: + + - + identifier: SearchCoreIndexer + options: + action: 'delete' + indexIdentifier: 'fe_users' + recordUid: '{FeUser.user.uid}' + +All three options are necessary, where + +``action`` + Is one of ``delete``, ``update`` or ``add``. +``indexIdentifier`` + Is a configured index identifier. +``recordUid`` + Has to be the uid of the record to index. + .. _usage_searching: Searching / Frontend Plugin diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index a8ac167..c21209d 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -21,13 +21,23 @@ namespace Codappix\SearchCore\Tests\Unit; */ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Object\ObjectManager; +use TYPO3\CMS\Form\Service\TranslationService; abstract class AbstractUnitTestCase extends CoreTestCase { + /** + * @var array A backup of registered singleton instances + */ + protected $singletonInstances = []; + public function setUp() { parent::setUp(); + $this->singletonInstances = GeneralUtility::getSingletonInstances(); + // Disable caching backends to make TYPO3 parts work in unit test mode. \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( @@ -39,6 +49,12 @@ abstract class AbstractUnitTestCase extends CoreTestCase ]); } + public function tearDown() + { + GeneralUtility::resetSingletonInstances($this->singletonInstances); + parent::tearDown(); + } + /** * @return \TYPO3\CMS\Core\Log\LogManager */ @@ -58,4 +74,25 @@ abstract class AbstractUnitTestCase extends CoreTestCase return $logger; } + + /** + * Configure translation service mock for Form Finisher. + * + * This way parseOption will always return the configured value. + */ + protected function configureMockedTranslationService() + { + $translationService = $this->getMockBuilder(TranslationService::class)->getMock(); + $translationService->expects($this->any()) + ->method('translateFinisherOption') + ->willReturnCallback(function ($formRuntime, $finisherIdentifier, $optionKey, $optionValue) { + return $optionValue; + }); + $objectManager = $this->getMockBuilder(ObjectManager::class)->getMock(); + $objectManager->expects($this->any()) + ->method('get') + ->with(TranslationService::class) + ->willReturn($translationService); + GeneralUtility::setSingletonInstance(ObjectManager::class, $objectManager); + } } diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php new file mode 100644 index 0000000..2211cc2 --- /dev/null +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -0,0 +1,140 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Service\DataHandler; +use Codappix\SearchCore\Integration\Form\Finisher\DataHandlerFinisher; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Form\Domain\Finishers\Exception\FinisherException; +use TYPO3\CMS\Form\Domain\Finishers\FinisherContext; + +class DataHandlerFinisherTest extends AbstractUnitTestCase +{ + /** + * @var DataHandlerFinisher + */ + protected $subject; + + /** + * @var DataHandler + */ + protected $dataHandlerMock; + + /** + * @var FinisherContext + */ + protected $finisherContextMock; + + public function setUp() + { + parent::setUp(); + + $this->configureMockedTranslationService(); + $this->dataHandlerMock = $this->getMockBuilder(DataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + $this->finisherContextMock = $this->getMockBuilder(FinisherContext::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->subject = new DataHandlerFinisher(); + $this->inject($this->subject, 'dataHandler', $this->dataHandlerMock); + } + + /** + * @test + * @dataProvider possibleFinisherSetup + */ + public function validConfiguration(string $action, array $nonCalledActions, $expectedSecondArgument) + { + $this->subject->setOptions([ + 'indexIdentifier' => 'test_identifier', + 'recordUid' => '23', + 'action' => $action, + ]); + + foreach ($nonCalledActions as $nonCalledAction) { + $this->dataHandlerMock->expects($this->never())->method($nonCalledAction); + } + $this->dataHandlerMock->expects($this->once())->method($action) + ->with('test_identifier', $expectedSecondArgument); + + $this->subject->execute($this->finisherContextMock); + } + + public function possibleFinisherSetup() : array + { + return [ + 'valid add configuration' => [ + 'action' => 'add', + 'nonCalledActions' => ['delete', 'update'], + 'expectedSecondArgument' => ['uid' => 23], + ], + 'valid update configuration' => [ + 'action' => 'update', + 'nonCalledActions' => ['delete', 'add'], + 'expectedSecondArgument' => ['uid' => 23], + ], + 'valid delete configuration' => [ + 'action' => 'delete', + 'nonCalledActions' => ['update', 'add'], + 'expectedSecondArgument' => 23, + ], + ]; + } + + /** + * @test + * @dataProvider invalidFinisherSetup + */ + public function nothingHappensIfUnknownActionIsConfigured(array $options) + { + $this->subject->setOptions($options); + + foreach (['add', 'update', 'delete'] as $nonCalledAction) { + $this->dataHandlerMock->expects($this->never())->method($nonCalledAction); + } + + $this->expectException(FinisherException::class); + $this->subject->execute($this->finisherContextMock); + } + + public function invalidFinisherSetup() : array + { + return [ + 'missing options' => [ + 'options' => [], + ], + 'missing action option' => [ + 'options' => [ + 'indexIdentifier' => 'identifier', + 'recordUid' => '20', + ], + ], + 'missing record uid option' => [ + 'options' => [ + 'indexIdentifier' => 'identifier', + 'action' => 'update', + ], + ], + ]; + } +} From a3a46f5cb5b942ae53356ac3092ab781039cb815 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 10 Nov 2017 13:22:15 +0100 Subject: [PATCH 098/158] FEATURE: Provide command to delete whole index This is necessary, e.g. for complete re-indexing. --- Classes/Command/IndexCommandController.php | 16 +++++- Classes/Connection/ConnectionInterface.php | 9 ++++ Classes/Connection/Elasticsearch.php | 12 +++++ Classes/Domain/Index/AbstractIndexer.php | 7 +++ Classes/Domain/Index/IndexerInterface.php | 11 +++- Documentation/source/usage.rst | 13 +++++ .../Elasticsearch/IndexDeletionTest.php | 50 +++++++++++++++++++ .../Command/IndexCommandControllerTest.php | 37 ++++++++++++++ 8 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index b14d4b3..70816ed 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -50,7 +50,6 @@ class IndexCommandController extends CommandController */ public function indexCommand($identifier) { - // TODO: Also allow to index everything? try { $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); $this->outputLine($identifier . ' was indexed.'); @@ -58,4 +57,19 @@ class IndexCommandController extends CommandController $this->outputLine('No indexer found for: ' . $identifier); } } + + /** + * Will delete the given identifier. + * + * @param string $identifier + */ + public function deleteCommand($identifier) + { + try { + $this->indexerFactory->getIndexer($identifier)->delete(); + $this->outputLine($identifier . ' was deleted.'); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $identifier); + } + } } diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index a130f9e..59cf9f8 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -77,4 +77,13 @@ interface ConnectionInterface * @return SearchResultInterface */ public function search(SearchRequestInterface $searchRequest); + + /** + * Will delete the whole index / db. + * + * @param string $documentType + * + * @return void + */ + public function deleteIndex($documentType); } diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 4c66b6a..8a3cb2b 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -156,6 +156,18 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } + public function deleteIndex($documentType) + { + $index = $this->connection->getClient()->getIndex('typo3content'); + + if (! $index->exists()) { + $this->logger->notice('Index did not exist, therefore was not deleted.', [$documentType, 'typo3content']); + return; + } + + $index->delete(); + } + /** * Execute given callback with Elastica Type based on provided documentType * diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..0b4d675 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -104,6 +104,13 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger->info('Finish indexing'); } + public function delete() + { + $this->logger->info('Start deletion of index.'); + $this->connection->deleteIndex($this->getDocumentName()); + $this->logger->info('Finish deletion.'); + } + /** * @return \Generator */ diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 5a4ca6c..72ebb9d 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -33,9 +33,9 @@ interface IndexerInterface public function indexAllDocuments(); /** - * Fetches a single document from the indexerService and pushes it to the connection. + * Fetches a single document and pushes it to the connection. * - * @param string $identifier identifier, the indexer needs to identify a single document + * @param string $identifier * * @return void */ @@ -49,4 +49,11 @@ interface IndexerInterface * @return void */ public function setIdentifier($identifier); + + /** + * Delete the whole index. + * + * @return void + */ + public function delete(); } diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 1ca50d2..fb98a99 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -18,6 +18,19 @@ This will index the table ``tt_content`` using the :ref:`TcaIndexer`. Only one index per call is available, to run multiple indexers, just make multiple calls. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. +.. _usage_manual_deletion: + +Manual deletion +--------------- + +You can trigger deletion for a single index from CLI:: + + ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content' + +This will delete the index for the table ``tt_content``. + +Only one delete per call is available, to run multiple deletions, just make multiple calls. + .. _usage_auto_indexing: Auto indexing diff --git a/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php new file mode 100644 index 0000000..8297a2a --- /dev/null +++ b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php @@ -0,0 +1,50 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class IndexDeletionTest extends AbstractFunctionalTestCase +{ + /** + * @test + */ + public function indexIsDeleted() + { + $this->client->getIndex('typo3content')->create(); + $this->assertTrue( + $this->client->getIndex('typo3content')->exists(), + 'Could not create index for test.' + ); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->delete() + ; + + $this->assertFalse( + $this->client->getIndex('typo3content')->exists(), + 'Index could not be deleted through command controller.' + ); + } +} diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php index 4c81672..9dcc7f3 100644 --- a/Tests/Unit/Command/IndexCommandControllerTest.php +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -91,4 +91,41 @@ class IndexCommandControllerTest extends AbstractUnitTestCase $this->subject->indexCommand('allowedTable'); } + + /** + * @test + */ + public function deletionIsPossible() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('allowedTable was deleted.'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('allowedTable') + ->will($this->returnValue($indexerMock)); + + $indexerMock->expects($this->once()) + ->method('delete'); + $this->subject->deleteCommand('allowedTable'); + } + + /** + * @test + */ + public function deletionForNonExistingIndexerDoesNotWork() + { + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('No indexer found for: nonAllowedTable'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('nonAllowedTable') + ->will($this->throwException(new NoMatchingIndexerException)); + + $this->subject->deleteCommand('nonAllowedTable'); + } } From 0815eaff6bb0a3deaadbcae51950df73075e036f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 10 Nov 2017 13:47:26 +0100 Subject: [PATCH 099/158] BUGFIX: Remove records during update if no longer available E.g. update is to deactivate a record. In this case we will not be able to update the record but should delete him instead. --- Classes/Domain/Index/AbstractIndexer.php | 3 +- .../Elasticsearch/IndexTcaTableTest.php | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..de8072d 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -99,7 +99,8 @@ abstract class AbstractIndexer implements IndexerInterface $this->connection->addDocument($this->getDocumentName(), $record); } catch (NoRecordFoundException $e) { - $this->logger->info('Could not index document.', [$e->getMessage()]); + $this->logger->info('Could not index document. Try to delete it therefore.', [$e->getMessage()]); + $this->connection->deleteDocument($this->getDocumentName(), $identifier); } $this->logger->info('Finish indexing'); } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 022fb3b..b1e8191 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -205,4 +205,35 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase 'Record was indexed with resolved category relation, but should not have any.' ); } + + /** + * @test + */ + public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + + $this->getConnectionPool()->getConnectionForTable('tt_content') + ->update( + 'tt_content', + ['hidden' => true], + ['uid' => 10] + ); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexDocument(10) + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.'); + } } From b7b783a7fed644ac6c302cc0c8103f719b2721c3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 11 Nov 2017 16:46:03 +0100 Subject: [PATCH 100/158] TASK: Use Code Sniffer at travis This way we need no external service. Each developer can fully run all tests and cgl on local environment. Also this integrated better into IDEs and editors. --- .travis.yml | 1 + Classes/Connection/Elasticsearch.php | 5 +++- .../Index/TcaIndexer/TcaTableService.php | 9 ++++++-- Classes/Domain/Search/QueryFactory.php | 11 +++++++-- Classes/Hook/DataHandler.php | 9 ++++++-- Makefile | 5 +++- .../ConfigurationUtilityTest.php | 7 ++++-- Tests/Unit/Domain/Search/QueryFactoryTest.php | 1 - composer.json | 11 ++++----- phpcs.xml.dist | 23 +++++++++++++++++++ 10 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 phpcs.xml.dist diff --git a/.travis.yml b/.travis.yml index 4490198..0adb894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ services: install: make install script: + - make cgl - make unitTests - make functionalTests diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 8a3cb2b..b86bebf 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -132,7 +132,10 @@ class Elasticsearch implements Singleton, ConnectionInterface } ); } catch (\Elastica\Exception\NotFoundException $exception) { - $this->logger->debug('Tried to delete document in index, which does not exist.', [$documentType, $identifier]); + $this->logger->debug( + 'Tried to delete document in index, which does not exist.', + [$documentType, $identifier] + ); } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 7a322a0..bf631b1 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -159,7 +159,9 @@ class TcaTableService $parameters = []; $whereClause = $this->getSystemWhereClause(); - $userDefinedWhere = $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.additionalWhereClause'); + $userDefinedWhere = $this->configuration->getIfExists( + 'indexing.' . $this->getTableName() . '.additionalWhereClause' + ); if (is_string($userDefinedWhere)) { $whereClause .= ' AND ' . $userDefinedWhere; } @@ -361,6 +363,9 @@ class TcaTableService */ protected function getBlackListedRootLine() : array { - return GeneralUtility::intExplode(',', $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist')); + return GeneralUtility::intExplode( + ',', + $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') + ); } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index f73372d..5879b9f 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -163,7 +163,11 @@ class QueryFactory { try { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), + 'stored_fields' => GeneralUtility::trimExplode( + ',', + $this->configuration->get('searching.fields.stored_fields'), + true + ), ]); } catch (InvalidArgumentException $e) { // Nothing configured @@ -171,7 +175,10 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); - $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent( + $searchRequest, + $scriptFields + ); $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); if ($scriptFields !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index d0eb1ba..0caac7c 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -103,8 +103,13 @@ class DataHandler implements Singleton * * @return bool False if hook was not processed. */ - public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler) - { + public function processDatamap_afterDatabaseOperations( + $status, + $table, + $uid, + array $fieldArray, + CoreDataHandler $dataHandler + ) { if (! $this->shouldProcessHookForTable($table)) { $this->logger->debug('Database update not processed.', [$table, $uid]); return false; diff --git a/Makefile b/Makefile index b6bc72b..b8f4ac5 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,12 @@ typo3DatabaseHost ?= "127.0.0.1" .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist typo3/cms="$(TYPO3_VERSION)" git checkout composer.json +cgl: + ./.Build/bin/phpcs + functionalTests: typo3DatabaseName=$(typo3DatabaseName) \ typo3DatabaseUsername=$(typo3DatabaseUsername) \ diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php index b2dc5f8..4db367c 100644 --- a/Tests/Unit/Configuration/ConfigurationUtilityTest.php +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -31,8 +31,11 @@ class ConfigurationUtilityTest extends AbstractUnitTestCase * @test * @dataProvider possibleRequestAndConfigurationForFluidtemplate */ - public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected) - { + public function recursiveEntriesAreProcessedAsFluidtemplate( + SearchRequestInterface $searchRequest, + array $array, + array $expected + ) { $subject = new ConfigurationUtility(); $this->assertSame( diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 6301cdd..218f8e5 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -465,7 +465,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $query->toArray()['script_fields'], 'Script fields were not added to query as expected.' ); - } /** diff --git a/composer.json b/composer.json index 985be01..efa5146 100644 --- a/composer.json +++ b/composer.json @@ -16,13 +16,13 @@ } }, "require" : { - "php": ">=7.1.0", - "typo3/cms": "~8.7", + "php": ">=5.6.0", "ruflin/elastica": "~3.2" }, "require-dev": { - "typo3/testing-framework": "~1.1.0", - "phpunit/phpunit": "~6.2.0" + "phpunit/phpunit": "~5.7.0", + "squizlabs/php_codesniffer": "~3.1.1", + "typo3/cms": "~7.6" }, "config": { "optimize-autoloader": true, @@ -36,9 +36,6 @@ ] }, "extra": { - "branch-alias": { - "dev-develop": "1.0.x-dev" - }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" diff --git a/phpcs.xml.dist b/phpcs.xml.dist new file mode 100644 index 0000000..e722968 --- /dev/null +++ b/phpcs.xml.dist @@ -0,0 +1,23 @@ + + + The coding standard for search_core. + + Classes/ + Tests/ + + + + + + + + + + + + + + + Classes/Hook/DataHandler.php + + From 0bac2df6a4bb8636e15db523486818c03343dabb Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 11 Nov 2017 17:01:30 +0100 Subject: [PATCH 101/158] BUGFIX: Fix broken dependencies --- composer.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index efa5146..2761473 100644 --- a/composer.json +++ b/composer.json @@ -16,13 +16,14 @@ } }, "require" : { - "php": ">=5.6.0", + "php": ">=7.1.0", + "typo3/cms": "~8.7", "ruflin/elastica": "~3.2" }, "require-dev": { - "phpunit/phpunit": "~5.7.0", - "squizlabs/php_codesniffer": "~3.1.1", - "typo3/cms": "~7.6" + "phpunit/phpunit": "~6.4.4", + "typo3/testing-framework": "~1.1.5", + "squizlabs/php_codesniffer": "~3.1.1" }, "config": { "optimize-autoloader": true, From 02ef86b67b8f2eab59528496b977ec5fe35c72f1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 18:57:09 +0100 Subject: [PATCH 102/158] FEATURE: Provide new feature to configure fields to search in This enables you to search only in some fields. Also if some fields contain mapping, you can add them in addition to e.g. `_all`. --- Classes/Domain/Search/QueryFactory.php | 18 ++++++++---------- Configuration/TypoScript/setup.txt | 6 ++++++ .../source/configuration/searching.rst | 13 ++++++++++++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index f73372d..980b763 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -99,20 +99,18 @@ class QueryFactory return; } - $query = ArrayUtility::setValueByPath( - $query, - 'query.bool.must.0.match._all.query', - $searchRequest->getSearchTerm() - ); + $matchExpression = [ + 'type' => 'most_fields', + 'query' => $searchRequest->getSearchTerm(), + 'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.query')), + ]; $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); if ($minimumShouldMatch) { - $query = ArrayUtility::setValueByPath( - $query, - 'query.bool.must.0.match._all.minimum_should_match', - $minimumShouldMatch - ); + $matchExpression['minimum_should_match'] = $minimumShouldMatch; } + + $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression); } protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 67612e1..1a1577f 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -21,6 +21,12 @@ plugin { abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} } } + + searching { + fields { + query = _all + } + } } } } diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index e953dcb..ae73fad 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -151,7 +151,18 @@ filtering. This way you can use arbitrary filter names and map them to existing fields ------ -Defines the fields to fetch from elasticsearch. Two sub entries exist: +Defines the fields to fetch and search from elasticsearch. With the following sub keys: + +``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch. +Configure a comma separated list of fields to search in. This is necessary if you have configured +special mapping for some fields, or just want to search some fields. +The most hits get ranked highest. The following is an example configuration:: + + fields { + query = _all, city + } + +The following sub properties configure the fields to fetch from elasticsearch: First ``stored_fields`` which is a list of comma separated fields which actually exist and will be added. Typically you will use ``_source`` to fetch the whole indexed fields. From 0006148a525e38d5af3dc94fda56c16431b0d994 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 19:43:16 +0100 Subject: [PATCH 103/158] TASK: Fix broken functional tests Add new default TypoScript to not break tests. --- Tests/Functional/Fixtures/BasicSetup.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 1e2b3a9..b7b0c6a 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -42,6 +42,10 @@ plugin { field = CType } } + + fields { + query = _all + } } } } From e3151e802c55aec78d7ff99fa7dffc04fe5720a6 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 19:52:10 +0100 Subject: [PATCH 104/158] TASK: Fix broken unit tests Adjust tests to match new queries built with multiple fields. --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 84 ++++++++++--------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 6301cdd..ba845d4 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -56,9 +56,7 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertInstanceOf( @@ -73,9 +71,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); @@ -95,9 +91,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptyFilterIsNotAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ @@ -122,9 +116,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function facetsAreAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); @@ -153,9 +145,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function sizeIsAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setLimit(45); $searchRequest->setOffset(35); @@ -179,9 +169,7 @@ class QueryFactoryTest extends AbstractUnitTestCase public function searchTermIsAddedToQuery() { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -189,9 +177,11 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'query' => 'SearchWord', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', ], ], ], @@ -219,9 +209,7 @@ class QueryFactoryTest extends AbstractUnitTestCase '50%', null )); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertArraySubset( @@ -229,10 +217,13 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'minimum_should_match' => '50%', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', ], + 'minimum_should_match' => '50%', ], ], ], @@ -253,12 +244,14 @@ class QueryFactoryTest extends AbstractUnitTestCase $this->configuration->expects($this->any()) ->method('get') ->withConsecutive( + ['searching.fields.query'], ['searching.boost'], ['searching.fields.stored_fields'], ['searching.fields.script_fields'], ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( + '_all', [ 'search_title' => 3, 'search_abstract' => 1.5, @@ -308,12 +301,14 @@ class QueryFactoryTest extends AbstractUnitTestCase $this->configuration->expects($this->any()) ->method('get') ->withConsecutive( + ['searching.fields.query'], ['searching.boost'], ['searching.fields.stored_fields'], ['searching.fields.script_fields'], ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( + '_all', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), @@ -328,9 +323,11 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'query' => 'SearchWord', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', ], ], ], @@ -352,9 +349,7 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest(); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertInstanceOf( @@ -433,12 +428,14 @@ class QueryFactoryTest extends AbstractUnitTestCase $this->configuration->expects($this->any()) ->method('get') ->withConsecutive( + ['searching.fields.query'], ['searching.boost'], ['searching.fields.stored_fields'], ['searching.fields.script_fields'], ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( + '_all', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), [ @@ -522,9 +519,7 @@ class QueryFactoryTest extends AbstractUnitTestCase ] )); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -559,9 +554,7 @@ class QueryFactoryTest extends AbstractUnitTestCase null )); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertTrue( @@ -569,4 +562,17 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Sort was added to query even if not configured.' ); } + + protected function configureConfigurationMockWithDefault() + { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->returnCallback(function ($configName) { + if ($configName === 'searching.fields.query') { + return '_all'; + } + + throw new InvalidArgumentException(); + })); + } } From 5ba860b8de7854ecb4ed76c6ae492259645bae37 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 20:00:10 +0100 Subject: [PATCH 105/158] TASK: Add new test covering new feature --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index ba845d4..e0fc86d 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -359,6 +359,54 @@ class QueryFactoryTest extends AbstractUnitTestCase ); } + /** + * @test + */ + public function configuredQueryFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.fields.query'], + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + '_all, field1, field2', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertArraySubset( + [ + 'bool' => [ + 'must' => [ + [ + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', + 'field1', + 'field2', + ], + ], + ], + ], + ], + ], + $query->toArray()['query'], + 'Configured fields were not added to query as configured.' + ); + } + /** * @test */ @@ -462,7 +510,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $query->toArray()['script_fields'], 'Script fields were not added to query as expected.' ); - } /** From bdecbf9699d207cdaa387738c835837c8b14afdc Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 29 Jan 2018 22:45:53 +0100 Subject: [PATCH 106/158] TASK: Fix license for packagist --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 985be01..b704e77 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "type": "typo3-cms-extension", "description": "Codappix Search Core.", "homepage": "https://github.com/Codappix/search_core", - "license": ["GPL-2.0+"], + "license": ["GPL-2.0-or-later"], "autoload": { "psr-4": { "Codappix\\SearchCore\\": "Classes" From c994a32ac1f4d62ef76cdac79125d17a5e31af94 Mon Sep 17 00:00:00 2001 From: Justus Moroni Date: Mon, 29 Jan 2018 22:11:53 +0100 Subject: [PATCH 107/158] BUGFIX: Make BackendUtility usable in frontend BackendUtility used LanguageService which only works in the backend. Extend BackendUtility and use TSFE instead. --- .../Index/TcaIndexer/RelationResolver.php | 26 +++++++++---- Classes/Utility/FrontendUtility.php | 39 +++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 Classes/Utility/FrontendUtility.php diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 88aa982..a2302ae 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ +use Codappix\SearchCore\Utility\FrontendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -39,13 +40,15 @@ class RelationResolver implements Singleton if ($column === 'pid') { continue; } - $record[$column] = BackendUtility::getProcessedValueExtra( - $service->getTableName(), - $column, - $record[$column], - 0, - $record['uid'] - ); + + $record[$column] = GeneralUtility::makeInstance($this->getUtilityForMode()) + ::getProcessedValueExtra( + $service->getTableName(), + $column, + $record[$column], + 0, + $record['uid'] + ); try { $config = $service->getColumnConfig($column); @@ -93,4 +96,13 @@ class RelationResolver implements Singleton { return array_map('trim', explode(',', $value)); } + + protected function getUtilityForMode(): string + { + if (TYPO3_MODE === 'BE') { + return BackendUtility::class; + } + + return FrontendUtility::class; + } } diff --git a/Classes/Utility/FrontendUtility.php b/Classes/Utility/FrontendUtility.php new file mode 100644 index 0000000..1282421 --- /dev/null +++ b/Classes/Utility/FrontendUtility.php @@ -0,0 +1,39 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; + +/** + * Overwrite BackendUtility to use in frontend. + * LanguageService was only usable in backend. + */ +class FrontendUtility extends BackendUtility +{ + /** + * @return TypoScriptFrontendController + */ + protected static function getLanguageService() + { + return $GLOBALS['TSFE']; + } +} From 33d4116fe6a024e05fadc7cad8eae5600c11a847 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 29 Jan 2018 22:45:53 +0100 Subject: [PATCH 108/158] TASK: Fix license for packagist --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 2761473..f3bd4c3 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "type": "typo3-cms-extension", "description": "Codappix Search Core.", "homepage": "https://github.com/Codappix/search_core", - "license": ["GPL-2.0+"], + "license": ["GPL-2.0-or-later"], "autoload": { "psr-4": { "Codappix\\SearchCore\\": "Classes" From 92af364b8dbde0f9e633ef1e4ebcec72639c6360 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Feb 2018 20:49:25 +0100 Subject: [PATCH 109/158] FEATURE: Allow to disable elasticsearch integration This extension currently ships with Elasticsearch integration which is enabled by default. This behaviour is kept for backwards compatibility. Still you now have the possibility to disable this integration in extension manager, just check the "disable" box for elasticsearch. In the future elasticsearch will become another extension and no default is shipped with search_core. But for now, as we are still in alpha / beta phase we keep things together to keep development fast. Resolves: #111 --- Documentation/source/configuration.rst | 5 ++++- Documentation/source/installation.rst | 4 ++++ ext_conf_template.txt | 4 ++++ ext_localconf.php | 17 ++++++++++++----- 4 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 ext_conf_template.txt diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index eca5ba4..09289ac 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -5,9 +5,12 @@ Configuration ============= +Installation wide configuration is handled inside of the extension manager. Just check out the +options there, they all have labels. + The extension offers the following configuration options through TypoScript. If you overwrite them through `setup` make sure to keep them in the `module` area as they will be accessed from backend -mode of TYPO3. Do so by placing the following line at the end:: +mode of TYPO3 for indexing. Do so by placing the following line at the end:: module.tx_searchcore < plugin.tx_searchcore diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index ddfb065..5bb0c56 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -19,4 +19,8 @@ In that case you need to install all dependencies yourself. Dependencies are: Afterwards you need to enable the extension through the extension manager and include the static TypoScript setup. +If you **don't** want to use the included elasticsearch integration, you have to disable it in the +extension manager configuration of the extension by checking the checkbox. +It's currently enabled by default but will be moved into its own extension in the future. + .. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip diff --git a/ext_conf_template.txt b/ext_conf_template.txt new file mode 100644 index 0000000..5ddadb4 --- /dev/null +++ b/ext_conf_template.txt @@ -0,0 +1,4 @@ +disable { + # cat=basic/enable; type=boolean; label=Disable Elasticsearch, which is enabled by default + elasticsearch = true +} diff --git a/ext_localconf.php b/ext_localconf.php index c52d05d..a713d0b 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -37,11 +37,18 @@ call_user_func( ] ); - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance('TYPO3\CMS\Extbase\Object\Container\Container') - ->registerImplementation( - 'Codappix\SearchCore\Connection\ConnectionInterface', - 'Codappix\SearchCore\Connection\Elasticsearch' - ); + // API does make use of object manager, therefore use GLOBALS + $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); + if ($extensionConfiguration === false + || !isset($extensionConfiguration['disable.']['elasticsearch']) + || $extensionConfiguration['disable.']['elasticsearch'] !== '1' + ) { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) + ->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); + } }, $_EXTKEY ); From ebaeaf4c92e224618211723d80f3acd6ba73fb68 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Feb 2018 21:59:13 +0100 Subject: [PATCH 110/158] TASK: Support PHP 7.0 As some (e.g. debian) do not provide PHP 7.1 and we did not use so much features which were introduced in PHP 7.1, we add support for PHP 7.0. --- .travis.yml | 1 + Classes/Domain/Index/TcaIndexer/RelationResolver.php | 2 +- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 4 ++-- Makefile | 2 +- composer.json | 4 ++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4490198..7414a0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ before_install: language: php php: + - 7.0 - 7.1 env: diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index a2302ae..fb19a4f 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -33,7 +33,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RelationResolver implements Singleton { - public function resolveRelationsForRecord(TcaTableService $service, array &$record) : void + public function resolveRelationsForRecord(TcaTableService $service, array &$record) { foreach (array_keys($record) as $column) { // TODO: Define / configure fields to exclude?! diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 7a322a0..f059248 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -129,7 +129,7 @@ class TcaTableService * @param array &$records * @return void */ - public function filterRecordsByRootLineBlacklist(array &$records) : void + public function filterRecordsByRootLineBlacklist(array &$records) { $records = array_filter( $records, @@ -142,7 +142,7 @@ class TcaTableService /** * @param array &$record */ - public function prepareRecord(array &$record) : void + public function prepareRecord(array &$record) { $this->relationResolver->resolveRelationsForRecord($this, $record); diff --git a/Makefile b/Makefile index b6bc72b..4fb0ce4 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ typo3DatabaseHost ?= "127.0.0.1" .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist --ignore-platform-reqs typo3/cms="$(TYPO3_VERSION)" + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist typo3/cms="$(TYPO3_VERSION)" git checkout composer.json functionalTests: diff --git a/composer.json b/composer.json index b704e77..c260226 100644 --- a/composer.json +++ b/composer.json @@ -15,8 +15,8 @@ "TYPO3\\CMS\\Core\\Tests\\": ".Build/vendor/typo3/cms/typo3/sysext/core/Tests/" } }, - "require" : { - "php": ">=7.1.0", + "require": { + "php": ">=7.0.0", "typo3/cms": "~8.7", "ruflin/elastica": "~3.2" }, From 47b32820346282fe7969e904ebf18583d0b8c1ae Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 22 Feb 2018 21:55:52 +0100 Subject: [PATCH 111/158] BUGFIX: Allow indexing of new records with their relations Relations were inserted by TYPO3's DataHandler after indexing and were therefore not indexed. We now use a later hook after DataHandler has finished everything, so we know that we can index. As it's not relevant, we do not differentiate between add and update anymore, as both trigger "indexDocument" internal. Resolves: #112 --- Classes/Domain/Service/DataHandler.php | 10 ---- Classes/Hook/DataHandler.php | 50 ++++++++--------- .../Form/Finisher/DataHandlerFinisher.php | 4 +- .../IgnoresUnkownOperationTest.php | 54 ------------------- .../DataHandler/NonAllowedTablesTest.php | 6 +-- .../ProcessesAllowedTablesTest.php | 6 +-- .../Form/Finisher/DataHandlerFinisherTest.php | 11 ++-- 7 files changed, 32 insertions(+), 109 deletions(-) delete mode 100644 Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 6ac8069..10a01fe 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -83,16 +83,6 @@ class DataHandler implements Singleton $this->indexerFactory = $indexerFactory; } - /** - * @param string $table - * @param array $record - */ - public function add($table, array $record) - { - $this->logger->debug('Record received for add.', [$table, $record]); - $this->getIndexer($table)->indexDocument($record['uid']); - } - /** * @param string $table */ diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index d0eb1ba..fa4b09f 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -92,42 +92,36 @@ class DataHandler implements Singleton return true; } - /** - * Called by CoreDataHandler on database operations, e.g. if new records were created or records were updated. - * - * @param string $status - * @param string $table - * @param string|int $uid - * @param array $fieldArray - * @param CoreDataHandler $dataHandler - * - * @return bool False if hook was not processed. - */ - public function processDatamap_afterDatabaseOperations($status, $table, $uid, array $fieldArray, CoreDataHandler $dataHandler) + public function processDatamap_afterAllOperations(CoreDataHandler $dataHandler) + { + foreach ($dataHandler->datamap as $table => $record) { + $uid = key($record); + $fieldData = current($record); + + if (isset($fieldArray['uid'])) { + $uid = $fieldArray['uid']; + } elseif (isset($dataHandler->substNEWwithIDs[$uid])) { + $uid = $dataHandler->substNEWwithIDs[$uid]; + } + + $this->processRecord($table, $uid); + } + } + + protected function processRecord(string $table, int $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { - $this->logger->debug('Database update not processed.', [$table, $uid]); + $this->logger->debug('Indexing of record not processed.', [$table, $uid]); return false; } - if ($status === 'new') { - $fieldArray['uid'] = $dataHandler->substNEWwithIDs[$uid]; - $this->dataHandler->add($table, $fieldArray); + $record = $this->getRecord($table, $uid); + if ($record !== null) { + $this->dataHandler->update($table, $record); return true; } - if ($status === 'update') { - $record = $this->getRecord($table, $uid); - if ($record !== null) { - $this->dataHandler->update($table, $record); - } - return true; - } - - $this->logger->debug( - 'Database update not processed, cause status is unhandled.', - [$status, $table, $uid, $fieldArray] - ); + $this->logger->debug('Indexing of record not processed, as he was not found in Database.', [$table, $uid]); return false; } diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php index 02d6bde..696d8ad 100644 --- a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -58,10 +58,8 @@ class DataHandlerFinisher extends AbstractFinisher switch ($action) { case 'update': - $this->dataHandler->update($tableName, $record); - break; case 'add': - $this->dataHandler->add($tableName, $record); + $this->dataHandler->update($tableName, $record); break; case 'delete': $this->dataHandler->delete($tableName, $record['uid']); diff --git a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php b/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php deleted file mode 100644 index b1676e3..0000000 --- a/Tests/Functional/Hooks/DataHandler/IgnoresUnkownOperationTest.php +++ /dev/null @@ -1,54 +0,0 @@ - - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Domain\Service\DataHandler as DataHandlerService; -use Codappix\SearchCore\Hook\DataHandler as DataHandlerHook; -use TYPO3\CMS\Core\DataHandling\DataHandler as Typo3DataHandler; -use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Object\ObjectManager; - -class IgnoresUnkownOperationTest extends AbstractDataHandlerTest -{ - /** - * @var DataHandlerService|\PHPUnit_Framework_MockObject_MockObject|AccessibleObjectInterface - */ - protected $subject; - - /** - * @test - */ - public function dataHandlerCommandSomethingIsIgnored() - { - $subject = new DataHandlerHook($this->subject); - $this->assertFalse( - $subject->processDatamap_afterDatabaseOperations( - 'something', - 'tt_content', - 1, - [], - new Typo3DataHandler - ), - 'Hook processed status "something".' - ); - } -} diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index c33701d..509f053 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -64,7 +64,7 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function updateWillNotBeTriggeredForSysCategory() + public function updateWillNotBeTriggeredForExistingSysCategory() { $this->subject->expects($this->exactly(0))->method('update'); @@ -83,9 +83,9 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function addWillNotBeTriggeredForSysCategoy() + public function updateWillNotBeTriggeredForNewSysCategoy() { - $this->subject->expects($this->exactly(0))->method('add'); + $this->subject->expects($this->exactly(0))->method('update'); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 6a027cf..29ee14c 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -66,7 +66,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function updateWillBeTriggeredForTtContent() + public function updateWillBeTriggeredForExistingTtContent() { $this->subject->expects($this->exactly(1))->method('update') ->with( @@ -94,9 +94,9 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest /** * @test */ - public function addWillBeTriggeredForTtContent() + public function updateWillBeTriggeredForNewTtContent() { - $this->subject->expects($this->exactly(1))->method('add') + $this->subject->expects($this->exactly(1))->method('update') ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php index 2211cc2..24c5059 100644 --- a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -83,19 +83,14 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase public function possibleFinisherSetup() : array { return [ - 'valid add configuration' => [ - 'action' => 'add', - 'nonCalledActions' => ['delete', 'update'], - 'expectedSecondArgument' => ['uid' => 23], - ], 'valid update configuration' => [ 'action' => 'update', - 'nonCalledActions' => ['delete', 'add'], + 'nonCalledActions' => ['delete'], 'expectedSecondArgument' => ['uid' => 23], ], 'valid delete configuration' => [ 'action' => 'delete', - 'nonCalledActions' => ['update', 'add'], + 'nonCalledActions' => ['update'], 'expectedSecondArgument' => 23, ], ]; @@ -109,7 +104,7 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase { $this->subject->setOptions($options); - foreach (['add', 'update', 'delete'] as $nonCalledAction) { + foreach (['update', 'delete'] as $nonCalledAction) { $this->dataHandlerMock->expects($this->never())->method($nonCalledAction); } From 350f8a52b621500433f3a776fb6bf03c34c1a96b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 20 Feb 2018 12:01:28 +0100 Subject: [PATCH 112/158] FEATURE: Use extbase for processor instantiation This way injects will be resolved inside of processors, enabling developers to inject dependencies. We use inject instead of constructor as indexers mostly will change the constructor and should not need to add the objectmanager. Resolves: #117 --- Classes/Domain/Index/AbstractIndexer.php | 9 ++++++++- Documentation/source/configuration/indexing.rst | 3 +++ Tests/Unit/Domain/Index/AbstractIndexerTest.php | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index bc8674e..c48ecb6 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -48,6 +48,12 @@ abstract class AbstractIndexer implements IndexerInterface */ protected $logger; + /** + * @var \TYPO3\CMS\Extbase\Object\ObjectManagerInterface + * @inject + */ + protected $objectManager; + /** * Inject log manager to get concrete logger from it. * @@ -141,7 +147,8 @@ abstract class AbstractIndexer implements IndexerInterface } else { $className = $configuration['_typoScriptNodeValue']; } - $dataProcessor = GeneralUtility::makeInstance($className); + + $dataProcessor = $this->objectManager->get($className); if ($dataProcessor instanceof ProcessorInterface) { $record = $dataProcessor->processRecord($record, $configuration); } diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index b605008..619e56a 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -205,3 +205,6 @@ class name) as done in the examples above. By implementing also the same interface as necessary for TYPO3 :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code also for Fluid to prepare the same record fetched from DB for your fluid. + +Dependency injection is possible inside of processors, as we instantiate through extbase +``ObjectManager``. diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php index 3bbc97d..8c1e527 100644 --- a/Tests/Unit/Domain/Index/AbstractIndexerTest.php +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\DataProcessing\CopyToProcessor; use Codappix\SearchCore\Domain\Index\AbstractIndexer; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; class AbstractIndexerTest extends AbstractUnitTestCase { @@ -44,17 +45,25 @@ class AbstractIndexerTest extends AbstractUnitTestCase */ protected $connection; + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + public function setUp() { parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + $this->objectManager = $this->getMockBuilder(ObjectManagerInterface::class)->getMock(); + $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ $this->connection, $this->configuration ]); + $this->inject($this->subject, 'objectManager', $this->objectManager); $this->subject->injectLogger($this->getMockedLogger()); $this->subject->setIdentifier('testTable'); $this->subject->expects($this->any()) @@ -73,6 +82,11 @@ class AbstractIndexerTest extends AbstractUnitTestCase $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; $expectedRecord['search_abstract'] = ''; + $this->objectManager->expects($this->any()) + ->method('get') + ->with(CopyToProcessor::class) + ->willReturn(new CopyToProcessor()); + $this->configuration->expects($this->exactly(2)) ->method('get') ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) From 2998c43ba824492bbb888e6b99c1c0f349c7e5ad Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 1 Mar 2018 08:03:51 +0100 Subject: [PATCH 113/158] TASK: Refactor data processing Use own service to handle data processing. Classes like indexer should not know anything about the structure and how to process the data. Also rename record to data, as we can process just any data in form of an array. Relates: #116 --- Classes/DataProcessing/CopyToProcessor.php | 2 +- Classes/DataProcessing/GeoPointProcessor.php | 2 +- Classes/DataProcessing/ProcessorInterface.php | 7 +-- Classes/DataProcessing/RemoveProcessor.php | 2 +- Classes/DataProcessing/Service.php | 56 +++++++++++++++++++ Classes/Domain/Index/AbstractIndexer.php | 18 +++--- .../DataProcessing/CopyToProcessorTest.php | 16 +++--- .../DataProcessing/GeoPointProcessorTest.php | 20 +++---- .../DataProcessing/RemoveProcessorTest.php | 22 ++++---- 9 files changed, 98 insertions(+), 47 deletions(-) create mode 100644 Classes/DataProcessing/Service.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index eea555f..28c4294 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class CopyToProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) : array + public function processData(array $record, array $configuration) : array { $all = []; diff --git a/Classes/DataProcessing/GeoPointProcessor.php b/Classes/DataProcessing/GeoPointProcessor.php index f9256b3..c5b6d18 100644 --- a/Classes/DataProcessing/GeoPointProcessor.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class GeoPointProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) : array + public function processData(array $record, array $configuration) : array { if (! $this->canApply($record, $configuration)) { return $record; diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php index 4dc0794..f7513f2 100644 --- a/Classes/DataProcessing/ProcessorInterface.php +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -21,14 +21,13 @@ namespace Codappix\SearchCore\DataProcessing; */ /** - * All DataProcessing Processors should implement this interface, otherwise they - * will not be executed. + * All DataProcessing Processors should implement this interface. */ interface ProcessorInterface { /** - * Processes the given record. + * Processes the given data. * Also retrieves the configuration for this processor instance. */ - public function processRecord(array $record, array $configuration) : array; + public function processData(array $record, array $configuration) : array; } diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php index 6e74237..b8d6283 100644 --- a/Classes/DataProcessing/RemoveProcessor.php +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -27,7 +27,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RemoveProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) : array + public function processData(array $record, array $configuration) : array { if (!isset($configuration['fields'])) { return $record; diff --git a/Classes/DataProcessing/Service.php b/Classes/DataProcessing/Service.php new file mode 100644 index 0000000..3e3b053 --- /dev/null +++ b/Classes/DataProcessing/Service.php @@ -0,0 +1,56 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; + +/** + * Eases work with data processing. + */ +class Service +{ + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + public function __construct(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + + /** + * Executes the dataprocessor depending on configuration and returns the result. + * + * @param array|string $configuration Either the full configuration or only the class name. + */ + public function executeDataProcessor($configuration, array $data) : array + { + if (is_string($configuration)) { + $configuration = [ + '_typoScriptNodeValue' => $configuration, + ]; + } + + return $this->objectManager->get($configuration['_typoScriptNodeValue']) + ->processData($data, $configuration); + } +} diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index bc8674e..2de7dd5 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -43,6 +43,12 @@ abstract class AbstractIndexer implements IndexerInterface */ protected $identifier = ''; + /** + * @var \Codappix\SearchCore\DataProcessing\Service + * @inject + */ + protected $dataProcessorService; + /** * @var \TYPO3\CMS\Core\Log\Logger */ @@ -134,17 +140,7 @@ abstract class AbstractIndexer implements IndexerInterface { try { foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { - $className = ''; - if (is_string($configuration)) { - $className = $configuration; - $configuration = []; - } else { - $className = $configuration['_typoScriptNodeValue']; - } - $dataProcessor = GeneralUtility::makeInstance($className); - if ($dataProcessor instanceof ProcessorInterface) { - $record = $dataProcessor->processRecord($record, $configuration); - } + $record = $this->dataProcessorService->executeDataProcessor($configuration, $record); } } catch (InvalidArgumentException $e) { // Nothing to do. diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php index 2f9e498..28545a3 100644 --- a/Tests/Unit/DataProcessing/CopyToProcessorTest.php +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -27,15 +27,15 @@ class CopyToProcessorTest extends AbstractUnitTestCase { /** * @test - * @dataProvider getPossibleRecordConfigurationCombinations + * @dataProvider getPossibleDataConfigurationCombinations */ - public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedData) { $subject = new CopyToProcessor(); - $processedRecord = $subject->processRecord($record, $configuration); + $processedData = $subject->processData($record, $configuration); $this->assertSame( - $expectedRecord, - $processedRecord, + $expectedData, + $processedData, 'The processor did not return the expected processed record.' ); } @@ -43,7 +43,7 @@ class CopyToProcessorTest extends AbstractUnitTestCase /** * @return array */ - public function getPossibleRecordConfigurationCombinations() + public function getPossibleDataConfigurationCombinations() { return [ 'Copy all fields to new field' => [ @@ -54,7 +54,7 @@ class CopyToProcessorTest extends AbstractUnitTestCase 'configuration' => [ 'to' => 'new_field', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field 2' => 'Some more content like ipsum', 'new_field' => 'Some content like lorem' . PHP_EOL . 'Some more content like ipsum', @@ -71,7 +71,7 @@ class CopyToProcessorTest extends AbstractUnitTestCase 'configuration' => [ 'to' => 'new_field', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field with sub2' => [ 'Tag 1', diff --git a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php index 99b8eb3..02db659 100644 --- a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php +++ b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php @@ -27,15 +27,15 @@ class GeoPointProcessorTest extends AbstractUnitTestCase { /** * @test - * @dataProvider getPossibleRecordConfigurationCombinations + * @dataProvider getPossibleDataConfigurationCombinations */ - public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord) + public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedData) { $subject = new GeoPointProcessor(); - $processedRecord = $subject->processRecord($record, $configuration); + $processedData = $subject->processData($record, $configuration); $this->assertSame( - $expectedRecord, - $processedRecord, + $expectedData, + $processedData, 'The processor did not return the expected processed record.' ); } @@ -43,7 +43,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase /** * @return array */ - public function getPossibleRecordConfigurationCombinations() + public function getPossibleDataConfigurationCombinations() { return [ 'Create new field with existing lat and lng' => [ @@ -56,7 +56,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'lat' => 'lat', 'lon' => 'lng', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => 23.232, 'lng' => 45.43, 'location' => [ @@ -73,7 +73,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'configuration' => [ 'to' => 'location', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => 23.232, 'lng' => 45.43, ], @@ -88,7 +88,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'lat' => 'lat', 'lon' => 'lng', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => '', 'lng' => '', ], @@ -103,7 +103,7 @@ class GeoPointProcessorTest extends AbstractUnitTestCase 'lat' => 'lat', 'lon' => 'lng', ], - 'expectedRecord' => [ + 'expectedData' => [ 'lat' => 'av', 'lng' => 'dsf', ], diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php index dc55f73..cd23d16 100644 --- a/Tests/Unit/DataProcessing/RemoveProcessorTest.php +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -27,15 +27,15 @@ class RemoveProcessorTest extends AbstractUnitTestCase { /** * @test - * @dataProvider getPossibleRecordConfigurationCombinations + * @dataProvider getPossibleDataConfigurationCombinations */ - public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedData) { $subject = new RemoveProcessor(); - $processedRecord = $subject->processRecord($record, $configuration); + $processedData = $subject->processData($record, $configuration); $this->assertSame( - $expectedRecord, - $processedRecord, + $expectedData, + $processedData, 'The processor did not return the expected processed record.' ); } @@ -43,7 +43,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase /** * @return array */ - public function getPossibleRecordConfigurationCombinations() + public function getPossibleDataConfigurationCombinations() { return [ 'Nothing configured' => [ @@ -56,7 +56,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase ], 'configuration' => [ ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field with sub2' => [ 'Tag 1', @@ -76,7 +76,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'field with sub2', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', ], ], @@ -92,7 +92,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'non existing', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', 'field with sub2' => [ 'Tag 1', @@ -113,7 +113,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'field 3, field with sub2', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ 'field 1' => 'Some content like lorem', ], ], @@ -125,7 +125,7 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'fields' => 'field 1', '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', ], - 'expectedRecord' => [ + 'expectedData' => [ ], ], ]; From 769fb8237b14671f8e50717c100b39a7f06dc97c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:04:47 +0100 Subject: [PATCH 114/158] TASK: Add data processing to search result Search results are now processed through data processing by search service. The result will be a SearchResult model from our domain. Also SearchResult will execute new queries, e.g. from paginate widget, through SearchService to apply data processing again. Remove duplicate stub code to trait, to keep own logic and code clean. --- .../Connection/Elasticsearch/SearchResult.php | 39 +----- Classes/Connection/ResultItemInterface.php | 9 +- Classes/Connection/SearchRequestInterface.php | 7 + .../Domain/Model/QueryResultInterfaceStub.php | 61 +++++++++ .../Model}/ResultItem.php | 11 +- Classes/Domain/Model/SearchRequest.php | 13 +- Classes/Domain/Model/SearchResult.php | 129 ++++++++++++++++++ Classes/Domain/Search/SearchService.php | 46 ++++++- .../Unit/Domain/Index/AbstractIndexerTest.php | 35 ++++- .../Unit/Domain/Search/SearchServiceTest.php | 84 +++++++++--- 10 files changed, 369 insertions(+), 65 deletions(-) create mode 100644 Classes/Domain/Model/QueryResultInterfaceStub.php rename Classes/{Connection/Elasticsearch => Domain/Model}/ResultItem.php (88%) create mode 100644 Classes/Domain/Model/SearchResult.php diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 3919722..867d823 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -24,10 +24,14 @@ use Codappix\SearchCore\Connection\FacetInterface; use Codappix\SearchCore\Connection\ResultItemInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub; +use Codappix\SearchCore\Domain\Model\ResultItem; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; class SearchResult implements SearchResultInterface { + use QueryResultInterfaceStub; + /** * @var SearchRequestInterface */ @@ -104,7 +108,7 @@ class SearchResult implements SearchResultInterface } foreach ($this->result->getResults() as $result) { - $this->results[] = new ResultItem($result); + $this->results[] = new ResultItem($result->getData()); } } @@ -153,41 +157,8 @@ class SearchResult implements SearchResultInterface $this->position = 0; } - // Extbase QueryResultInterface - Implemented to support Pagination of Fluid. - public function getQuery() { return $this->searchRequest; } - - public function getFirst() - { - throw new \BadMethodCallException('Method is not implemented yet.', 1502195121); - } - - public function toArray() - { - throw new \BadMethodCallException('Method is not implemented yet.', 1502195135); - } - - public function offsetExists($offset) - { - // Return false to allow Fluid to use appropriate getter methods. - return false; - } - - public function offsetGet($offset) - { - throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933); - } - - public function offsetSet($offset, $value) - { - throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934); - } - - public function offsetUnset($offset) - { - throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936); - } } diff --git a/Classes/Connection/ResultItemInterface.php b/Classes/Connection/ResultItemInterface.php index 56add2a..d7fe1a9 100644 --- a/Classes/Connection/ResultItemInterface.php +++ b/Classes/Connection/ResultItemInterface.php @@ -25,5 +25,12 @@ namespace Codappix\SearchCore\Connection; */ interface ResultItemInterface extends \ArrayAccess { - + /** + * Returns every information as array. + * + * Provide key/column/field => data. + * + * Used e.g. for dataprocessing. + */ + public function getPlainData() : array; } diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index 7c7956e..e24adc3 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ +use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Persistence\QueryInterface; interface SearchRequestInterface extends QueryInterface @@ -40,4 +41,10 @@ interface SearchRequestInterface extends QueryInterface * @return array */ public function getFilter(); + + /** + * Workaround for paginate widget support which will + * use the request to build another search. + */ + public function setSearchService(SearchService $searchService); } diff --git a/Classes/Domain/Model/QueryResultInterfaceStub.php b/Classes/Domain/Model/QueryResultInterfaceStub.php new file mode 100644 index 0000000..960fd40 --- /dev/null +++ b/Classes/Domain/Model/QueryResultInterfaceStub.php @@ -0,0 +1,61 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * As we have to stay compatible with QueryResultInterface + * of extbase but can and need not to provide all methods, + * this stub will provde the non implemented methods to + * keep real implementations clean. + */ +trait QueryResultInterfaceStub +{ + public function getFirst() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502195121); + } + + public function toArray() + { + throw new \BadMethodCallException('Method is not implemented yet.', 1502195135); + } + + public function offsetExists($offset) + { + // Return false to allow Fluid to use appropriate getter methods. + return false; + } + + public function offsetGet($offset) + { + throw new \BadMethodCallException('Use getter to fetch properties.', 1502196933); + } + + public function offsetSet($offset, $value) + { + throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196934); + } + + public function offsetUnset($offset) + { + throw new \BadMethodCallException('You are not allowed to modify the result.', 1502196936); + } +} diff --git a/Classes/Connection/Elasticsearch/ResultItem.php b/Classes/Domain/Model/ResultItem.php similarity index 88% rename from Classes/Connection/Elasticsearch/ResultItem.php rename to Classes/Domain/Model/ResultItem.php index e56783c..f63b408 100644 --- a/Classes/Connection/Elasticsearch/ResultItem.php +++ b/Classes/Domain/Model/ResultItem.php @@ -1,5 +1,5 @@ @@ -29,9 +29,14 @@ class ResultItem implements ResultItemInterface */ protected $data = []; - public function __construct(\Elastica\Result $result) + public function __construct(array $result) { - $this->data = $result->getData(); + $this->data = $data; + } + + public function getPlainData() : array + { + return $this->data; } public function offsetExists($offset) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index d751ce8..d835e7b 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Domain\Model; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\FacetRequestInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Domain\Search\SearchService; /** * Represents a search request used to process an actual search. @@ -63,6 +64,11 @@ class SearchRequest implements SearchRequestInterface */ protected $connection = null; + /** + * @var SearchService + */ + protected $searchService = null; + /** * @param string $query */ @@ -143,12 +149,17 @@ class SearchRequest implements SearchRequestInterface $this->connection = $connection; } + public function setSearchService(SearchService $searchService) + { + $this->searchService = $searchService; + } + // Extbase QueryInterface // Current implementation covers only paginate widget support. public function execute($returnRawQueryResult = false) { if ($this->connection instanceof ConnectionInterface) { - return $this->connection->search($this); + return $this->searchService->processResult($this->connection->search($this)); } throw new \InvalidArgumentException( diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php new file mode 100644 index 0000000..d91820d --- /dev/null +++ b/Classes/Domain/Model/SearchResult.php @@ -0,0 +1,129 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\QueryResultInterfaceStub; + +/** + * Generic model for mapping a concrete search result from a connection. + */ +class SearchResult implements SearchResultInterface +{ + use QueryResultInterfaceStub; + + /** + * @var SearchResultInterface + */ + protected $originalSearchResult; + + /** + * @var array + */ + protected $resultItems = []; + + /** + * @var array + */ + protected $results = []; + + /** + * For Iterator interface. + * + * @var int + */ + protected $position = 0; + + public function __construct(SearchResultInterface $originalSearchResult, array $resultItems) + { + $this->originalSearchResult = $originalSearchResult; + $this->resultItems = $resultItems; + } + + /** + * @return array + */ + public function getResults() + { + $this->initResults(); + + return $this->results; + } + + protected function initResults() + { + if ($this->results !== []) { + return; + } + + foreach ($this->resultItems as $item) { + $this->results[] = new ResultItem($item); + } + } + + public function getFacets() + { + return $this->originalSearchResult->getFacets(); + } + + public function getCurrentCount() + { + return $this->originalSearchResult->getCurrentCount(); + } + + public function count() + { + return $this->originalSearchResult->count(); + } + + public function current() + { + return $this->getResults()[$this->position]; + } + + public function next() + { + ++$this->position; + + return $this->current(); + } + + public function key() + { + return $this->position; + } + + public function valid() + { + return isset($this->getResults()[$this->position]); + } + + public function rewind() + { + $this->position = 0; + } + + public function getQuery() + { + return $this->originalSearchResult->getQuery(); + } +} diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 120ab3b..48be32c 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Model\FacetRequest; +use Codappix\SearchCore\Domain\Model\SearchResult; use TYPO3\CMS\Core\Utility\ArrayUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -49,19 +51,27 @@ class SearchService */ protected $objectManager; + /** + * @var DataProcessorService + */ + protected $dataProcessorService; + /** * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration * @param ObjectManagerInterface $objectManager + * @param DataProcessorService $dataProcessorService */ public function __construct( ConnectionInterface $connection, ConfigurationContainerInterface $configuration, - ObjectManagerInterface $objectManager + ObjectManagerInterface $objectManager, + DataProcessorService $dataProcessorService ) { $this->connection = $connection; $this->configuration = $configuration; $this->objectManager = $objectManager; + $this->dataProcessorService = $dataProcessorService; } /** @@ -70,12 +80,15 @@ class SearchService */ public function search(SearchRequestInterface $searchRequest) { - $searchRequest->setConnection($this->connection); $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); $this->addConfiguredFilters($searchRequest); - return $this->connection->search($searchRequest); + // Add connection to request to enable paginate widget support + $searchRequest->setConnection($this->connection); + $searchRequest->setSearchService($this); + + return $this->processResult($this->connection->search($searchRequest)); } /** @@ -138,4 +151,31 @@ class SearchService // Nothing todo, no filter configured. } } + + /** + * Processes the result, e.g. applies configured data processing to result. + */ + public function processResult(SearchResultInterface $searchResult) : SearchResultInterface + { + try { + $newSearchResultItems = []; + foreach ($this->configuration->get('searching.dataProcessing') as $configuration) { + foreach ($searchResult as $resultItem) { + $newSearchResultItems[] = $this->dataProcessorService->executeDataProcessor( + $configuration, + $resultItem->getPlainData() + ); + } + } + + return $this->objectManager->get( + SearchResult::class, + $searchResult, + $newSearchResultItems + ); + } + catch (InvalidArgumentException $e) { + return $searchResult; + } + } } diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php index 3bbc97d..411cc24 100644 --- a/Tests/Unit/Domain/Index/AbstractIndexerTest.php +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -24,6 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Index\AbstractIndexer; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -44,17 +45,26 @@ class AbstractIndexerTest extends AbstractUnitTestCase */ protected $connection; + /** + * @var DataProcessorService + */ + protected $dataProcessorService; + public function setUp() { parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + $this->dataProcessorService = $this->getMockBuilder(DataProcessorService::class) + ->disableOriginalConstructor() + ->getMock(); $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ $this->connection, $this->configuration ]); + $this->inject($this->subject, 'dataProcessorService', $this->dataProcessorService); $this->subject->injectLogger($this->getMockedLogger()); $this->subject->setIdentifier('testTable'); $this->subject->expects($this->any()) @@ -73,7 +83,30 @@ class AbstractIndexerTest extends AbstractUnitTestCase $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; $expectedRecord['search_abstract'] = ''; - $this->configuration->expects($this->exactly(2)) + $this->dataProcessorService->expects($this->any()) + ->method('executeDataProcessor') + ->withConsecutive( + [ + [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + $record, + ], + [ + [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + array_merge($record, ['new_test_field' => 'test']), + ] + ) + ->will($this->onConsecutiveCalls( + array_merge($record, ['new_test_field' => 'test']), + $expectedRecord + )); + + $this->configuration->expects($this->any()) ->method('get') ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) ->will($this->onConsecutiveCalls([ diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index c63e29e..aaf2e5c 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -23,6 +23,8 @@ namespace Copyright\SearchCore\Tests\Unit\Domain\Search; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\SearchService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -35,24 +37,59 @@ class SearchServiceTest extends AbstractUnitTestCase */ protected $subject; + /** + * @var SearchResultInterface + */ + protected $result; + + /** + * @var ConnectionInterface + */ + protected $connection; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + /** + * @var DataProcessorService + */ + protected $dataProcessorService; + public function setUp() { parent::setUp(); + $this->result = $this->getMockBuilder(SearchResultInterface::class) + ->disableOriginalConstructor() + ->getMock(); $this->connection = $this->getMockBuilder(ConnectionInterface::class) ->disableOriginalConstructor() ->getMock(); + $this->connection->expects($this->any()) + ->method('search') + ->willReturn($this->result); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) ->disableOriginalConstructor() ->getMock(); $this->objectManager = $this->getMockBuilder(ObjectManagerInterface::class) ->disableOriginalConstructor() ->getMock(); + $this->dataProcessorService = $this->getMockBuilder(DataProcessorService::class) + ->setConstructorArgs([$this->objectManager]) + ->getMock(); $this->subject = new SearchService( $this->connection, $this->configuration, - $this->objectManager + $this->objectManager, + $this->dataProcessorService ); } @@ -61,13 +98,12 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function sizeIsAddedFromConfiguration() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(45, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) ->method('search') @@ -84,13 +120,12 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function defaultSizeIsAddedIfNothingIsConfigured() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) ->method('search') @@ -107,14 +142,16 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function configuredFilterAreAddedToRequestWithoutAnyFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') - ->willReturn(['property' => 'something']); + ->will($this->onConsecutiveCalls( + ['property' => 'something'], + $this->throwException(new InvalidArgumentException) + )); $this->connection->expects($this->once()) ->method('search') @@ -131,14 +168,16 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function configuredFilterAreAddedToRequestWithExistingFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') - ->willReturn(['property' => 'something']); + ->will($this->onConsecutiveCalls( + ['property' => 'something'], + $this->throwException(new InvalidArgumentException) + )); $this->connection->expects($this->once()) ->method('search') @@ -159,13 +198,12 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function nonConfiguredFilterIsNotChangingRequestWithExistingFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') ->will($this->throwException(new InvalidArgumentException)); $this->connection->expects($this->once()) @@ -184,14 +222,16 @@ class SearchServiceTest extends AbstractUnitTestCase */ public function emptyConfiguredFilterIsNotChangingRequestWithExistingFilter() { - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('getIfExists') ->withConsecutive(['searching.size'], ['searching.facets']) ->will($this->onConsecutiveCalls(null, null)); - $this->configuration->expects($this->exactly(1)) + $this->configuration->expects($this->any()) ->method('get') - ->with('searching.filter') - ->willReturn(['anotherProperty' => '']); + ->will($this->onConsecutiveCalls( + ['anotherProperty' => ''], + $this->throwException(new InvalidArgumentException) + )); $this->connection->expects($this->once()) ->method('search') From 79aba3c544003afc6e2628f864aeb9edd1da5cf5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:19:09 +0100 Subject: [PATCH 115/158] TASK: Add test cases for ResultItem Model Check whether all methods work as expected. E.g. we can retrieve data in all ways, but not change anything. --- Classes/Domain/Model/ResultItem.php | 2 +- Tests/Unit/Domain/Model/ResultItemTest.php | 113 +++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 Tests/Unit/Domain/Model/ResultItemTest.php diff --git a/Classes/Domain/Model/ResultItem.php b/Classes/Domain/Model/ResultItem.php index f63b408..d0e370a 100644 --- a/Classes/Domain/Model/ResultItem.php +++ b/Classes/Domain/Model/ResultItem.php @@ -31,7 +31,7 @@ class ResultItem implements ResultItemInterface public function __construct(array $result) { - $this->data = $data; + $this->data = $result; } public function getPlainData() : array diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php new file mode 100644 index 0000000..df11a5d --- /dev/null +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Model\ResultItem; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class ResultItemTest extends AbstractUnitTestCase +{ + /** + * @test + */ + public function plainDataCanBeRetrieved() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->assertSame( + $expectedData, + $subject->getPlainData(), + 'Could not retrieve plain data from result item.' + ); + } + + /** + * @test + */ + public function dataCanBeRetrievedInArrayNotation() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->assertSame( + 'Some title', + $subject['title'], + 'Could not retrieve title in array notation.' + ); + } + + /** + * @test + */ + public function existenceOfDataCanBeChecked() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->assertTrue(isset($subject['title']), 'Could not determine that title exists.'); + $this->assertFalse(isset($subject['title2']), 'Could not determine that title2 does not exists.'); + } + + /** + * @test + */ + public function dataCanNotBeChanged() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->expectException(\BadMethodCallException::class); + $subject['title'] = 'New Title'; + } + + /** + * @test + */ + public function dataCanNotBeRemoved() + { + $originalData = [ + 'uid' => 10, + 'title' => 'Some title', + ]; + $expectedData = $originalData; + + $subject = new ResultItem($originalData); + $this->expectException(\BadMethodCallException::class); + unset($subject['title']); + } +} From cf91251be36e08cd749457adbbf1e1ce5404f15f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:36:39 +0100 Subject: [PATCH 116/158] TASK :Add tests for SearchRequest Make sure exceptions with helpful messages are thrown if one object is missing when execute is called. Also make sure the expected methods are called. --- Classes/Domain/Model/SearchRequest.php | 18 +++-- Tests/Unit/Domain/Model/SearchRequestTest.php | 69 +++++++++++++++++-- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index d835e7b..69237bb 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -158,14 +158,20 @@ class SearchRequest implements SearchRequestInterface // Current implementation covers only paginate widget support. public function execute($returnRawQueryResult = false) { - if ($this->connection instanceof ConnectionInterface) { - return $this->searchService->processResult($this->connection->search($this)); + if (! ($this->connection instanceof ConnectionInterface)) { + throw new \InvalidArgumentException( + 'Connection was not set before, therefore execute can not work. Use `setConnection` before.', + 1502197732 + ); + } + if (! ($this->searchService instanceof SearchService)) { + throw new \InvalidArgumentException( + 'SearchService was not set before, therefore execute can not work. Use `setSearchService` before.', + 1520325175 + ); } - throw new \InvalidArgumentException( - 'Connection was not set before, therefore execute can not work. Use `setConnection` before.', - 1502197732 - ); + return $this->searchService->processResult($this->connection->search($this)); } public function setLimit($limit) diff --git a/Tests/Unit/Domain/Model/SearchRequestTest.php b/Tests/Unit/Domain/Model/SearchRequestTest.php index 127baf2..28f90e3 100644 --- a/Tests/Unit/Domain/Model/SearchRequestTest.php +++ b/Tests/Unit/Domain/Model/SearchRequestTest.php @@ -20,7 +20,10 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Model; * 02110-1301, USA. */ +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; class SearchRequestTest extends AbstractUnitTestCase @@ -31,12 +34,12 @@ class SearchRequestTest extends AbstractUnitTestCase */ public function emptyFilterWillNotBeSet(array $filter) { - $searchRequest = new SearchRequest(); - $searchRequest->setFilter($filter); + $subject = new SearchRequest(); + $subject->setFilter($filter); $this->assertSame( [], - $searchRequest->getFilter(), + $subject->getFilter(), 'Empty filter were set, even if they should not.' ); } @@ -68,13 +71,67 @@ class SearchRequestTest extends AbstractUnitTestCase public function filterIsSet() { $filter = ['someField' => 'someValue']; - $searchRequest = new SearchRequest(); - $searchRequest->setFilter($filter); + $subject = new SearchRequest(); + $subject->setFilter($filter); $this->assertSame( $filter, - $searchRequest->getFilter(), + $subject->getFilter(), 'Filter was not set.' ); } + + /** + * @test + */ + public function exceptionIsThrownIfSearchServiceWasNotSet() + { + $subject = new SearchRequest(); + $subject->setConnection($this->getMockBuilder(ConnectionInterface::class)->getMock()); + $this->expectException(\InvalidArgumentException::class); + $subject->execute(); + } + + /** + * @test + */ + public function exceptionIsThrownIfConnectionWasNotSet() + { + $subject = new SearchRequest(); + $subject->setSearchService( + $this->getMockBuilder(SearchService::class) + ->disableOriginalConstructor() + ->getMock() + ); + $this->expectException(\InvalidArgumentException::class); + $subject->execute(); + } + + /** + * @test + */ + public function executionMakesUseOfProvidedConnectionAndSearchService() + { + $searchServiceMock = $this->getMockBuilder(SearchService::class) + ->disableOriginalConstructor() + ->getMock(); + $connectionMock = $this->getMockBuilder(ConnectionInterface::class) + ->getMock(); + $searchResultMock = $this->getMockBuilder(SearchResultInterface::class) + ->getMock(); + + $subject = new SearchRequest(); + $subject->setSearchService($searchServiceMock); + $subject->setConnection($connectionMock); + + $connectionMock->expects($this->once()) + ->method('search') + ->with($subject) + ->willReturn($searchResultMock); + $searchServiceMock->expects($this->once()) + ->method('processResult') + ->with($searchResultMock); + + $subject->execute(); + } } From 45bb12cf519bc51f69a15fff38e56fd1bdba86cd Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 09:44:26 +0100 Subject: [PATCH 117/158] TASK: Add tests for search result model --- Tests/Unit/Domain/Model/SearchResultTest.php | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 Tests/Unit/Domain/Model/SearchResultTest.php diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php new file mode 100644 index 0000000..910e06e --- /dev/null +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -0,0 +1,100 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\ResultItemInterface; +use Codappix\SearchCore\Connection\SearchResultInterface; +use Codappix\SearchCore\Domain\Model\SearchResult; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class SearchResultTest extends AbstractUnitTestCase +{ + /** + * @test + */ + public function countIsRetrievedFromOriginalResult() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $originalSearchResultMock->expects($this->once())->method('count'); + + $subject = new SearchResult($originalSearchResultMock, []); + $subject->count(); + } + + /** + * @test + */ + public function currentCountIsRetrievedFromOriginalResult() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $originalSearchResultMock->expects($this->once())->method('getCurrentCount'); + + $subject = new SearchResult($originalSearchResultMock, []); + $subject->getCurrentCount(); + } + + /** + * @test + */ + public function facetsAreRetrievedFromOriginalResult() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $originalSearchResultMock->expects($this->once())->method('getFacets'); + + $subject = new SearchResult($originalSearchResultMock, []); + $subject->getFacets(); + } + + /** + * @test + */ + public function resultItemsCanBeRetrieved() + { + $originalSearchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $data = [ + [ + 'uid' => 10, + 'title' => 'Some Title', + ], + [ + 'uid' => 11, + 'title' => 'Some Title 2', + ], + [ + 'uid' => 12, + 'title' => 'Some Title 3', + ], + ]; + + $subject = new SearchResult($originalSearchResultMock, $data); + $resultItems = $subject->getResults(); + + $this->assertCount(3, $resultItems); + + $this->assertSame(10, $resultItems[0]['uid']); + $this->assertSame(11, $resultItems[1]['uid']); + $this->assertSame(12, $resultItems[2]['uid']); + + $this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]); + $this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]); + $this->assertInstanceOf(ResultItemInterface::class, $resultItems[2]); + } +} From 0210110ccfcdee1e22316ddbfc94dd26c499bf86 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 10:26:39 +0100 Subject: [PATCH 118/158] TASK: Add test for applied data processing on search result --- .../Unit/Domain/Search/SearchServiceTest.php | 91 +++++++++++++++++-- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index aaf2e5c..6263f23 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -1,5 +1,5 @@ @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\DataProcessing\Service as DataProcessorService; use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Model\SearchResult; use Codappix\SearchCore\Domain\Search\SearchService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -72,9 +73,6 @@ class SearchServiceTest extends AbstractUnitTestCase $this->connection = $this->getMockBuilder(ConnectionInterface::class) ->disableOriginalConstructor() ->getMock(); - $this->connection->expects($this->any()) - ->method('search') - ->willReturn($this->result); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class) ->disableOriginalConstructor() ->getMock(); @@ -109,7 +107,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getLimit() === 45; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); @@ -131,7 +130,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getLimit() === 10; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); @@ -157,7 +157,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === ['property' => 'something']; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $this->subject->search($searchRequest); @@ -186,7 +187,8 @@ class SearchServiceTest extends AbstractUnitTestCase 'anotherProperty' => 'anything', 'property' => 'something', ]; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['anotherProperty' => 'anything']); @@ -210,7 +212,8 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['anotherProperty' => 'anything']); @@ -237,10 +240,78 @@ class SearchServiceTest extends AbstractUnitTestCase ->method('search') ->with($this->callback(function ($searchRequest) { return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; - })); + })) + ->willReturn($this->getMockBuilder(SearchResultInterface::class)->getMock()); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['anotherProperty' => 'anything']); $this->subject->search($searchRequest); } + + /** + * @test + */ + public function originalSearchResultIsReturnedIfNoDataProcessorIsConfigured() + { + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + + $this->connection->expects($this->once()) + ->method('search') + ->willReturn($searchResultMock); + + $this->dataProcessorService->expects($this->never())->method('executeDataProcessor'); + + $searchRequest = new SearchRequest(''); + $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result without applied data processing'); + } + + /** + * @test + */ + public function configuredDataProcessorsAreExecutedOnSearchResult() + { + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + ['SomeProcessorClass'] + )); + + $searchResultMock = $this->getMockBuilder(SearchResultInterface::class)->getMock(); + $searchResult = new SearchResult($searchResultMock, [['field 1' => 'value 1']]); + + $this->connection->expects($this->once()) + ->method('search') + ->willReturn($searchResult); + + $this->dataProcessorService->expects($this->once()) + ->method('executeDataProcessor') + ->with('SomeProcessorClass', ['field 1' => 'value 1']) + ->willReturn([ + 'field 1' => 'value 1', + 'field 2' => 'value 2', + ]); + + $this->objectManager->expects($this->once()) + ->method('get') + ->with(SearchResult::class, $searchResult, [ + ['field 1' => 'value 1', 'field 2' => 'value 2'] + ]) + ->willReturn($searchResultMock); + + $searchRequest = new SearchRequest(''); + $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result with applied data processing'); + } } From 176c466d7e49f072ed11ca23c03b5c84608b33ed Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 10:47:14 +0100 Subject: [PATCH 119/158] TASK: Update documentation for data processing --- Documentation/source/concepts.rst | 2 + .../dataProcessing/availableAndPlanned.rst | 32 ++++++++++++++++ .../source/configuration/indexing.rst | 37 ++----------------- .../source/configuration/searching.rst | 34 +++++++++++++++++ Documentation/source/features.rst | 10 +++++ 5 files changed, 81 insertions(+), 34 deletions(-) create mode 100644 Documentation/source/configuration/dataProcessing/availableAndPlanned.rst diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index 0fe6c5d..f4c5bbd 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -36,5 +36,7 @@ DataProcessing Before data is transfered to search service, it can be processed by "DataProcessors" like already known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`. +The same is true for retrieved search results. They can be processed again by "DataProcessors" to +prepare data for display in Templates or further usage. Configuration is done through TypoScript, see :ref:`dataProcessing`. diff --git a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst new file mode 100644 index 0000000..8284820 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst @@ -0,0 +1,32 @@ +The following Processor are available: + +.. toctree:: + :maxdepth: 1 + :glob: + + /configuration/dataProcessing/CopyToProcessor + /configuration/dataProcessing/RemoveProcessor + /configuration/dataProcessing/GeoPointProcessor + +The following Processor are planned: + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index b605008..7d6c3d4 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -146,7 +146,7 @@ Example:: dataProcessing -------------- -Used by: All connections while indexing. +Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``. Configure modifications on each document before sending it to the configured connection. Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through @@ -170,38 +170,7 @@ Example:: The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards all fields, including ``search_spellcheck`` will be copied to ``search_all``. -E.g. used to index all information into a field for :ref:`spellchecking` or searching with -different :ref:`mapping`. -The following Processor are available: +.. include:: /configuration/dataProcessing/availableAndPlanned.rst -.. toctree:: - :maxdepth: 1 - :glob: - - dataProcessing/CopyToProcessor - dataProcessing/RemoveProcessor - dataProcessing/GeoPointProcessor - -The following Processor are planned: - - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. - - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. - - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". - - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. - -Of course you are able to provide further processors. Just implement -``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified -class name) as done in the examples above. - -By implementing also the same interface as necessary for TYPO3 -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code -also for Fluid to prepare the same record fetched from DB for your fluid. +Also data processors are available for search results too, see :ref:`searching_dataProcessing`. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index ae73fad..bfed51e 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -216,3 +216,37 @@ Example:: } Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. + +.. _searching_dataProcessing: + +dataProcessing +-------------- + +Used by: All connections while indexing, due to implementation inside ``SearchService``. + +Configure modifications on each document before returning search result. Same as provided by TYPO3 +for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + +All processors are applied in configured order. Allowing to work with already processed data. + +Example:: + + plugin.tx_searchcore.settings.searching.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. + +.. include:: /configuration/dataProcessing/availableAndPlanned.rst + +Also data processors are available while indexing too, see :ref:`dataProcessing`. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 8baf453..1a4abd6 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -29,6 +29,16 @@ Also multiple filter are supported. Filtering results by fields for string conte Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. +.. _features_dataProcessing: + +DataProcessing +============== + +DataProcessing, as known from ``FLUIDTEMPLATE`` is available while indexing and for search results. +Each item can be processed by multiple processor to prepare data for indexing and output. + +See :ref:`concepts_indexing_dataprocessing` in :ref:`concepts` section. + .. _features_planned: Planned From 04aaad12fe611b994f0c9e7f704e0c73fcfc7079 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 11:16:00 +0100 Subject: [PATCH 120/158] FEATURE: Provide ContentObjectDataProcessorAdapterProcessor Allow integrator to execute any existing data processor for content objects. Resolves: #118 --- ...entObjectDataProcessorAdapterProcessor.php | 59 +++++++++++++++++++ ...entObjectDataProcessorAdapterProcessor.rst | 32 ++++++++++ .../dataProcessing/availableAndPlanned.rst | 3 +- ...bjectDataProcessorAdapterProcessorTest.php | 55 +++++++++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php create mode 100644 Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst create mode 100644 Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php new file mode 100644 index 0000000..322486b --- /dev/null +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -0,0 +1,59 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Service\TypoScriptService; +use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; + +/** + * Executes an existing TYPO3 DataProcessor on the given data. + */ +class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface +{ + /** + * @var TypoScriptService + */ + protected $typoScriptService; + + public function __construct(TypoScriptService $typoScriptService) + { + $this->typoScriptService = $typoScriptService; + } + + public function processData(array $data, array $configuration) : array + { + $dataProcessor = GeneralUtility::makeInstance($configuration['_dataProcessor']); + $contentObjectRenderer = GeneralUtility::makeInstance(ContentObjectRenderer::class); + + $contentObjectRenderer->data = $data; + if (isset($configuration['_table'])) { + $contentObjectRenderer->start($data, $configuration['_table']); + } + + return $dataProcessor->process( + $contentObjectRenderer, + [], + $this->typoScriptService->convertPlainArrayToTypoScriptArray($configuration), + $data + ); + } +} diff --git a/Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst b/Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst new file mode 100644 index 0000000..7d8f0b2 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor.rst @@ -0,0 +1,32 @@ +``Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor`` +================================================================================= + +Will execute an existing TYPO3 data processor. + +Possible Options: + +``_dataProcessor`` + Necessary, defined which data processor to apply. Provide the same as you would to call the + processor. +``_table`` + Defines the "current" table as used by some processors, e.g. + ``TYPO3\CMS\Frontend\DataProcessing\FilesProcessor``. + +All further options are passed to the configured data processor. Therefore they are documented at +each data processor. + +Example:: + + plugin.tx_searchcore.settings.searching.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor + 1 { + _table = pages + _dataProcessor = TYPO3\CMS\Frontend\DataProcessing\FilesProcessor + + references.fieldName = media + as = images + } + } + +The above example will create a new field ``images`` with resolved FAL relations from ``media`` +field. diff --git a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst index 0d9385b..9f31736 100644 --- a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst +++ b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst @@ -4,9 +4,10 @@ The following Processor are available: :maxdepth: 1 :glob: + /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor /configuration/dataProcessing/CopyToProcessor - /configuration/dataProcessing/RemoveProcessor /configuration/dataProcessing/GeoPointProcessor + /configuration/dataProcessing/RemoveProcessor The following Processor are planned: diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php new file mode 100644 index 0000000..871b205 --- /dev/null +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -0,0 +1,55 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor; +use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; +use TYPO3\CMS\Extbase\Service\TypoScriptService; +use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor; + +class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase +{ + /** + * @test + */ + public function contentObjectDataProcessorIsExecuted() + { + $record = ['content' => 'value1, value2']; + $configuration = [ + '_dataProcessor' => SplitProcessor::class, + 'delimiter' => ',', + 'fieldName' => 'content', + 'as' => 'new_content', + ]; + $expectedData = [ + 'content' => 'value1, value2', + 'new_content' => ['value1', 'value2'], + ]; + + $subject = new ContentObjectDataProcessorAdapterProcessor(new TypoScriptService); + $processedData = $subject->processData($record, $configuration); + $this->assertSame( + $expectedData, + $processedData, + 'The processor did not return the expected processed record.' + ); + } +} From 5d1e7c41bc6bfdddea1a2818c6df3e8b6e373a7a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 11:36:05 +0100 Subject: [PATCH 121/158] !!!|FEATURE: Pass facet configuration to search service Do not limit integrator in possibilities to configure. Therefore previously configure facets for a field need to be adjusted to contain full configuration for elasticsearch. See changelog. Resolves: #120 --- Classes/Connection/Elasticsearch/Facet.php | 23 +++--- Classes/Connection/FacetRequestInterface.php | 10 +-- Classes/Domain/Model/FacetRequest.php | 24 ++---- Classes/Domain/Search/QueryFactory.php | 6 +- Classes/Domain/Search/SearchService.php | 7 +- Documentation/source/changelog.rst | 8 ++ .../20180406-120-facet-configuration.rst | 40 ++++++++++ Documentation/source/index.rst | 1 + .../Connection/Elasticsearch/FacetTest.php | 75 +++++++++++++++++++ .../Connection/Elasticsearch/FilterTest.php | 33 -------- Tests/Functional/Fixtures/BasicSetup.ts | 6 -- Tests/Functional/Fixtures/Searching/Facet.ts | 17 +++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 4 +- 13 files changed, 167 insertions(+), 87 deletions(-) create mode 100644 Documentation/source/changelog.rst create mode 100644 Documentation/source/changelog/20180406-120-facet-configuration.rst create mode 100644 Tests/Functional/Connection/Elasticsearch/FacetTest.php create mode 100644 Tests/Functional/Fixtures/Searching/Facet.ts diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index e24a659..1142d88 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -45,25 +45,26 @@ class Facet implements FacetInterface */ protected $options = []; - public function __construct($name, array $aggregation, ConfigurationContainerInterface $configuration) + public function __construct(string $name, array $aggregation, ConfigurationContainerInterface $configuration) { $this->name = $name; $this->buckets = $aggregation['buckets']; - $this->field = $configuration->getIfExists('searching.facets.' . $this->name . '.field') ?: ''; + + $config = $configuration->getIfExists('searching.facets.' . $this->name) ?: []; + foreach ($config as $configEntry) { + if (isset($configEntry['field'])) { + $this->field = $configEntry['field']; + break; + } + } } - /** - * @return string - */ - public function getName() + public function getName() : string { return $this->name; } - /** - * @return string - */ - public function getField() + public function getField() : string { return $this->field; } @@ -73,7 +74,7 @@ class Facet implements FacetInterface * * @return array */ - public function getOptions() + public function getOptions() : array { $this->initOptions(); diff --git a/Classes/Connection/FacetRequestInterface.php b/Classes/Connection/FacetRequestInterface.php index 9352f96..cdc16b1 100644 --- a/Classes/Connection/FacetRequestInterface.php +++ b/Classes/Connection/FacetRequestInterface.php @@ -28,15 +28,11 @@ interface FacetRequestInterface /** * The identifier of the facet, used as key in arrays and to get the facet * from search request, etc. - * - * @return string */ - public function getIdentifier(); + public function getIdentifier() : string; /** - * The field to use for facet building. - * - * @return string + * The config to use for facet building. */ - public function getField(); + public function getConfig() : array; } diff --git a/Classes/Domain/Model/FacetRequest.php b/Classes/Domain/Model/FacetRequest.php index b1dbc4c..0d82ad3 100644 --- a/Classes/Domain/Model/FacetRequest.php +++ b/Classes/Domain/Model/FacetRequest.php @@ -30,37 +30,27 @@ class FacetRequest implements FacetRequestInterface protected $identifier = ''; /** - * @var string + * @var array */ - protected $field = ''; + protected $config = []; /** - * TODO: Add validation / exception? * As the facets come from configuration this might be a good idea to help * integrators find issues. - * - * @param string $identifier - * @param string $field */ - public function __construct($identifier, $field) + public function __construct(string $identifier, array $config) { $this->identifier = $identifier; - $this->field = $field; + $this->config = $config; } - /** - * @return string - */ - public function getIdentifier() + public function getIdentifier() : string { return $this->identifier; } - /** - * @return string - */ - public function getField() + public function getConfig() : array { - return $this->field; + return $this->config; } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 980b763..8e122ab 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -239,11 +239,7 @@ class QueryFactory foreach ($searchRequest->getFacets() as $facet) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'aggs' => [ - $facet->getIdentifier() => [ - 'terms' => [ - 'field' => $facet->getField(), - ], - ], + $facet->getIdentifier() => $facet->getConfig(), ], ]); } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 120ab3b..40c99bd 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -103,15 +103,10 @@ class SearchService } foreach ($facetsConfig as $identifier => $facetConfig) { - if (!isset($facetConfig['field']) || trim($facetConfig['field']) === '') { - // TODO: Finish throw - throw new \Exception('message', 1499171142); - } - $searchRequest->addFacet($this->objectManager->get( FacetRequest::class, $identifier, - $facetConfig['field'] + $facetConfig )); } } diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst new file mode 100644 index 0000000..0d7f020 --- /dev/null +++ b/Documentation/source/changelog.rst @@ -0,0 +1,8 @@ +Changelog +========= + +.. toctree:: + :maxdepth: 1 + :glob: + + changelog/* diff --git a/Documentation/source/changelog/20180406-120-facet-configuration.rst b/Documentation/source/changelog/20180406-120-facet-configuration.rst new file mode 100644 index 0000000..53021d9 --- /dev/null +++ b/Documentation/source/changelog/20180406-120-facet-configuration.rst @@ -0,0 +1,40 @@ +Breacking Change 120 "Pass facets configuration to elasticsearch" +================================================================= + +In order to allow arbitrary facet configuration, we do not process the facet configuration anymore. +Instead integrators are able to configure facets for search service "as is". We just pipe the +configuration through. + +Therefore the following, which worked before, does not work anymore: + +.. code-block:: typoscript + :linenos: + :emphasize-lines: 4 + + plugin.tx_searchcore.settings.search { + facets { + category { + field = categories + } + } + } + +Instead you have to provide the full configuration yourself: + +.. code-block:: typoscript + :linenos: + :emphasize-lines: 4,6 + + plugin.tx_searchcore.settings.search { + facets { + category { + terms { + field = categories + } + } + } + } + +You need to add line 4 and 6, the additional level ``terms`` for elasticsearch. + +See :issue:`120`. diff --git a/Documentation/source/index.rst b/Documentation/source/index.rst index 5427fc9..2edfae3 100644 --- a/Documentation/source/index.rst +++ b/Documentation/source/index.rst @@ -15,3 +15,4 @@ Table of Contents connections indexer development + changelog diff --git a/Tests/Functional/Connection/Elasticsearch/FacetTest.php b/Tests/Functional/Connection/Elasticsearch/FacetTest.php new file mode 100644 index 0000000..c89bc10 --- /dev/null +++ b/Tests/Functional/Connection/Elasticsearch/FacetTest.php @@ -0,0 +1,75 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Domain\Search\SearchService; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class FacetTest extends AbstractFunctionalTestCase +{ + protected function getTypoScriptFilesForFrontendRootPage() + { + return array_merge(parent::getTypoScriptFilesForFrontendRootPage(), ['EXT:search_core/Tests/Functional/Fixtures/Searching/Facet.ts']); + } + + protected function getDataSets() + { + return array_merge( + parent::getDataSets(), + ['Tests/Functional/Fixtures/Searching/Filter.xml'] + ); + } + + /** + * @test + */ + public function itsPossibleToFetchFacetsForField() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(SearchService::class); + + $searchRequest = new SearchRequest(); + $result = $searchService->search($searchRequest); + + $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); + + $facet = current($result->getFacets()); + $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.'); + $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.'); + + $options = $facet->getOptions(); + $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.'); + $option = $options['HTML']; + $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.'); + $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); + $option = $options['Header']; + $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.'); + $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); + } +} diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index e364a4a..8bbe0f0 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -58,37 +58,4 @@ class FilterTest extends AbstractFunctionalTestCase $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } - - /** - * @test - */ - public function itsPossibleToFetchFacetsForField() - { - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) - ->get(IndexerFactory::class) - ->getIndexer('tt_content') - ->indexAllDocuments() - ; - - $searchService = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) - ->get(SearchService::class); - - $searchRequest = new SearchRequest('Search Word'); - $result = $searchService->search($searchRequest); - - $this->assertSame(1, count($result->getFacets()), 'Did not receive the single defined facet.'); - - $facet = current($result->getFacets()); - $this->assertSame('contentTypes', $facet->getName(), 'Name of facet was not as expected.'); - $this->assertSame('CType', $facet->getField(), 'Field of facet was not expected.'); - - $options = $facet->getOptions(); - $this->assertSame(2, count($options), 'Did not receive the expected number of possible options for facet.'); - $option = $options['HTML']; - $this->assertSame('HTML', $option->getName(), 'Option did not have expected Name.'); - $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); - $option = $options['Header']; - $this->assertSame('Header', $option->getName(), 'Option did not have expected Name.'); - $this->assertSame(1, $option->getCount(), 'Option did not have expected count.'); - } } diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index b7b0c6a..4129362 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -37,12 +37,6 @@ plugin { } searching { - facets { - contentTypes { - field = CType - } - } - fields { query = _all } diff --git a/Tests/Functional/Fixtures/Searching/Facet.ts b/Tests/Functional/Fixtures/Searching/Facet.ts new file mode 100644 index 0000000..10eeae6 --- /dev/null +++ b/Tests/Functional/Fixtures/Searching/Facet.ts @@ -0,0 +1,17 @@ +plugin { + tx_searchcore { + settings { + searching { + facets { + contentTypes { + terms { + field = CType + } + } + } + } + } + } +} + +module.tx_searchcore < plugin.tx_searchcore diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index e0fc86d..881cd8b 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -118,8 +118,8 @@ class QueryFactoryTest extends AbstractUnitTestCase { $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); - $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); - $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); + $searchRequest->addFacet(new FacetRequest('Identifier', ['terms' => ['field' => 'FieldName']])); + $searchRequest->addFacet(new FacetRequest('Identifier 2', ['terms' => ['field' => 'FieldName 2']])); $query = $this->subject->create($searchRequest); $this->assertSame( From a893303939178090c03f60ca6aed61673962e7e8 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 13:04:42 +0100 Subject: [PATCH 122/158] TASK: Improve ResultItemTest Do not repeat content, use variable which is also better to read. And do not add unnecessary, unused, variables. --- Tests/Unit/Domain/Model/ResultItemTest.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Tests/Unit/Domain/Model/ResultItemTest.php b/Tests/Unit/Domain/Model/ResultItemTest.php index df11a5d..53477c7 100644 --- a/Tests/Unit/Domain/Model/ResultItemTest.php +++ b/Tests/Unit/Domain/Model/ResultItemTest.php @@ -57,7 +57,7 @@ class ResultItemTest extends AbstractUnitTestCase $subject = new ResultItem($originalData); $this->assertSame( - 'Some title', + $originalData['title'], $subject['title'], 'Could not retrieve title in array notation.' ); @@ -72,7 +72,6 @@ class ResultItemTest extends AbstractUnitTestCase 'uid' => 10, 'title' => 'Some title', ]; - $expectedData = $originalData; $subject = new ResultItem($originalData); $this->assertTrue(isset($subject['title']), 'Could not determine that title exists.'); @@ -88,7 +87,6 @@ class ResultItemTest extends AbstractUnitTestCase 'uid' => 10, 'title' => 'Some title', ]; - $expectedData = $originalData; $subject = new ResultItem($originalData); $this->expectException(\BadMethodCallException::class); @@ -104,7 +102,6 @@ class ResultItemTest extends AbstractUnitTestCase 'uid' => 10, 'title' => 'Some title', ]; - $expectedData = $originalData; $subject = new ResultItem($originalData); $this->expectException(\BadMethodCallException::class); From 6456f315030fd6a60f007816ac0155328d92db4a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 13:23:08 +0100 Subject: [PATCH 123/158] TASK: Make test more readable Make sure everyone knows what we compare, do not add hardcoded information. --- Tests/Unit/Domain/Model/SearchResultTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/Unit/Domain/Model/SearchResultTest.php b/Tests/Unit/Domain/Model/SearchResultTest.php index 910e06e..eebb3c1 100644 --- a/Tests/Unit/Domain/Model/SearchResultTest.php +++ b/Tests/Unit/Domain/Model/SearchResultTest.php @@ -89,9 +89,9 @@ class SearchResultTest extends AbstractUnitTestCase $this->assertCount(3, $resultItems); - $this->assertSame(10, $resultItems[0]['uid']); - $this->assertSame(11, $resultItems[1]['uid']); - $this->assertSame(12, $resultItems[2]['uid']); + $this->assertSame($data[0]['uid'], $resultItems[0]['uid']); + $this->assertSame($data[1]['uid'], $resultItems[1]['uid']); + $this->assertSame($data[2]['uid'], $resultItems[2]['uid']); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[0]); $this->assertInstanceOf(ResultItemInterface::class, $resultItems[1]); From 3731bcf4743821a6906078a9f546848294f56b00 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 16:39:07 +0100 Subject: [PATCH 124/158] TASK: Fix CGL --- Classes/Domain/Search/SearchService.php | 3 +-- Tests/Unit/Domain/Search/SearchServiceTest.php | 12 ++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index a331ab4..03fc11b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -168,8 +168,7 @@ class SearchService $searchResult, $newSearchResultItems ); - } - catch (InvalidArgumentException $e) { + } catch (InvalidArgumentException $e) { return $searchResult; } } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 6263f23..3d6791e 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -270,7 +270,11 @@ class SearchServiceTest extends AbstractUnitTestCase $this->dataProcessorService->expects($this->never())->method('executeDataProcessor'); $searchRequest = new SearchRequest(''); - $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result without applied data processing'); + $this->assertSame( + $searchResultMock, + $this->subject->search($searchRequest), + 'Did not get created result without applied data processing' + ); } /** @@ -312,6 +316,10 @@ class SearchServiceTest extends AbstractUnitTestCase ->willReturn($searchResultMock); $searchRequest = new SearchRequest(''); - $this->assertSame($searchResultMock, $this->subject->search($searchRequest), 'Did not get created result with applied data processing'); + $this->assertSame( + $searchResultMock, + $this->subject->search($searchRequest), + 'Did not get created result with applied data processing' + ); } } From 6544ec07d315a6d3b1624d9a5add948d078e14cb Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 15:56:14 +0100 Subject: [PATCH 125/158] FEATURE: Support display name for facet option As some search services, like elasticsearch, allow generation of a string that should be displayed in frontend, we provide a new getter for that. The old existing name can be a fallback in custom implementations. --- .../Connection/Elasticsearch/FacetOption.php | 21 +++--- Classes/Connection/FacetOptionInterface.php | 14 ++-- .../Elasticsearch/FacetOptionTest.php | 64 +++++++++++++++++++ 3 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php index 7359434..6b544bb 100644 --- a/Classes/Connection/Elasticsearch/FacetOption.php +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -29,6 +29,11 @@ class FacetOption implements FacetOptionInterface */ protected $name = ''; + /** + * @var string + */ + protected $displayName = ''; + /** * @var int */ @@ -40,21 +45,21 @@ class FacetOption implements FacetOptionInterface public function __construct(array $bucket) { $this->name = $bucket['key']; + $this->displayName = isset($bucket['key_as_string']) ? $bucket['key_as_string'] : $this->getName(); $this->count = $bucket['doc_count']; } - /** - * @return string - */ - public function getName() + public function getName() : string { return $this->name; } - /** - * @return int - */ - public function getCount() + public function getDisplayName() : string + { + return $this->displayName; + } + + public function getCount() : int { return $this->count; } diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php index 51a1efd..bf998db 100644 --- a/Classes/Connection/FacetOptionInterface.php +++ b/Classes/Connection/FacetOptionInterface.php @@ -28,15 +28,17 @@ interface FacetOptionInterface /** * Returns the name of this option. Equivalent * to value used for filtering. - * - * @return string */ - public function getName(); + public function getName() : string; + + /** + * If a pre-rendered name is provided, this will be returned. + * Otherwise it's the same as getName(). + */ + public function getDisplayName() : string; /** * Returns the number of found results for this option. - * - * @return int */ - public function getCount(); + public function getCount() : int; } diff --git a/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php b/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php new file mode 100644 index 0000000..696762c --- /dev/null +++ b/Tests/Unit/Connection/Elasticsearch/FacetOptionTest.php @@ -0,0 +1,64 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\Elasticsearch\FacetOption; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class FacetOptionTest extends AbstractUnitTestCase +{ + /** + * @test + */ + public function displayNameIsReturnedAsExpected() + { + $bucket = [ + 'key' => 'Name', + 'key_as_string' => 'DisplayName', + 'doc_count' => 10, + ]; + $subject = new FacetOption($bucket); + + $this->assertSame( + $bucket['key_as_string'], + $subject->getDisplayName(), + 'Display name was not returned as expected.' + ); + } + + /** + * @test + */ + public function displayNameIsReturnedAsExpectedIfNotProvided() + { + $bucket = [ + 'key' => 'Name', + 'doc_count' => 10, + ]; + $subject = new FacetOption($bucket); + + $this->assertSame( + $bucket['key'], + $subject->getDisplayName(), + 'Display name was not returned as expected.' + ); + } +} From 88f301f22888f0b3ce411dcdc1b2af7252005296 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 16:34:59 +0100 Subject: [PATCH 126/158] FEATURE: Allow range queries for elasticsearch Allow "raw" configuration and support "range" type. Also prevent adding boosts if no search term was submitted which can be boosted. Resolves: #119 --- Classes/Domain/Search/QueryFactory.php | 16 ++++++ .../source/configuration/searching.rst | 47 ++++++++++++++--- Tests/Unit/Domain/Search/QueryFactoryTest.php | 52 +++++++++++++++++++ 3 files changed, 107 insertions(+), 8 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 978d88a..dcf6ce5 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -121,6 +121,10 @@ class QueryFactory return; } + if (trim($searchRequest->getSearchTerm()) === '') { + return; + } + $boostQueryParts = []; foreach ($fields as $fieldName => $boostValue) { @@ -238,6 +242,18 @@ class QueryFactory } } + if (isset($config['raw'])) { + $filter = array_merge($config['raw'], $filter); + } + + if ($config['type'] === 'range') { + return [ + 'range' => [ + $config['field'] => $filter, + ], + ]; + } + return [$config['field'] => $filter]; } diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index bfed51e..857aa74 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -119,20 +119,48 @@ E.g. you submit a filter in form of: .. code-block:: html - - - + + Due to TYPO3 7.x fluid limitations, we build this input ourself. + No longer necessary in 8 and above + + + This will create a ``distance`` filter with subproperties. To make this filter actually work, you can add the following TypoScript, which will be added to the filter:: mapping { filter { - distance { - field = geo_distance + month { + type = range + field = released + raw { + format = yyyy-MM + } + fields { - distance = distance - location = location + gte = from + lte = to } } } @@ -143,9 +171,12 @@ in elasticsearch. In above example they do match, but you can also use different On the left hand side is the elasticsearch field name, on the right side the one submitted as a filter. -The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +The ``field``, in above example ``released``, will be used as the elasticsearch field for filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. +Everything that is configured inside ``raw`` is passed, as is, to search service, e.g. +elasticsearch. + .. _fields: fields diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 881cd8b..dc824c5 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -86,6 +86,58 @@ class QueryFactoryTest extends AbstractUnitTestCase ); } + /** + * @test + */ + public function rangeFilterIsAddedToQuery() + { + $this->configureConfigurationMockWithDefault(); + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->will($this->returnCallback(function ($configName) { + if ($configName === 'searching.mapping.filter.month') { + return [ + 'type' => 'range', + 'field' => 'released', + 'raw' => [ + 'format' => 'yyyy-MM', + ], + 'fields' => [ + 'gte' => 'from', + 'lte' => 'to', + ], + ]; + } + + return []; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter([ + 'month' => [ + 'from' => '2016-03', + 'to' => '2017-11', + ], + ]); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + [ + 'range' => [ + 'released' => [ + 'format' => 'yyyy-MM', + 'gte' => '2016-03', + 'lte' => '2017-11', + ], + ], + ] + ], + $query->toArray()['query']['bool']['filter'], + 'Filter was not added to query.' + ); + } + /** * @test */ From 560597dcff89b4e7f9f5618af0005f1074f07c83 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 17:40:49 +0100 Subject: [PATCH 127/158] TASK: Cleanup type annotations and phpstan issues --- Classes/Command/IndexCommandController.php | 4 +-- .../Configuration/ConfigurationContainer.php | 5 ++- .../ConfigurationContainerInterface.php | 4 +-- Classes/Connection/ConnectionInterface.php | 30 ++++------------ Classes/Connection/Elasticsearch.php | 29 +++++---------- .../Connection/Elasticsearch/Connection.php | 10 +++--- .../Elasticsearch/DocumentFactory.php | 14 ++------ Classes/Connection/Elasticsearch/Facet.php | 1 + .../Connection/Elasticsearch/FacetOption.php | 10 ++---- .../Connection/Elasticsearch/IndexFactory.php | 21 ++--------- .../Elasticsearch/MappingFactory.php | 12 ++----- .../Connection/Elasticsearch/SearchResult.php | 6 ++-- .../Connection/Elasticsearch/TypeFactory.php | 7 +--- Classes/Connection/FacetInterface.php | 7 ++-- Classes/Connection/FacetOptionInterface.php | 8 ++--- Classes/Connection/SearchRequestInterface.php | 23 ++++++++---- Classes/Connection/SearchResultInterface.php | 8 ++--- Classes/Domain/Index/AbstractIndexer.php | 34 +++++------------- Classes/Domain/Index/IndexerFactory.php | 11 ++---- Classes/Domain/Index/IndexerInterface.php | 8 ++--- Classes/Domain/Index/TcaIndexer.php | 23 +++++------- .../Domain/Index/TcaIndexer/PagesIndexer.php | 15 ++------ .../Index/TcaIndexer/TcaTableService.php | 31 +++------------- Classes/Domain/Model/SearchRequest.php | 35 ++++-------------- Classes/Domain/Model/SearchResult.php | 6 ++-- Classes/Domain/Search/SearchService.php | 12 +------ Classes/Domain/Service/DataHandler.php | 23 +++--------- Classes/Hook/DataHandler.php | 36 ++++++------------- .../Form/Finisher/DataHandlerFinisher.php | 2 +- Classes/Utility/FrontendUtility.php | 5 +-- 30 files changed, 117 insertions(+), 323 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 70816ed..61aad53 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -48,7 +48,7 @@ class IndexCommandController extends CommandController * * @param string $identifier */ - public function indexCommand($identifier) + public function indexCommand(string $identifier) { try { $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); @@ -63,7 +63,7 @@ class IndexCommandController extends CommandController * * @param string $identifier */ - public function deleteCommand($identifier) + public function deleteCommand(string $identifier) { try { $this->indexerFactory->getIndexer($identifier)->delete(); diff --git a/Classes/Configuration/ConfigurationContainer.php b/Classes/Configuration/ConfigurationContainer.php index 7481692..eab76e0 100644 --- a/Classes/Configuration/ConfigurationContainer.php +++ b/Classes/Configuration/ConfigurationContainer.php @@ -39,7 +39,6 @@ class ConfigurationContainer implements ConfigurationContainerInterface /** * Inject settings via ConfigurationManager. * - * @param ConfigurationManagerInterface $configurationManager * @throws NoConfigurationException */ public function injectConfigurationManager(ConfigurationManagerInterface $configurationManager) @@ -59,7 +58,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface * @return mixed * @throws InvalidArgumentException */ - public function get($path) + public function get(string $path) { $value = ArrayUtility::getValueByPath($this->settings, $path); @@ -77,7 +76,7 @@ class ConfigurationContainer implements ConfigurationContainerInterface * @param string $path In dot notation. * @return mixed */ - public function getIfExists($path) + public function getIfExists(string $path) { return ArrayUtility::getValueByPath($this->settings, $path); } diff --git a/Classes/Configuration/ConfigurationContainerInterface.php b/Classes/Configuration/ConfigurationContainerInterface.php index 9429535..1978453 100644 --- a/Classes/Configuration/ConfigurationContainerInterface.php +++ b/Classes/Configuration/ConfigurationContainerInterface.php @@ -37,7 +37,7 @@ interface ConfigurationContainerInterface extends Singleton * * @throws InvalidArgumentException */ - public function get($path); + public function get(string $path); /** * Same as get but will not throw an exception but return null. @@ -45,5 +45,5 @@ interface ConfigurationContainerInterface extends Singleton * @param string $path In dot notation. * @return mixed|null */ - public function getIfExists($path); + public function getIfExists(string $path); } diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index 59cf9f8..a336793 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -28,62 +28,44 @@ interface ConnectionInterface /** * Will add a new document. * - * @param string $documentType - * @param array $document - * * @return void */ - public function addDocument($documentType, array $document); + public function addDocument(string $documentType, array $document); /** * Add the given documents. * - * @param string $documentType - * @param array $documents - * * @return void */ - public function addDocuments($documentType, array $documents); + public function addDocuments(string $documentType, array $documents); /** * Will update an existing document. * * NOTE: Batch updating is not yet supported. * - * @param string $documentType - * @param array $document - * * @return void */ - public function updateDocument($documentType, array $document); + public function updateDocument(string $documentType, array $document); /** * Will remove an existing document. * * NOTE: Batch deleting is not yet supported. * - * @param string $documentType - * @param int $identifier - * * @return void */ - public function deleteDocument($documentType, $identifier); + public function deleteDocument(string $documentType, string $identifier); /** * Search by given request and return result. - * - * @param SearchRequestInterface $searchRequest - * - * @return SearchResultInterface */ - public function search(SearchRequestInterface $searchRequest); + public function search(SearchRequestInterface $searchRequest) : SearchResultInterface; /** * Will delete the whole index / db. * - * @param string $documentType - * * @return void */ - public function deleteIndex($documentType); + public function deleteIndex(string $documentType); } diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index b86bebf..555d4e7 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -112,7 +112,7 @@ class Elasticsearch implements Singleton, ConnectionInterface $this->queryFactory = $queryFactory; } - public function addDocument($documentType, array $document) + public function addDocument(string $documentType, array $document) { $this->withType( $documentType, @@ -122,7 +122,7 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } - public function deleteDocument($documentType, $identifier) + public function deleteDocument(string $documentType, string $identifier) { try { $this->withType( @@ -139,7 +139,7 @@ class Elasticsearch implements Singleton, ConnectionInterface } } - public function updateDocument($documentType, array $document) + public function updateDocument(string $documentType, array $document) { $this->withType( $documentType, @@ -149,7 +149,7 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } - public function addDocuments($documentType, array $documents) + public function addDocuments(string $documentType, array $documents) { $this->withType( $documentType, @@ -159,7 +159,7 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } - public function deleteIndex($documentType) + public function deleteIndex(string $documentType) { $index = $this->connection->getClient()->getIndex('typo3content'); @@ -173,11 +173,8 @@ class Elasticsearch implements Singleton, ConnectionInterface /** * Execute given callback with Elastica Type based on provided documentType - * - * @param string $documentType - * @param callable $callback */ - protected function withType($documentType, callable $callback) + protected function withType(string $documentType, callable $callback) { $type = $this->getType($documentType); // TODO: Check whether it's to heavy to send it so often e.g. for every single document. @@ -191,12 +188,7 @@ class Elasticsearch implements Singleton, ConnectionInterface $type->getIndex()->refresh(); } - /** - * @param SearchRequestInterface $searchRequest - * - * @return SearchResultInterface - */ - public function search(SearchRequestInterface $searchRequest) + public function search(SearchRequestInterface $searchRequest) : SearchResultInterface { $this->logger->debug('Search for', [$searchRequest->getSearchTerm()]); @@ -207,12 +199,7 @@ class Elasticsearch implements Singleton, ConnectionInterface return $this->objectManager->get(SearchResult::class, $searchRequest, $search->search()); } - /** - * @param string $documentType - * - * @return \Elastica\Type - */ - protected function getType($documentType) + protected function getType(string $documentType) : \Elastica\Type { return $this->typeFactory->getType( $this->indexFactory->getIndex( diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index cd0a1fd..10a62d3 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -52,9 +52,8 @@ class Connection implements Singleton ) { $this->configuration = $configuration; - $this->elasticaClient = $elasticaClient; - if ($this->elasticaClient === null) { - $this->elasticaClient = new \Elastica\Client([ + if ($elasticaClient === null) { + $elasticaClient = new \Elastica\Client([ 'host' => $this->configuration->get('connections.elasticsearch.host'), 'port' => $this->configuration->get('connections.elasticsearch.port'), // TODO: Make configurable @@ -63,14 +62,13 @@ class Connection implements Singleton // TODO: Make configurable. // new \Elastica\Log($this->elasticaClient); } + $this->elasticaClient = $elasticaClient; } /** * Get the concrete client for internal usage! - * - * @return \Elastica\Client */ - public function getClient() + public function getClient() : \ Elastica\Client { return $this->elasticaClient; } diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 390c592..beb091a 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -44,13 +44,8 @@ class DocumentFactory implements Singleton /** * Creates document from document. - * - * @param string $documentType - * @param array $document - * - * @return \Elastica\Document */ - public function getDocument($documentType, array $document) + public function getDocument(string $documentType, array $document) : \Elastica\Document { // TODO: Use DocumentType for further configuration. @@ -70,13 +65,8 @@ class DocumentFactory implements Singleton /** * Creates documents based on documents. - * - * @param string $documentType - * @param array $documents - * - * @return array */ - public function getDocuments($documentType, array $documents) + public function getDocuments(string $documentType, array $documents) : array { foreach ($documents as &$document) { $document = $this->getDocument($documentType, $document); diff --git a/Classes/Connection/Elasticsearch/Facet.php b/Classes/Connection/Elasticsearch/Facet.php index 1142d88..27de076 100644 --- a/Classes/Connection/Elasticsearch/Facet.php +++ b/Classes/Connection/Elasticsearch/Facet.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\FacetInterface; +use Codappix\SearchCore\Connection\FacetOptionInterface; class Facet implements FacetInterface { diff --git a/Classes/Connection/Elasticsearch/FacetOption.php b/Classes/Connection/Elasticsearch/FacetOption.php index 7359434..300a505 100644 --- a/Classes/Connection/Elasticsearch/FacetOption.php +++ b/Classes/Connection/Elasticsearch/FacetOption.php @@ -43,18 +43,12 @@ class FacetOption implements FacetOptionInterface $this->count = $bucket['doc_count']; } - /** - * @return string - */ - public function getName() + public function getName() : string { return $this->name; } - /** - * @return int - */ - public function getCount() + public function getCount() : int { return $this->count; } diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 66b448b..6ddab67 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -49,13 +49,8 @@ class IndexFactory implements Singleton /** * Get an index bases on TYPO3 table name. - * - * @param Connection $connection - * @param string $documentType - * - * @return \Elastica\Index */ - public function getIndex(Connection $connection, $documentType) + public function getIndex(Connection $connection, string $documentType) : \Elastica\Index { $index = $connection->getClient()->getIndex('typo3content'); @@ -66,12 +61,7 @@ class IndexFactory implements Singleton return $index; } - /** - * @param string $documentType - * - * @return array - */ - protected function getConfigurationFor($documentType) + protected function getConfigurationFor(string $documentType) : array { try { $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); @@ -88,12 +78,7 @@ class IndexFactory implements Singleton } } - /** - * @param array $analyzer - * - * @return array - */ - protected function prepareAnalyzerConfiguration(array $analyzer) + protected function prepareAnalyzerConfiguration(array $analyzer) : array { $fieldsToExplode = ['char_filter', 'filter']; diff --git a/Classes/Connection/Elasticsearch/MappingFactory.php b/Classes/Connection/Elasticsearch/MappingFactory.php index 44a05c1..3882556 100644 --- a/Classes/Connection/Elasticsearch/MappingFactory.php +++ b/Classes/Connection/Elasticsearch/MappingFactory.php @@ -44,12 +44,8 @@ class MappingFactory implements Singleton /** * Get an mapping based on type. - * - * @param \Elastica\Type $type - * - * @return \Elastica\Mapping */ - public function getMapping(\Elastica\Type $type) + public function getMapping(\Elastica\Type $type) : \Elastica\Type\Mapping { $mapping = new \Elastica\Type\Mapping(); $mapping->setType($type); @@ -64,11 +60,7 @@ class MappingFactory implements Singleton return $mapping; } - /** - * @param string $identifier - * @return array - */ - protected function getConfiguration($identifier) + protected function getConfiguration(string $identifier) : array { try { return $this->configuration->get('indexing.' . $identifier . '.mapping'); diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index 867d823..5b3d381 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -77,7 +77,7 @@ class SearchResult implements SearchResultInterface /** * @return array */ - public function getResults() + public function getResults() : array { $this->initResults(); @@ -89,14 +89,14 @@ class SearchResult implements SearchResultInterface * * @return array */ - public function getFacets() + public function getFacets() : array { $this->initFacets(); return $this->facets; } - public function getCurrentCount() + public function getCurrentCount() : int { return $this->result->count(); } diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index d5283f8..de8e336 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -32,13 +32,8 @@ class TypeFactory implements Singleton { /** * Get an index bases on TYPO3 table name. - * - * @param \Elastica\Index $index - * @param string $documentType - * - * @return \Elastica\Type */ - public function getType(\Elastica\Index $index, $documentType) + public function getType(\Elastica\Index $index, string $documentType) : \Elastica\Type { return $index->getType($documentType); } diff --git a/Classes/Connection/FacetInterface.php b/Classes/Connection/FacetInterface.php index b1cc421..3ec549d 100644 --- a/Classes/Connection/FacetInterface.php +++ b/Classes/Connection/FacetInterface.php @@ -25,15 +25,12 @@ namespace Codappix\SearchCore\Connection; */ interface FacetInterface { - /** - * @return string - */ - public function getName(); + public function getName() : string; /** * Returns all possible options for this facet. * * @return array */ - public function getOptions(); + public function getOptions() : array; } diff --git a/Classes/Connection/FacetOptionInterface.php b/Classes/Connection/FacetOptionInterface.php index 51a1efd..cb1c529 100644 --- a/Classes/Connection/FacetOptionInterface.php +++ b/Classes/Connection/FacetOptionInterface.php @@ -28,15 +28,11 @@ interface FacetOptionInterface /** * Returns the name of this option. Equivalent * to value used for filtering. - * - * @return string */ - public function getName(); + public function getName() : string; /** * Returns the number of found results for this option. - * - * @return int */ - public function getCount(); + public function getCount() : int; } diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index e24adc3..051f32d 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -20,6 +20,8 @@ namespace Codappix\SearchCore\Connection; * 02110-1301, USA. */ +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Connection\FacetRequestInterface; use Codappix\SearchCore\Domain\Search\SearchService; use TYPO3\CMS\Extbase\Persistence\QueryInterface; @@ -27,20 +29,27 @@ interface SearchRequestInterface extends QueryInterface { /** * Returns the actual string the user searched for. - * - * @return string */ - public function getSearchTerm(); + public function getSearchTerm() : string; + + public function hasFilter() : bool; + + public function getFilter() : array; + + public function setFilter(array $filter); + + public function addFacet(FacetRequestInterface $facet); /** - * @return bool + * @return array */ - public function hasFilter(); + public function getFacets() : array; /** - * @return array + * Workaround for paginate widget support which will + * use the request to build another search. */ - public function getFilter(); + public function setConnection(ConnectionInterface $connection); /** * Workaround for paginate widget support which will diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index 60718cd..be698fa 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -30,19 +30,17 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf /** * @return array */ - public function getResults(); + public function getResults() : array; /** * Return all facets, if any. * * @return array */ - public function getFacets(); + public function getFacets() : array; /** * Returns the number of results in current result - * - * @return int */ - public function getCurrentCount(); + public function getCurrentCount() : int; } diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 2de7dd5..5687cb1 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -64,7 +64,7 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger = $logManager->getLogger(__CLASS__); } - public function setIdentifier($identifier) + public function setIdentifier(string $identifier) { $this->identifier = $identifier; } @@ -97,11 +97,11 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger->info('Finish indexing'); } - public function indexDocument($identifier) + public function indexDocument(string $identifier) { $this->logger->info('Start indexing single record.', [$identifier]); try { - $record = $this->getRecord($identifier); + $record = $this->getRecord((int) $identifier); $this->prepareRecord($record); $this->connection->addDocument($this->getDocumentName(), $record); @@ -119,10 +119,7 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger->info('Finish deletion.'); } - /** - * @return \Generator - */ - protected function getRecordGenerator() + protected function getRecordGenerator() : \Generator { $offset = 0; $limit = $this->getLimit(); @@ -133,9 +130,6 @@ abstract class AbstractIndexer implements IndexerInterface } } - /** - * @param array &$record - */ protected function prepareRecord(array &$record) { try { @@ -149,9 +143,6 @@ abstract class AbstractIndexer implements IndexerInterface $this->handleAbstract($record); } - /** - * @param array &$record - */ protected function handleAbstract(array &$record) { $record['search_abstract'] = ''; @@ -177,31 +168,22 @@ abstract class AbstractIndexer implements IndexerInterface /** * Returns the limit to use to fetch records. - * - * @return int */ - protected function getLimit() + protected function getLimit() : int { // TODO: Make configurable. return 50; } /** - * @param int $offset - * @param int $limit * @return array|null */ - abstract protected function getRecords($offset, $limit); + abstract protected function getRecords(int $offset, int $limit); /** - * @param int $identifier - * @return array * @throws NoRecordFoundException If record could not be found. */ - abstract protected function getRecord($identifier); + abstract protected function getRecord(int $identifier) : array; - /** - * @return string - */ - abstract protected function getDocumentName(); + abstract protected function getDocumentName() : string; } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index dbae818..6efe035 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -56,12 +56,9 @@ class IndexerFactory implements Singleton } /** - * @param string $identifier - * - * @return IndexerInterface * @throws NoMatchingIndexer */ - public function getIndexer($identifier) + public function getIndexer(string $identifier) : IndexerInterface { try { return $this->buildIndexer($this->configuration->get('indexing.' . $identifier . '.indexer'), $identifier); @@ -75,13 +72,9 @@ class IndexerFactory implements Singleton } /** - * @param string $indexerClass - * @param string $identifier - * - * @return IndexerInterface * @throws NoMatchingIndexer */ - protected function buildIndexer($indexerClass, $identifier) + protected function buildIndexer(string $indexerClass, string $identifier) : IndexerInterface { $indexer = null; if (is_subclass_of($indexerClass, TcaIndexer\PagesIndexer::class) diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 72ebb9d..4acfb28 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -35,20 +35,16 @@ interface IndexerInterface /** * Fetches a single document and pushes it to the connection. * - * @param string $identifier - * * @return void */ - public function indexDocument($identifier); + public function indexDocument(string $identifier); /** * Recieves the identifier of the indexer itself. * - * @param string $identifier - * * @return void */ - public function setIdentifier($identifier); + public function setIdentifier(string $identifier); /** * Delete the whole index. diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index c35b9ea..477bf18 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -46,17 +47,14 @@ class TcaIndexer extends AbstractIndexer ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { + parent::__construct($connection, $configuration); $this->tcaTableService = $tcaTableService; - $this->connection = $connection; - $this->configuration = $configuration; } /** - * @param int $offset - * @param int $limit * @return array|null */ - protected function getRecords($offset, $limit) + protected function getRecords(int $offset, int $limit) { $records = $this->getQuery() ->setFirstResult($offset) @@ -77,14 +75,12 @@ class TcaIndexer extends AbstractIndexer } /** - * @param int $identifier - * @return array * @throws NoRecordFoundException If record could not be found. */ - protected function getRecord($identifier) + protected function getRecord(int $identifier) : array { $query = $this->getQuery(); - $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . (int) $identifier); + $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . $identifier); $record = $query->execute()->fetch(); if ($record === false || $record === null) { @@ -98,15 +94,12 @@ class TcaIndexer extends AbstractIndexer return $record; } - /** - * @return string - */ - protected function getDocumentName() + protected function getDocumentName() : string { return $this->tcaTableService->getTableName(); } - protected function getQuery($tcaTableService = null) : QueryBuilder + protected function getQuery(TcaTableService $tcaTableService = null) : QueryBuilder { if ($tcaTableService === null) { $tcaTableService = $this->tcaTableService; @@ -126,7 +119,7 @@ class TcaIndexer extends AbstractIndexer return $query; } - protected function getDatabaseConnection() + protected function getDatabaseConnection() : ConnectionPool { return GeneralUtility::makeInstance(ConnectionPool::class); } diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index b396b73..4f6d03d 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -36,7 +36,7 @@ class PagesIndexer extends TcaIndexer /** * @param TcaTableService $tcaTableService - * @param TcaTableService $tcaTableService + * @param TcaTableService $contentTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ @@ -46,15 +46,10 @@ class PagesIndexer extends TcaIndexer ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { - $this->tcaTableService = $tcaTableService; + parent::__construct($tcaTableService, $connection, $configuration); $this->contentTableService = $contentTableService; - $this->connection = $connection; - $this->configuration = $configuration; } - /** - * @param array &$record - */ protected function prepareRecord(array &$record) { $possibleTitleFields = ['nav_title', 'tx_tqseo_pagetitle_rel', 'title']; @@ -69,11 +64,7 @@ class PagesIndexer extends TcaIndexer parent::prepareRecord($record); } - /** - * @param int $uid - * @return string - */ - protected function fetchContentForPage($uid) + protected function fetchContentForPage(int $uid) : string { $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index f39248f..61e30a9 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -107,17 +107,11 @@ class TcaTableService $this->relationResolver = $relationResolver; } - /** - * @return string - */ public function getTableName() : string { return $this->tableName; } - /** - * @return string - */ public function getTableClause() : string { return $this->tableName; @@ -125,9 +119,6 @@ class TcaTableService /** * Filter the given records by root line blacklist settings. - * - * @param array &$records - * @return void */ public function filterRecordsByRootLineBlacklist(array &$records) { @@ -139,9 +130,6 @@ class TcaTableService ); } - /** - * @param array &$record - */ public function prepareRecord(array &$record) { $this->relationResolver->resolveRelationsForRecord($this, $record); @@ -166,8 +154,8 @@ class TcaTableService $whereClause .= ' AND ' . $userDefinedWhere; } - if ($this->isBlacklistedRootLineConfigured()) { - $parameters[':blacklistedRootLine'] = $this->getBlacklistedRootLine(); + if ($this->isBlackListedRootLineConfigured()) { + $parameters[':blacklistedRootLine'] = $this->getBlackListedRootLine(); $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' . ' AND pages.pid NOT IN (:blacklistedRootLine)'; } @@ -231,11 +219,7 @@ class TcaTableService return $whereClause; } - /** - * @param string - * @return bool - */ - protected function isSystemField($columnName) : bool + protected function isSystemField(string $columnName) : bool { $systemFields = [ // Versioning fields, @@ -267,11 +251,9 @@ class TcaTableService } /** - * @param string $columnName - * @return array * @throws InvalidArgumentException */ - public function getColumnConfig($columnName) : array + public function getColumnConfig(string $columnName) : array { if (!isset($this->tca['columns'][$columnName])) { throw new InvalidArgumentException( @@ -290,9 +272,6 @@ class TcaTableService * Also further TYPO3 mechanics are taken into account. Does a valid root * line exist, is page inside a recycler, is inherited start- endtime * excluded, etc. - * - * @param array &$record - * @return bool */ protected function isRecordBlacklistedByRootline(array &$record) : bool { @@ -348,8 +327,6 @@ class TcaTableService /** * Checks whether any page uids are black listed. - * - * @return bool */ protected function isBlackListedRootLineConfigured() : bool { diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 69237bb..2addff1 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -72,56 +72,39 @@ class SearchRequest implements SearchRequestInterface /** * @param string $query */ - public function __construct($query = '') + public function __construct(string $query = '') { - $this->query = (string) $query; + $this->query = $query; } - /** - * @return string - */ - public function getQuery() + public function getQuery() : string { return $this->query; } - /** - * @return string - */ - public function getSearchTerm() + public function getSearchTerm() : string { return $this->query; } - /** - * @param array $filter - */ public function setFilter(array $filter) { $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); } - /** - * @return bool - */ - public function hasFilter() + public function hasFilter() : bool { return count($this->filter) > 0; } - /** - * @return array - */ - public function getFilter() + public function getFilter() : array { return $this->filter; } /** * Add a facet to gather in this search request. - * - * @param FacetRequestInterface $facet */ public function addFacet(FacetRequestInterface $facet) { @@ -130,10 +113,8 @@ class SearchRequest implements SearchRequestInterface /** * Returns all configured facets to fetch in this search request. - * - * @return array */ - public function getFacets() + public function getFacets() : array { return $this->facets; } @@ -141,8 +122,6 @@ class SearchRequest implements SearchRequestInterface /** * Define connection to use for this request. * Necessary to allow implementation of execute for interface. - * - * @param ConnectionInterface $connection */ public function setConnection(ConnectionInterface $connection) { diff --git a/Classes/Domain/Model/SearchResult.php b/Classes/Domain/Model/SearchResult.php index d91820d..163b996 100644 --- a/Classes/Domain/Model/SearchResult.php +++ b/Classes/Domain/Model/SearchResult.php @@ -62,7 +62,7 @@ class SearchResult implements SearchResultInterface /** * @return array */ - public function getResults() + public function getResults() : array { $this->initResults(); @@ -80,12 +80,12 @@ class SearchResult implements SearchResultInterface } } - public function getFacets() + public function getFacets() : array { return $this->originalSearchResult->getFacets(); } - public function getCurrentCount() + public function getCurrentCount() : int { return $this->originalSearchResult->getCurrentCount(); } diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 03fc11b..3a83b7b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -74,11 +74,7 @@ class SearchService $this->dataProcessorService = $dataProcessorService; } - /** - * @param SearchRequestInterface $searchRequest - * @return SearchResultInterface - */ - public function search(SearchRequestInterface $searchRequest) + public function search(SearchRequestInterface $searchRequest) : SearchResultInterface { $this->addSize($searchRequest); $this->addConfiguredFacets($searchRequest); @@ -93,8 +89,6 @@ class SearchService /** * Add configured size of search result items to request. - * - * @param SearchRequestInterface $searchRequest */ protected function addSize(SearchRequestInterface $searchRequest) { @@ -105,8 +99,6 @@ class SearchService /** * Add facets from configuration to request. - * - * @param SearchRequestInterface $searchRequest */ protected function addConfiguredFacets(SearchRequestInterface $searchRequest) { @@ -126,8 +118,6 @@ class SearchService /** * Add filters from configuration, e.g. flexform or TypoScript. - * - * @param SearchRequestInterface $searchRequest */ protected function addConfiguredFilters(SearchRequestInterface $searchRequest) { diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 10a01fe..fd6b08d 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -22,6 +22,7 @@ namespace Codappix\SearchCore\Domain\Service; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Index\IndexerFactory; +use Codappix\SearchCore\Domain\Index\IndexerInterface; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; @@ -83,41 +84,27 @@ class DataHandler implements Singleton $this->indexerFactory = $indexerFactory; } - /** - * @param string $table - */ - public function update($table, array $record) + public function update(string $table, array $record) { $this->logger->debug('Record received for update.', [$table, $record]); $this->getIndexer($table)->indexDocument($record['uid']); } - /** - * @param string $table - * @param int $identifier - */ - public function delete($table, $identifier) + public function delete(string $table, string $identifier) { $this->logger->debug('Record received for delete.', [$table, $identifier]); $this->connection->deleteDocument($table, $identifier); } /** - * @param string $table - * @return IndexerInterface - * * @throws NoMatchingIndexerException */ - protected function getIndexer($table) + protected function getIndexer(string $table) : IndexerInterface { return $this->indexerFactory->getIndexer($table); } - /** - * @param string $table - * @return bool - */ - public function canHandle($table) + public function canHandle(string $table) : bool { try { $this->getIndexer($table); diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index fa4b09f..6277708 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -49,46 +49,38 @@ class DataHandler implements Singleton /** * Dependency injection as TYPO3 doesn't provide it on it's own. * Still you can submit your own dataHandler. - * - * @param OwnDataHandler $dataHandler - * @param Logger $logger */ public function __construct(OwnDataHandler $dataHandler = null, Logger $logger = null) { - $this->dataHandler = $dataHandler; - if ($this->dataHandler === null) { + if ($dataHandler === null) { try { - $this->dataHandler = GeneralUtility::makeInstance(ObjectManager::class) + $dataHandler = GeneralUtility::makeInstance(ObjectManager::class) ->get(OwnDataHandler::class); } catch (NoConfigurationException $e) { // We have no configuration. That's fine, hooks will not be // executed due to check for existing DataHandler. } } + $this->dataHandler = $dataHandler; - $this->logger = $logger; - if ($this->logger === null) { - $this->logger = GeneralUtility::makeInstance(LogManager::class) + if ($logger === null) { + $logger = GeneralUtility::makeInstance(LogManager::class) ->getLogger(__CLASS__); } + $this->logger = $logger; } /** * Called by CoreDataHandler on deletion of records. - * - * @param string $table - * @param int $uid - * - * @return bool False if hook was not processed. */ - public function processCmdmap_deleteAction($table, $uid) + public function processCmdmap_deleteAction(string $table, int $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { $this->logger->debug('Delete not processed.', [$table, $uid]); return false; } - $this->dataHandler->delete($table, $uid); + $this->dataHandler->delete($table, (string) $uid); return true; } @@ -125,11 +117,7 @@ class DataHandler implements Singleton return false; } - /** - * @param string $table - * @return bool - */ - protected function shouldProcessHookForTable($table) + protected function shouldProcessHookForTable(string $table) : bool { if ($this->dataHandler === null) { $this->logger->debug('Datahandler could not be setup.'); @@ -146,11 +134,9 @@ class DataHandler implements Singleton /** * Wrapper to allow unit testing. * - * @param string $table - * @param int $uid - * @return null|array + * @return array|null */ - protected function getRecord($table, $uid) + protected function getRecord(string $table, int $uid) { return BackendUtility::getRecord($table, $uid); } diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php index 696d8ad..6a90e01 100644 --- a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -62,7 +62,7 @@ class DataHandlerFinisher extends AbstractFinisher $this->dataHandler->update($tableName, $record); break; case 'delete': - $this->dataHandler->delete($tableName, $record['uid']); + $this->dataHandler->delete($tableName, (string) $record['uid']); break; } } diff --git a/Classes/Utility/FrontendUtility.php b/Classes/Utility/FrontendUtility.php index 1282421..ffdbb6d 100644 --- a/Classes/Utility/FrontendUtility.php +++ b/Classes/Utility/FrontendUtility.php @@ -29,10 +29,7 @@ use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; */ class FrontendUtility extends BackendUtility { - /** - * @return TypoScriptFrontendController - */ - protected static function getLanguageService() + protected static function getLanguageService() : TypoScriptFrontendController { return $GLOBALS['TSFE']; } From 8d930448e0d1b63c0c8bfd30c7b0b556907ae253 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 17:58:19 +0100 Subject: [PATCH 128/158] TASK: Fix scurtinizer issues --- Classes/Connection/SearchRequestInterface.php | 7 +++++++ Classes/DataProcessing/GeoPointProcessor.php | 4 ++-- Classes/Domain/Index/AbstractIndexer.php | 2 +- Classes/Domain/Service/DataHandler.php | 4 +--- Classes/Hook/DataHandler.php | 6 +++--- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Classes/Connection/SearchRequestInterface.php b/Classes/Connection/SearchRequestInterface.php index 051f32d..a400447 100644 --- a/Classes/Connection/SearchRequestInterface.php +++ b/Classes/Connection/SearchRequestInterface.php @@ -38,6 +38,9 @@ interface SearchRequestInterface extends QueryInterface public function setFilter(array $filter); + /** + * @return void + */ public function addFacet(FacetRequestInterface $facet); /** @@ -48,12 +51,16 @@ interface SearchRequestInterface extends QueryInterface /** * Workaround for paginate widget support which will * use the request to build another search. + * + * @return void */ public function setConnection(ConnectionInterface $connection); /** * Workaround for paginate widget support which will * use the request to build another search. + * + * @return void */ public function setSearchService(SearchService $searchService); } diff --git a/Classes/DataProcessing/GeoPointProcessor.php b/Classes/DataProcessing/GeoPointProcessor.php index c5b6d18..971e2c4 100644 --- a/Classes/DataProcessing/GeoPointProcessor.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -27,7 +27,7 @@ class GeoPointProcessor implements ProcessorInterface { public function processData(array $record, array $configuration) : array { - if (! $this->canApply($record, $configuration)) { + if (! $this->isApplyable($record, $configuration)) { return $record; } @@ -39,7 +39,7 @@ class GeoPointProcessor implements ProcessorInterface return $record; } - protected function canApply(array $record, array $configuration) : bool + protected function isApplyable(array $record, array $configuration) : bool { if (!isset($record[$configuration['lat']]) || !is_numeric($record[$configuration['lat']]) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 5687cb1..e8f032e 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -152,7 +152,7 @@ abstract class AbstractIndexer implements IndexerInterface ',', $this->configuration->get('indexing.' . $this->identifier . '.abstractFields') ); - if (!$fieldsToUse) { + if ($fieldsToUse === []) { return; } foreach ($fieldsToUse as $fieldToUse) { diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index fd6b08d..5b587e4 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -104,7 +104,7 @@ class DataHandler implements Singleton return $this->indexerFactory->getIndexer($table); } - public function canHandle(string $table) : bool + public function supportsTable(string $table) : bool { try { $this->getIndexer($table); @@ -112,7 +112,5 @@ class DataHandler implements Singleton } catch (NoMatchingIndexerException $e) { return false; } - - return false; } } diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 6277708..3bd698f 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -90,8 +90,8 @@ class DataHandler implements Singleton $uid = key($record); $fieldData = current($record); - if (isset($fieldArray['uid'])) { - $uid = $fieldArray['uid']; + if (isset($fieldData['uid'])) { + $uid = $fieldData['uid']; } elseif (isset($dataHandler->substNEWwithIDs[$uid])) { $uid = $dataHandler->substNEWwithIDs[$uid]; } @@ -123,7 +123,7 @@ class DataHandler implements Singleton $this->logger->debug('Datahandler could not be setup.'); return false; } - if (! $this->dataHandler->canHandle($table)) { + if (! $this->dataHandler->supportsTable($table)) { $this->logger->debug('Table is not allowed.', [$table]); return false; } From 9a2d2734f29e319fd24c615c862480325c20df26 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 6 Mar 2018 17:58:27 +0100 Subject: [PATCH 129/158] TASK: Switch to new php scurtinizer engine --- .scrutinizer.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index ca92884..3122a2f 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,3 +1,11 @@ +build: + nodes: + analysis: + project_setup: + override: true + tests: + override: [php-scrutinizer-run] + filter: excluded_paths: - 'Configuration/*' @@ -19,7 +27,7 @@ tools: php_hhvm: enabled: true config: - use_undeclared_constant: false + use_undeclared_constant: false php_mess_detector: enabled: true @@ -34,5 +42,5 @@ tools: enabled: true # We generate code coverage during tests at travis and will send them here external_code_coverage: - runs: 2 - timeout: 1200 + runs: 2 + timeout: 1200 From 30a833ce68c5af08a97667a6e0b35b06ccb3aaa0 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 08:23:37 +0100 Subject: [PATCH 130/158] TASK: Add branch alias to provide version number Allow to require extension with 1.0.0, if dev is allowed. --- composer.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/composer.json b/composer.json index a630385..f58a062 100644 --- a/composer.json +++ b/composer.json @@ -37,6 +37,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "1.0.x-dev" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" From e960f4595b5727ec7ab3e870c7a9eedac611cda2 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 09:54:34 +0100 Subject: [PATCH 131/158] TASK: Add breaking change documentation Also order changelog entries news first. --- Documentation/source/changelog.rst | 3 ++- .../20180408-introduce-php70-type-hints.rst | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 Documentation/source/changelog/20180408-introduce-php70-type-hints.rst diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 0d7f020..3b121a3 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,4 +5,5 @@ Changelog :maxdepth: 1 :glob: - changelog/* + changelog/20180408-introduce-php70-type-hints + changelog/20180406-120-facet-configuration diff --git a/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst b/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst new file mode 100644 index 0000000..cc2dfe3 --- /dev/null +++ b/Documentation/source/changelog/20180408-introduce-php70-type-hints.rst @@ -0,0 +1,12 @@ +Breacking Change "Introduce PHP 7.0 TypeHints" +============================================== + +As PHP evolved, we now migrate the whole code base to use PHP 7.0 type hints. +We do not use PHP 7.1 Type Hints, as some customers still need PHP 7.0 support. + +Also we added missing methods to interfaces, that were already used in code. + +As this leads to changed method signatures, most custom implementations of interfaces, or overwrites +of existing methods are broken. + +To fix, just update the signatures as pointed out by PHP while running the code. From 43f31f75a8c7a6edbe3e1e7fc7f4baa265b52204 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:08:18 +0100 Subject: [PATCH 132/158] TASK: Add phan configuration Even if not used in CI yet, we still can use it local. --- .phan/config.php | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .phan/config.php diff --git a/.phan/config.php b/.phan/config.php new file mode 100644 index 0000000..11a17af --- /dev/null +++ b/.phan/config.php @@ -0,0 +1,96 @@ + '7.0', + + // Override to hardcode existence and types of (non-builtin) globals. + // Class names should be prefixed with '\\'. + // (E.g. ['_FOO' => '\\FooClass', 'page' => '\\PageClass', 'userId' => 'int']) + 'globals_type_map' => [ + '_EXTKEY' => 'string', + 'EM_CONF' => 'array', + ], + + // A list of directories that should be parsed for class and + // method information. After excluding the directories + // defined in exclude_analysis_directory_list, the remaining + // files will be statically analyzed for errors. + // + // Thus, both first-party and third-party code being used by + // your application should be included in this list. + 'directory_list' => [ + 'Classes', + '.Build/vendor', + ], + + // A list of files to include in analysis + 'file_list' => [ + 'ext_emconf.php', + 'ext_tables.php', + 'ext_localconf.php', + ], + + // A directory list that defines files that will be excluded + // from static analysis, but whose class and method + // information should be included. + // + // Generally, you'll want to include the directories for + // third-party code (such as "vendor/") in this list. + // + // n.b.: If you'd like to parse but not analyze 3rd + // party code, directories containing that code + // should be added to the `directory_list` as + // to `exclude_analysis_directory_list`. + "exclude_analysis_directory_list" => [ + '.Build/vendor' + ], + + // A list of directories that should be parsed for class and + // method information. After excluding the directories + // defined in exclude_analysis_directory_list, the remaining + // files will be statically analyzed for errors. + // + // Thus, both first-party and third-party code being used by + // your application should be included in this list. + 'directory_list' => [ + 'Classes', + // 'Tests', + '.Build/vendor', + ], + + // The number of processes to fork off during the analysis phase. + 'processes' => 3, + + // Add any issue types (such as 'PhanUndeclaredMethod') + // here to inhibit them from being reported + 'suppress_issue_types' => [ + 'PhanDeprecatedFunction', // For now + 'PhanParamTooMany', // For now, due to ObjectManager->get() + ], + + // A list of plugin files to execute. + // See https://github.com/phan/phan/tree/master/.phan/plugins for even more. + // (Pass these in as relative paths. + // The 0.10.2 release will allow passing 'AlwaysReturnPlugin' if referring to a plugin that is bundled with Phan) + 'plugins' => [ + // checks if a function, closure or method unconditionally returns. + 'AlwaysReturnPlugin', // can also be written as 'vendor/phan/phan/.phan/plugins/AlwaysReturnPlugin.php' + // Checks for syntactically unreachable statements in + // the global scope or function bodies. + 'UnreachableCodePlugin', + 'DollarDollarPlugin', + 'DuplicateArrayKeyPlugin', + 'PregRegexCheckerPlugin', + 'PrintfCheckerPlugin', + ], +]; From 4da9e86540bd0864651d70f563dc5841e9d3f730 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:09:07 +0100 Subject: [PATCH 133/158] TASK: Remove unused use statements --- Classes/Command/IndexCommandController.php | 1 - Classes/Connection/Elasticsearch/IndexFactory.php | 2 -- Classes/Connection/Elasticsearch/TypeFactory.php | 1 - Classes/Domain/Index/AbstractIndexer.php | 1 - Classes/Domain/Search/QueryFactory.php | 2 -- Classes/Domain/Service/DataHandler.php | 2 -- Classes/Hook/DataHandler.php | 1 - 7 files changed, 10 deletions(-) diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index 61aad53..e0bdeb9 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Command; use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; -use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Mvc\Controller\CommandController; /** diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 6ddab67..8b57783 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -22,10 +22,8 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; -use Elastica\Exception\ResponseException; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; /** * Factory to get indexes. diff --git a/Classes/Connection/Elasticsearch/TypeFactory.php b/Classes/Connection/Elasticsearch/TypeFactory.php index de8e336..e84cdd0 100644 --- a/Classes/Connection/Elasticsearch/TypeFactory.php +++ b/Classes/Connection/Elasticsearch/TypeFactory.php @@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Connection\Elasticsearch; */ use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; /** * Factory to get indexes. diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index e8f032e..d264b08 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; -use Codappix\SearchCore\DataProcessing\ProcessorInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index dcf6ce5..80c5151 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -23,8 +23,6 @@ namespace Codappix\SearchCore\Domain\Search; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; -use Codappix\SearchCore\Connection\ConnectionInterface; -use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; diff --git a/Classes/Domain/Service/DataHandler.php b/Classes/Domain/Service/DataHandler.php index 5b587e4..de226b9 100644 --- a/Classes/Domain/Service/DataHandler.php +++ b/Classes/Domain/Service/DataHandler.php @@ -24,9 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Index\IndexerFactory; use Codappix\SearchCore\Domain\Index\IndexerInterface; use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; -use Codappix\SearchCore\Domain\Index\TcaIndexer; use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Handles all data related things like updates, deletes and inserts. diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 3bd698f..71fe44b 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -21,7 +21,6 @@ namespace Codappix\SearchCore\Hook; */ use Codappix\SearchCore\Configuration\NoConfigurationException; -use Codappix\SearchCore\Domain\Index\NoMatchingIndexerException; use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; From fc8017fdf41a4e635baf7c635aed2bf1d5688087 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:09:28 +0100 Subject: [PATCH 134/158] TASK: Fix further issues reported by phan --- Classes/Connection/Elasticsearch/Connection.php | 2 +- .../ContentObjectDataProcessorAdapterProcessor.php | 2 +- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 2 +- Classes/Domain/Model/SearchRequest.php | 4 ++++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index 10a62d3..23ae703 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -44,7 +44,7 @@ class Connection implements Singleton /** * @param ConfigurationContainerInterface $configuration - * @param \Elastica\Client $elasticaClient + * @param \Elastica\Client|null $elasticaClient */ public function __construct( ConfigurationContainerInterface $configuration, diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php index 322486b..c3b4c32 100644 --- a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -21,7 +21,7 @@ namespace Codappix\SearchCore\DataProcessing; */ use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Service\TypoScriptService; +use TYPO3\CMS\Core\TypoScript\TypoScriptService; use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; /** diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 61e30a9..19f6a55 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,7 +21,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; +use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 2addff1..22bd697 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -156,11 +156,15 @@ class SearchRequest implements SearchRequestInterface public function setLimit($limit) { $this->limit = (int) $limit; + + return $this; } public function setOffset($offset) { $this->offset = (int) $offset; + + return $this; } public function getLimit() From 09cd655a7a2bc0336f9e8b20035d755b31596ffd Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 11:20:23 +0100 Subject: [PATCH 135/158] TASK: Remove whitespace in return type annotation --- Classes/Connection/Elasticsearch/Connection.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Connection/Elasticsearch/Connection.php b/Classes/Connection/Elasticsearch/Connection.php index 23ae703..a5e7d0f 100644 --- a/Classes/Connection/Elasticsearch/Connection.php +++ b/Classes/Connection/Elasticsearch/Connection.php @@ -68,7 +68,7 @@ class Connection implements Singleton /** * Get the concrete client for internal usage! */ - public function getClient() : \ Elastica\Client + public function getClient() : \Elastica\Client { return $this->elasticaClient; } From 4756da5c5d5df04ed94705d7cbd3b2caecaae3a4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 08:17:41 +0100 Subject: [PATCH 136/158] TASK: Update / cleanup documentation Code and docs have diverged a bit. We therefore need to update them. --- .../20180406-120-facet-configuration.rst | 4 +- Documentation/source/concepts.rst | 33 ++-- Documentation/source/conf.py | 10 +- Documentation/source/configuration.rst | 8 +- .../source/configuration/connections.rst | 11 +- .../dataProcessing/RemoveProcessor.rst | 2 +- .../dataProcessing/availableAndPlanned.rst | 36 ---- .../source/configuration/indexing.rst | 54 ++---- .../source/configuration/searching.rst | 166 ++++++++---------- Documentation/source/connections.rst | 25 ++- Documentation/source/dataprocessors.rst | 98 +++++++++++ Documentation/source/development.rst | 75 ++------ .../source/development/connection.rst | 11 ++ .../source/development/contribution.rst | 69 ++++++++ .../source/development/dataProcessor.rst | 15 ++ Documentation/source/development/indexer.rst | 21 +++ Documentation/source/features.rst | 40 +++-- Documentation/source/index.rst | 3 +- Documentation/source/indexer.rst | 39 +--- Documentation/source/installation.rst | 28 ++- Documentation/source/readme.rst | 19 +- Documentation/source/usage.rst | 18 +- composer.json | 3 + 23 files changed, 434 insertions(+), 354 deletions(-) delete mode 100644 Documentation/source/configuration/dataProcessing/availableAndPlanned.rst create mode 100644 Documentation/source/dataprocessors.rst create mode 100644 Documentation/source/development/connection.rst create mode 100644 Documentation/source/development/contribution.rst create mode 100644 Documentation/source/development/dataProcessor.rst create mode 100644 Documentation/source/development/indexer.rst diff --git a/Documentation/source/changelog/20180406-120-facet-configuration.rst b/Documentation/source/changelog/20180406-120-facet-configuration.rst index 53021d9..0c2338f 100644 --- a/Documentation/source/changelog/20180406-120-facet-configuration.rst +++ b/Documentation/source/changelog/20180406-120-facet-configuration.rst @@ -1,4 +1,4 @@ -Breacking Change 120 "Pass facets configuration to elasticsearch" +Breacking Change 120 "Pass facets configuration to Elasticsearch" ================================================================= In order to allow arbitrary facet configuration, we do not process the facet configuration anymore. @@ -35,6 +35,6 @@ Instead you have to provide the full configuration yourself: } } -You need to add line 4 and 6, the additional level ``terms`` for elasticsearch. +You need to add line 4 and 6, the additional level ``terms`` for Elasticsearch. See :issue:`120`. diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index f4c5bbd..f81121b 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -3,33 +3,35 @@ Concepts ======== -The extension is built with the following concepts in mind. +The main concept is to provide a foundation where other developers can profit from, to provide +integrations into search services like Elasticsearch, Algolia, ... . + +Our code contains the following concepts which should be understand: .. _concepts_connections: Connections ----------- -It should be possible to use different search services like elasticsearch and solr out of the box. -If a service is not contained, it should be possible to implement the necessary part by implementing -the necessary interfaces and configuring the extension to use the new connection. +Different search services can provide integrations. ``search_core`` only provides abstractions and +interfaces. The main purpose is to provide a stable API between TYPO3 and concrete connection. -Also it should be possible to use multiple connections at once. This way multiple search services -can be used in the same installation. - -Currently only :ref:`Elasticsearch` is provided. +For information about implementing a new connection, take a look at :ref:`development_connection`. .. _concepts_indexing: Indexing -------- -The indexing is done by one of the available indexer. For each identifier it's possible to define -the indexer to use. Also it's possible to write custom indexer to use. +Indexing is the process of collecting and preparing data, before sending it to a Connection. +The indexing is done by one of the available indexer. Indexer are identified by a key, as configured +in TypoScript. -Currently only the :ref:`TcaIndexer` is provided. +Currently :ref:`TcaIndexer` and :ref:`PagesIndexer` are provided. -.. _concepts_indexing_dataprocessing: +For information about implementing a new indexer, take a look at :ref:`development_indexer`. + +.. _concepts_dataprocessing: DataProcessing ^^^^^^^^^^^^^^ @@ -39,4 +41,9 @@ known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3 The same is true for retrieved search results. They can be processed again by "DataProcessors" to prepare data for display in Templates or further usage. -Configuration is done through TypoScript, see :ref:`dataProcessing`. +This should keep indexers simple and move logic to DataProcessors. This makes most parts highly +flexible as integrators are able to configure DataProcessors and change their order. + +Configuration is done through TypoScript, see :ref:`dataprocessors`. + +For information about implementing a new DataProcessor, take a look at :ref:`development_dataprocessor`. diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index 1419689..aa846e0 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -51,7 +51,7 @@ master_doc = 'index' # General information about the project. project = u'TYPO3 Extension search_core' -copyright = u'2016, Daniel Siepmann' +copyright = u'2016 - 2018, Daniel Siepmann' author = u'Daniel Siepmann' # The version info for the project you're documenting, acts as replacement for @@ -59,9 +59,9 @@ author = u'Daniel Siepmann' # built documents. # # The short X.Y version. -version = u'1.0.0' +version = u'0.0.1' # The full version, including alpha/beta/rc tags. -release = u'1.0.0' +release = u'0.0.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -133,7 +133,7 @@ html_theme_options = { # The name for this set of Sphinx documents. # " v documentation" by default. -#html_title = u'TYPO3 Extension search_core v1.0.0' +#html_title = u'TYPO3 Extension search_core v0.0.1' # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None @@ -150,7 +150,7 @@ html_theme_options = { # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 09289ac..53ac469 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -8,9 +8,9 @@ Configuration Installation wide configuration is handled inside of the extension manager. Just check out the options there, they all have labels. -The extension offers the following configuration options through TypoScript. If you overwrite them -through `setup` make sure to keep them in the `module` area as they will be accessed from backend -mode of TYPO3 for indexing. Do so by placing the following line at the end:: +Everything else is configured through TypoScript. If you overwrite them through `setup` make sure to +keep them in the `module` area as they will be accessed from backend mode of TYPO3 for indexing. Do +so by placing the following line at the end:: module.tx_searchcore < plugin.tx_searchcore @@ -26,12 +26,10 @@ Here is the example default configuration that's provided through static include .. literalinclude:: ../../Configuration/TypoScript/constants.txt :language: typoscript - :linenos: :caption: Static TypoScript Constants .. literalinclude:: ../../Configuration/TypoScript/setup.txt :language: typoscript - :linenos: :caption: Static TypoScript Setup .. _configuration_options: diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst index 5819730..6d0c524 100644 --- a/Documentation/source/configuration/connections.rst +++ b/Documentation/source/configuration/connections.rst @@ -4,7 +4,7 @@ Connections =========== Holds settings regarding the different possible connections for search services like Elasticsearch -or Solr. +or Algolia. Configured as:: @@ -22,15 +22,13 @@ Configured as:: Where ``connectionName`` is one of the available :ref:`connections`. -The following settings are available. For each setting its documented which connection consumes it. +The following settings are available. .. _host: ``host`` -------- -Used by: :ref:`Elasticsearch`. - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 installation. @@ -43,13 +41,8 @@ Example:: ``port`` -------- -Used by: :ref:`Elasticsearch`. - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. Example:: plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 - - - diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst index c8653a5..d39b42d 100644 --- a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -1,7 +1,7 @@ ``Codappix\SearchCore\DataProcessing\RemoveProcessor`` ====================================================== -Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. +Will remove fields from record. Possible Options: diff --git a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst b/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst deleted file mode 100644 index 9f31736..0000000 --- a/Documentation/source/configuration/dataProcessing/availableAndPlanned.rst +++ /dev/null @@ -1,36 +0,0 @@ -The following Processor are available: - -.. toctree:: - :maxdepth: 1 - :glob: - - /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor - /configuration/dataProcessing/CopyToProcessor - /configuration/dataProcessing/GeoPointProcessor - /configuration/dataProcessing/RemoveProcessor - -The following Processor are planned: - - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. - - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. - - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". - - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. - -Of course you are able to provide further processors. Just implement -``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified -class name) as done in the examples above. - -By implementing also the same interface as necessary for TYPO3 -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code -also for Fluid to prepare the same record fetched from DB for your fluid. - -Dependency injection is possible inside of processors, as we instantiate through extbase -``ObjectManager``. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 7d6c3d4..050f0ab 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -40,11 +40,7 @@ Contains a comma separated list of page uids. Spaces are trimmed. Example:: - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 - -Also it's possible to define some behaviour for the different document types. In context of TYPO3 -tables are used as document types 1:1. It's possible to configure different tables. The following -options are available: + plugin.tx_searchcore.settings.indexing.pages.rootLineBlacklist = 3, 10, 100 .. _additionalWhereClause: @@ -55,16 +51,16 @@ Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. Add additional SQL to where clauses to determine indexable records from the table. This way you can exclude specific records like ``tt_content`` records with specific ``CType`` values or -something else. E.g. you can add a new field to the table to exclude records from indexing. +something else. Example:: - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') .. attention:: Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. + database might contain joins and can lead to SQL errors if a field exists in multiple tables. .. _abstractFields: @@ -73,6 +69,10 @@ abstractFields Used by: :ref:`PagesIndexer`. +.. note:: + + Will be migrated to :ref:`dataprocessors` in the future. + Define which field should be used to provide the auto generated field "search_abstract". The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also possible. @@ -80,7 +80,7 @@ possible. Example:: # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + plugin.tx_searchcore.settings.indexing.pages.abstractFields := addToList(content) Default:: @@ -91,10 +91,10 @@ Default:: mapping ------- -Used by: Elasticsearch connection while indexing. +Used by: :ref:`connection_elasticsearch` connection while indexing. Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html -You are able to define the mapping for each property / columns. +You are able to define the mapping for each property / column. Example:: @@ -112,7 +112,7 @@ makes building a facet possible. index ----- -Used by: Elasticsearch connection while indexing. +Used by: :ref:`connection_elasticsearch` connection while indexing. Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html @@ -141,36 +141,12 @@ Example:: ``char_filter`` and ``filter`` are a comma separated list of options. -.. _dataProcessing: +.. _indexing_dataProcessing: dataProcessing -------------- Used by: All connections while indexing, due to implementation inside ``AbstractIndexer``. -Configure modifications on each document before sending it to the configured connection. Same as -provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - -All processors are applied in configured order. Allowing to work with already processed data. - -Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } - } - -The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards -all fields, including ``search_spellcheck`` will be copied to ``search_all``. - -.. include:: /configuration/dataProcessing/availableAndPlanned.rst - -Also data processors are available for search results too, see :ref:`searching_dataProcessing`. +Configure modifications on each document before sending it to the configured connection. +For full documentation check out :ref:`dataprocessors`. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index 857aa74..52abc07 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -8,45 +8,50 @@ Searching size ---- -Used by: Elasticsearch connection while building search query. - Defined how many search results should be fetched to be available in search result. Example:: plugin.tx_searchcore.settings.searching.size = 50 -Default if not configured is 10. +Default is ``10``. .. _facets: facets ------ -Used by: Elasticsearch connection while building search query. - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html -Currently only the term facet is provided. Example:: - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType + category { + terms { + field = categories } } -The above example will provide a facet with options for all found ``CType`` results together -with a count. + month { + date_histogram { + field = released + interval = month + format = Y-MM-01 + order { + _time = desc + } + } + } + + +The above example will provide a facet with options for all found ``categories`` results together +with a count. Also a facet for ``released`` will be provided. .. _filter: filter ------ -Used by: While building search request. - -Define filter that should be set for all requests. +Define filter that should be set for all search requests. Example:: @@ -54,6 +59,9 @@ Example:: property = value } +Also see :ref:`mapping.filter` to map incoming request information, e.g. from a ``select``, to build +more complex filters. + For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. .. _minimumShouldMatch: @@ -61,9 +69,8 @@ For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``key minimumShouldMatch ------------------ -Used by: Elasticsearch connection while building search query. - -Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html +Define the minimum match for Elasticsearch, have a look at the official docs: +https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html Example:: @@ -74,8 +81,6 @@ Example:: boost ----- -Used by: Elasticsearch connection while building search query. - Define fields that should boost the score for results. Example:: @@ -93,10 +98,9 @@ https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses. fieldValueFactor ---------------- -Used by: Elasticsearch connection while building search query. - -Define a field to use as a factor for scoring. The configuration is passed through to elastic -search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor +Define a field to use as a factor for scoring. The configuration is passed through to Elasticsearch +``field_value_factor``, see: +https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor Example:: @@ -119,34 +123,34 @@ E.g. you submit a filter in form of: .. code-block:: html - - Due to TYPO3 7.x fluid limitations, we build this input ourself. - No longer necessary in 8 and above - - - + + Due to TYPO3 7.x fluid limitations, we build this input ourself. + No longer necessary in 8 and above + + + -This will create a ``distance`` filter with subproperties. To make this filter actually work, you +This will create a ``month`` filter with sub properties. To make this filter actually work, you can add the following TypoScript, which will be added to the filter:: mapping { @@ -167,38 +171,36 @@ can add the following TypoScript, which will be added to the filter:: } ``fields`` has a special meaning here. This will actually map the properties of the filter to fields -in elasticsearch. In above example they do match, but you can also use different names in your form. -On the left hand side is the elasticsearch field name, on the right side the one submitted as a -filter. +in Elasticsearch. On the left hand side is the Elasticsearch field name, on the right side the one +submitted as a filter. -The ``field``, in above example ``released``, will be used as the elasticsearch field for -filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. +The ``field``, in above example ``released``, will be used as the Elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing Elasticsearch fields. Everything that is configured inside ``raw`` is passed, as is, to search service, e.g. -elasticsearch. +Elasticsearch. .. _fields: fields ------ -Defines the fields to fetch and search from elasticsearch. With the following sub keys: +Defines the fields to fetch and search from Elasticsearch. With the following sub keys: -``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch. -Configure a comma separated list of fields to search in. This is necessary if you have configured -special mapping for some fields, or just want to search some fields. -The most hits get ranked highest. The following is an example configuration:: +``query`` defines the fields to search in. Configure a comma separated list of fields to search in. +This is necessary if you have configured special mapping for some fields, or just want to search +some fields. The following is an example configuration:: fields { query = _all, city } -The following sub properties configure the fields to fetch from elasticsearch: +The following sub properties configure the fields to fetch from Elasticsearch: First ``stored_fields`` which is a list of comma separated fields which actually exist and will be added. Typically you will use ``_source`` to fetch the whole indexed fields. -Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +Second is ``script_fields``, which allow you to configure scripted fields for Elasticsearch. An example might look like the following:: fields { @@ -221,7 +223,7 @@ In above example we add a single ``script_field`` called ``distance``. We add a field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the query. The ``condition`` will be removed and everything else is submitted one to one to -elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +Elasticsearch, except each property is run through Fluidtemplate, to allow you to use information from search request, e.g. to insert latitude and longitude from a filter, like in the above example. .. _sort: @@ -246,38 +248,14 @@ Example:: mode = filter } -Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. +Only ``filter`` is allowed as value, as ``search`` is default behaviour. Using ``filter`` will +trigger a search to provide data while visiting the page, possible :ref:`filter` allow you to build +pages like "News". -.. _searching_dataProcessing: +.. _searching_dataprocessing: dataProcessing -------------- -Used by: All connections while indexing, due to implementation inside ``SearchService``. - -Configure modifications on each document before returning search result. Same as provided by TYPO3 -for :ref:`t3tsref:cobj-fluidtemplate` through -:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - -All processors are applied in configured order. Allowing to work with already processed data. - -Example:: - - plugin.tx_searchcore.settings.searching.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } - } - -The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards -all fields, including ``search_spellcheck`` will be copied to ``search_all``. - -.. include:: /configuration/dataProcessing/availableAndPlanned.rst - -Also data processors are available while indexing too, see :ref:`dataProcessing`. +Configure modifications on each document before returning search result. +For full documentation check out :ref:`dataprocessors`. diff --git a/Documentation/source/connections.rst b/Documentation/source/connections.rst index b51f280..73ce273 100644 --- a/Documentation/source/connections.rst +++ b/Documentation/source/connections.rst @@ -5,26 +5,25 @@ Connections See Concept of :ref:`concepts_connections` for further background information. -The extension provides the following connections out of the box: +For information about implementing a new connection, take a look at :ref:`development_connection`. -.. _Elasticsearch: +The following connections were developed, or are in development, for ``search_core``: + +.. _connection_elasticsearch: Elasticsearch ------------- -Integrates `elastic Elasticsearch`_ using `elastica`_ into TYPO3. +Support for `Elasticsearch`_ is provided out of the box by `search_core` at the moment. -Provides basic support like indexing without mappings and full text search at the moment. +.. _Elasticsearch: https://www.elastic.co/products/elasticsearch -The connection is configurable through the following options: +.. _connection_algolia: -* :ref:`host` +Algolia +------- -* :ref:`port` +`search_algolia`_ will integrate `Algolia`_ and is currently under development by Martin Hummer. -* :ref:`mapping` - -* :ref:`facets` - -.. _elastic Elasticsearch: https://www.elastic.co/products/elasticsearch -.. _elastica: http://elastica.io/ +.. _search_algolia: https://github.com/martinhummer/search_algolia/ +.. _Algolia: https://www.algolia.com/ diff --git a/Documentation/source/dataprocessors.rst b/Documentation/source/dataprocessors.rst new file mode 100644 index 0000000..67ff655 --- /dev/null +++ b/Documentation/source/dataprocessors.rst @@ -0,0 +1,98 @@ +.. _dataprocessors: + +DataProcessors +============== + +See Concept of :ref:`concepts_dataprocessing` for further background information. + +For information about implementing a new DataProcessor, take a look at +:ref:`development_dataprocessor`. + +Same as provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + +.. _dataprocessors_usage: + +Usage +----- + +All processors are applied in configured order. Allowing to work with already processed data. +They can be applied during indexing and for search results. + +Example for indexing:: + + plugin.tx_searchcore.settings.indexing.pages.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. + +Example for search results:: + + plugin.tx_searchcore.settings.searching.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. + +.. _dataprocessors_availableDataProcessors: + +Available DataProcessors +------------------------ + +.. toctree:: + :maxdepth: 1 + :glob: + + /configuration/dataProcessing/ContentObjectDataProcessorAdapterProcessor + /configuration/dataProcessing/CopyToProcessor + /configuration/dataProcessing/GeoPointProcessor + /configuration/dataProcessing/RemoveProcessor + +.. _dataprocessors_plannedDataProcessors: + +Planned DataProcessors +---------------------- + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + This is currently done through indexer. + +.. Of course you are able to provide further processors. Just implement +.. ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +.. class name) as done in the examples above. + +.. By implementing also the same interface as necessary for TYPO3 +.. :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +.. also for Fluid to prepare the same record fetched from DB for your fluid. + +.. Dependency injection is possible inside of processors, as we instantiate through extbase +.. ``ObjectManager``. diff --git a/Documentation/source/development.rst b/Documentation/source/development.rst index e73425c..e65dcdf 100644 --- a/Documentation/source/development.rst +++ b/Documentation/source/development.rst @@ -1,71 +1,16 @@ .. highlight:: bash -.. _contribution: - -Contribution -============ - -Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind. - -Below is a documentation what to respect during contributions. - -.. _contribution_setup: - -Setup ------ - -To start contributions regarding code, make sure your environment matches the following -requirements: - -* composer is executable - -* PHP on CLI is executable - -* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the - environment variables, see :file:`Makefile`. - And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see - https://review.typo3.org/#/c/26725/3/INSTALL.md. - -* Elasticsearch is installed and up and running on *localhost:9200*. - -Then setup your system:: - - git clone git@github.com:DanielSiepmann/search_core.git \ - && cd search_core \ - && export typo3DatabaseName="searchcoretest76" \ - && export TYPO3_VERSION="~7.6" \ - && make install \ - && make unitTests \ - && make functionalTests - -If all tests are okay, start your work. - -If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and -`TYPO3_VERSION` in your environment like:: - - export typo3DatabaseName="searchcoretest62" - export TYPO3_VERSION="~6.2" - -Also run the install command for each version before running any tests. Only this will make sure you -are testing against the actual TYPO3 Version and database scheme. - -.. _contribution_development: - Development ------------ +=========== -All changes are introduced through pull requests at `Github`_ and should contain the following: +There are some ways we will cover here. One is how you can develop own parts like Indexer, +DataProcessor and Connection. The other is how to contribute. -* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_. +.. toctree:: + :maxdepth: 1 + :glob: -* New tests whenever possible and useful. - -* Code has to follow `PSR-2`_. - -* Adjusted documentation. - -* Make sure to follow the documented :ref:`concepts`. - -.. _Github: https://github.com/DanielSiepmann/search_core -.. _travis-ci: https://travis-ci.org/ -.. _PSR-2: http://www.php-fig.org/psr/psr-2/ + development/indexer + development/dataProcessor + development/connection + development/contribution diff --git a/Documentation/source/development/connection.rst b/Documentation/source/development/connection.rst new file mode 100644 index 0000000..af0ab14 --- /dev/null +++ b/Documentation/source/development/connection.rst @@ -0,0 +1,11 @@ +.. _development_connection: + +Develop a new Connection +======================== + +Make sure you understood :ref:`concepts_connections`. + +Each Connection has to be a single class which implements +``Codappix\SearchCore\Connection\ConnectionInterface``. + +Dependency Injection is working for custom Connections. diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst new file mode 100644 index 0000000..32b6400 --- /dev/null +++ b/Documentation/source/development/contribution.rst @@ -0,0 +1,69 @@ +.. _contribution: + +Contribution +============ + +Everyone is welcome to contribute, whether it's code, issues, feature requests or any other kind. + +Below is a documentation what to respect during contributions. + +.. _contribution_setup: + +Setup +----- + +To start contributions regarding code, make sure your environment matches the following +requirements: + +* composer is executable + +* PHP on CLI is executable + +* MySQL is up and running with user *dev* and password *dev* on *127.0.0.1* or to overwrite the + environment variables, see :file:`Makefile`. + And MySQL is not set to strict mode as TYPO3 doesn't support strict mode, see + https://review.typo3.org/#/c/26725/3/INSTALL.md. + +* Elasticsearch is installed and up and running on *localhost:9200*. + +Then setup your system:: + + git clone git@github.com:DanielSiepmann/search_core.git \ + && cd search_core \ + && export typo3DatabaseName="searchcoretest76" \ + && export TYPO3_VERSION="~8.7" \ + && make install \ + && make unitTests \ + && make functionalTests + +If all tests are okay, start your work. + +If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and +`TYPO3_VERSION` in your environment like:: + + export typo3DatabaseName="searchcoretest62" + export TYPO3_VERSION="~6.2" + +Also run the install command for each version before running any tests. Only this will make sure you +are testing against the actual TYPO3 Version and database scheme. + +.. _contribution_development: + +Development +----------- + +All changes are introduced through pull requests at `Github`_ and should contain the following: + +* Adjusted tests if tests existed before. Otherwise they will break on `travis-ci`_. + +* New tests whenever possible and useful. + +* Code has to follow `PSR-2`_. + +* Adjusted documentation. + +* Make sure to follow the documented :ref:`concepts`. + +.. _Github: https://github.com/DanielSiepmann/search_core +.. _travis-ci: https://travis-ci.org/ +.. _PSR-2: http://www.php-fig.org/psr/psr-2/ diff --git a/Documentation/source/development/dataProcessor.rst b/Documentation/source/development/dataProcessor.rst new file mode 100644 index 0000000..af2732b --- /dev/null +++ b/Documentation/source/development/dataProcessor.rst @@ -0,0 +1,15 @@ +.. _development_dataprocessor: + +Develop a new DataProcessor +=========================== + +Make sure you understood :ref:`concepts_dataprocessing`. + +Each DataProcessor has to be a single class which implements +``Codappix\SearchCore\DataProcessing\ProcessorInterface``. + +Make sure you support both, Frontend and Backend, as processors can be called during searching and +indexing. Therefore do not rely on e.g. ``TSFE``, make sure dependencies are met and your code will +work in both environments. + +Dependency Injection is working for custom DataProcessors. diff --git a/Documentation/source/development/indexer.rst b/Documentation/source/development/indexer.rst new file mode 100644 index 0000000..c44aabd --- /dev/null +++ b/Documentation/source/development/indexer.rst @@ -0,0 +1,21 @@ +.. _development_indexer: + +Develop a new Indexer +===================== + +Make sure you understood :ref:`concepts_indexing`. + +Each indexer has to be a single class which implements +``Codappix\SearchCore\Domain\Index\IndexerInterface``. + +The indexer should call the connection with all necessary information about the document(s) to +trigger indexing or deletion of whole index. + +As this is the "indexer", deletion of single documents is directly processed by the connection. + +``setIdentifier`` is called with the identifier of the current Indexer. This might be usefull to +fetch configuration, related to the indexing, from +``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``. + +Dependency Injection is working for custom indexers, therefore you are able to inject the +``ConfigurationContainerInterface``. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 1a4abd6..57014bd 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -3,28 +3,34 @@ Features ======== -The following features are currently provided: +The following features are available: .. _features_indexing: Indexing -------- -Indexing data to Elasticsearch is provided. The extension delivers an indexer for TCA with zero -configuration needs. Still it's possible to configure the indexer. +Indexing of data is possible. We deliver an indexer for TCA with zero configuration needs. You can +also provide custom indexer for arbitrary data which is not indexable through TCA. -Also custom classes can be used as indexers. +Also a finisher for TYPO3 Form-Extension is provided to integrate indexing after something was +update through the Form-Extension. -Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing. +Indexing is done through Hooks and CLI. We therefore provide commands to index and delete indexed +data. .. _features_search: Searching --------- -Currently all fields are searched for a single search input. +.. note:: + Currently only integrated for Elasticsearch with no abstraction. + If you need to implement your own search, please open an issue on Github and we will change the code + base. -Also multiple filter are supported. Filtering results by fields for string contents. +Via TypoScript it's possible to configure the fields to query, minimum match and script fields. +Also multiple filter are supported, filtering results by fields. Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. @@ -32,19 +38,25 @@ indexing, and the facets itself while searching. .. _features_dataProcessing: DataProcessing -============== +-------------- -DataProcessing, as known from ``FLUIDTEMPLATE`` is available while indexing and for search results. -Each item can be processed by multiple processor to prepare data for indexing and output. +DataProcessing, as known from ``FLUIDTEMPLATE``, is available while indexing and for search results. +Each record and result item can be processed by multiple processor to prepare data for indexing and +output. -See :ref:`concepts_indexing_dataprocessing` in :ref:`concepts` section. +See :ref:`concepts_dataprocessing` in :ref:`concepts` section. .. _features_planned: Planned ---------- +------- The following features are currently planned and will be integrated: -#. Pagination - Add a pagination to search results, to allow users to walk through all results. +#. :issue:`25` Multi language. +#. :issue:`94` Respect access rights while indexing relations. +#. :issue:`75` Configuration of index name (for Elasticsearch). + +For a full list, check out our `open issues`_. + +.. _open issues: https://github.com/Codappix/search_core/issues diff --git a/Documentation/source/index.rst b/Documentation/source/index.rst index 2edfae3..1cc9eb6 100644 --- a/Documentation/source/index.rst +++ b/Documentation/source/index.rst @@ -1,7 +1,7 @@ .. include:: readme.rst Table of Contents -================= +----------------- .. toctree:: :maxdepth: 1 @@ -14,5 +14,6 @@ Table of Contents concepts connections indexer + dataprocessors development changelog diff --git a/Documentation/source/indexer.rst b/Documentation/source/indexer.rst index 3bb10bc..733f5b9 100644 --- a/Documentation/source/indexer.rst +++ b/Documentation/source/indexer.rst @@ -5,6 +5,8 @@ Indexer See Concept of :ref:`concepts_indexing` for further background information. +For information about implementing a new indexer, take a look at :ref:`development_indexer`. + The extension provides the following indexer out of the box: .. _TcaIndexer: @@ -15,23 +17,12 @@ TcaIndexer Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start`. You just can start indexing TYPO3. -The indexer will use the TCA to fetch all necessary information like relations. Currently the -implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and -further stuff. +Just add the indexer for a TYPO3 table. The indexer will use the TCA to fetch all necessary +information like relations. -The indexer is configurable through the following options: +.. note:: -* :ref:`rootLineBlacklist` - -* :ref:`additionalWhereClause` - -* :ref:`abstractFields` - -* :ref:`mapping` - -* :ref:`index` - -* :ref:`dataProcessing` + Not all relations are resolved yet, see :issue:`17`. .. _PagesIndexer: @@ -42,23 +33,7 @@ Provides zero configuration TYPO3 integration by using the :ref:`t3tcaref:start` start indexing TYPO3. The indexer will use the TCA to fetch all necessary information like relations. Currently the -implementation is very basic. In future it will also provide mapping for :ref:`Elasticsearch` and -further stuff. Also all static content from each page will be concatenated into a single field to -improve search. - -The indexer is configurable through the following options: - -* :ref:`rootLineBlacklist` - -* :ref:`additionalWhereClause` - -* :ref:`abstractFields` - -* :ref:`mapping` - -* :ref:`index` - -* :ref:`dataProcessing` +implementation is very basic. .. note:: diff --git a/Documentation/source/installation.rst b/Documentation/source/installation.rst index 5bb0c56..0a0e5b6 100644 --- a/Documentation/source/installation.rst +++ b/Documentation/source/installation.rst @@ -4,23 +4,41 @@ Installation ============ +Composer +-------- + The extension can be installed through composer:: - composer require "leonmrni/search_core dev-master as 1.0.x-dev" + composer require "codappix/search_core" "~1.0.0" -or by `downloading`_ and placing it inside the :file:`typo3conf/ext`-Folder of your installation. -In that case you need to install all dependencies yourself. Dependencies are: +Note that you have to allow unstable packages: + +.. code-block:: json + + { + "minimum-stability": "dev", + "prefer-stable": true + } + +Download +-------- + +You can also `download`_ the extension and placing it inside the :file:`typo3conf/ext`-Folder of +your installation. In that case you need to install all dependencies yourself. Dependencies are: .. literalinclude:: ../../composer.json :caption: Dependencies from composer.json :lines: 19-21 :dedent: 8 +Setup +----- + Afterwards you need to enable the extension through the extension manager and include the static TypoScript setup. -If you **don't** want to use the included elasticsearch integration, you have to disable it in the +If you **don't** want to use the included Elasticsearch integration, you have to disable it in the extension manager configuration of the extension by checking the checkbox. It's currently enabled by default but will be moved into its own extension in the future. -.. _downloading: https://github.com/DanielSiepmann/search_core/archive/master.zip +.. _download: https://github.com/codappix/search_core/archive/develop.zip diff --git a/Documentation/source/readme.rst b/Documentation/source/readme.rst index ba77ae8..f4ad917 100644 --- a/Documentation/source/readme.rst +++ b/Documentation/source/readme.rst @@ -1,5 +1,5 @@ -TYPO3 Extension search_core's documentation! -============================================ +TYPO3 Extension search_core +=========================== Introduction ============ @@ -8,24 +8,21 @@ What does it do? ---------------- The goal of this extension is to provide search integrations into TYPO3 CMS. The extension will -abstract the concrete implementations to allow exchange of concrete backends like Elasticsearch or -solr. +provide a convenient API to allow developers to provide concrete implementations of backends like +Elasticsearch, Algolia or Solr. The extension provides integration into TYPO3 like a frontend plugin for searches and hooks to update search indexes on updates. Also a command line interface is provided for interactions like -reindexing. +re-indexing. Current state ------------- This is still a very early beta version. More information can be taken from Github at -`current issues`_ and `current projects`_. +`current issues`_. -We are also focusing on Code Quality and Testing through `travis ci`_, `scrutinizer`_ and `codacy`_. +We are also focusing on Code Quality and Testing through `travis ci`_, ``phpcs``, ``phpunit`` and +``phpstan``. .. _current issues: https://github.com/Codappix/search_core/issues -.. _current projects: https://github.com/Codappix/search_core/projects .. _travis ci: https://travis-ci.org/Codappix/search_core -.. _scrutinizer: https://scrutinizer-ci.com/g/Codappix/search_core/inspections -.. _codacy: https://www.codacy.com/app/Codappix/search_core/dashboard - diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index d57d643..fc6d08a 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -11,9 +11,10 @@ Manual indexing You can trigger indexing from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'tt_content' + ./typo3/cli_dispatch.phpsh extbase index:index --identifier 'pages' + ./bin/typo3cms index:index --identifier 'pages' -This will index the table ``tt_content`` using the :ref:`TcaIndexer`. +This will index the table ``pages`` using the :ref:`TcaIndexer`. Only one index per call is available, to run multiple indexers, just make multiple calls. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. @@ -25,9 +26,10 @@ Manual deletion You can trigger deletion for a single index from CLI:: - ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content' + ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'pages' + ./bin/typo3cms index:delete --identifier 'pages' -This will delete the index for the table ``tt_content``. +This will delete the index for the table ``pages``. Only one delete per call is available, to run multiple deletions, just make multiple calls. @@ -53,7 +55,6 @@ A form finisher is provided to integrate indexing into form extension. Add form finisher to your available finishers and configure it like: .. code-block:: yaml - :linenos: - identifier: SearchCoreIndexer @@ -62,7 +63,7 @@ Add form finisher to your available finishers and configure it like: indexIdentifier: 'fe_users' recordUid: '{FeUser.user.uid}' -All three options are necessary, where +All three options are necessary, where: ``action`` Is one of ``delete``, ``update`` or ``add``. @@ -81,7 +82,7 @@ plugin. The plugin is named *Search Core*. Please provide your own template, the extension will not deliver a useful template for now. -The extbase mapping is used, this way you can create a form: +The Extbase mapping is used, this way you can create a form: .. code-block:: html @@ -95,11 +96,10 @@ The extbase mapping is used, this way you can create a form: Filter """""" -Thanks to extbase mapping, filter are added to the form: +Thanks to Extbase mapping, filter are added to the form: .. code-block:: html - .. _usage_searching_facets: diff --git a/composer.json b/composer.json index a630385..a935819 100644 --- a/composer.json +++ b/composer.json @@ -37,6 +37,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "0.0.1" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" From 3bebfac7698541f0d9da6775e8f648b5683f418b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 19:26:36 +0100 Subject: [PATCH 137/158] TASK: Fix documentation As we have moved the repository at github, we should update all mentions to use new url. Also a small typo was fixed. --- Documentation/source/development/contribution.rst | 4 ++-- Documentation/source/development/indexer.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst index 32b6400..c362c37 100644 --- a/Documentation/source/development/contribution.rst +++ b/Documentation/source/development/contribution.rst @@ -28,7 +28,7 @@ requirements: Then setup your system:: - git clone git@github.com:DanielSiepmann/search_core.git \ + git clone git@github.com:codappix/search_core.git \ && cd search_core \ && export typo3DatabaseName="searchcoretest76" \ && export TYPO3_VERSION="~8.7" \ @@ -64,6 +64,6 @@ All changes are introduced through pull requests at `Github`_ and should contain * Make sure to follow the documented :ref:`concepts`. -.. _Github: https://github.com/DanielSiepmann/search_core +.. _Github: https://github.com/codappix/search_core .. _travis-ci: https://travis-ci.org/ .. _PSR-2: http://www.php-fig.org/psr/psr-2/ diff --git a/Documentation/source/development/indexer.rst b/Documentation/source/development/indexer.rst index c44aabd..d5a067e 100644 --- a/Documentation/source/development/indexer.rst +++ b/Documentation/source/development/indexer.rst @@ -13,7 +13,7 @@ trigger indexing or deletion of whole index. As this is the "indexer", deletion of single documents is directly processed by the connection. -``setIdentifier`` is called with the identifier of the current Indexer. This might be usefull to +``setIdentifier`` is called with the identifier of the current Indexer. This might be useful to fetch configuration, related to the indexing, from ``Codappix\SearchCore\Configuration\ConfigurationContainerInterface``. From a469f63aa651a2a6294846103740ca4ce1b505c2 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 10:51:59 +0100 Subject: [PATCH 138/158] TASK: Cleanup differences We have some small differences between develop and support/76 branch. This differences are just CGL and should not exist. --- Classes/Domain/Index/AbstractIndexer.php | 4 ---- Classes/Domain/Index/TcaIndexer/RelationResolver.php | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index d264b08..6644716 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -68,10 +68,6 @@ abstract class AbstractIndexer implements IndexerInterface $this->identifier = $identifier; } - /** - * @param ConnectionInterface $connection - * @param ConfigurationContainerInterface $configuration - */ public function __construct(ConnectionInterface $connection, ConfigurationContainerInterface $configuration) { $this->connection = $connection; diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index fb19a4f..6af330e 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -97,7 +97,7 @@ class RelationResolver implements Singleton return array_map('trim', explode(',', $value)); } - protected function getUtilityForMode(): string + protected function getUtilityForMode() : string { if (TYPO3_MODE === 'BE') { return BackendUtility::class; From ba19537f4e9e10800a41a1a7a12f3243a9a54e92 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 11:02:29 +0100 Subject: [PATCH 139/158] TASK: Migrate existing 7.6 features We had some features in 7.6 support which we didn't merge up yet. Mostly very small bug fixes or more helpful logging and processing of elasticsearch options. But also adding images of content elements while indexing pages. --- .../Connection/Elasticsearch/IndexFactory.php | 15 +++-- .../Domain/Index/TcaIndexer/PagesIndexer.php | 55 ++++++++++++++++--- Classes/Domain/Search/QueryFactory.php | 14 +++-- 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 8b57783..c56892a 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -53,7 +53,10 @@ class IndexFactory implements Singleton $index = $connection->getClient()->getIndex('typo3content'); if ($index->exists() === false) { - $index->create($this->getConfigurationFor($documentType)); + $config = $this->getConfigurationFor($documentType); + $this->logger->debug(sprintf('Create index %s.', $documentType), [$documentType, $config]); + $index->create($config); + $this->logger->debug(sprintf('Created index %s.', $documentType), [$documentType]); } return $index; @@ -64,9 +67,11 @@ class IndexFactory implements Singleton try { $configuration = $this->configuration->get('indexing.' . $documentType . '.index'); - if (isset($configuration['analysis']['analyzer'])) { - foreach ($configuration['analysis']['analyzer'] as $key => $analyzer) { - $configuration['analysis']['analyzer'][$key] = $this->prepareAnalyzerConfiguration($analyzer); + foreach (['analyzer', 'filter'] as $optionsToExpand) { + if (isset($configuration['analysis'][$optionsToExpand])) { + foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options); + } } } @@ -78,7 +83,7 @@ class IndexFactory implements Singleton protected function prepareAnalyzerConfiguration(array $analyzer) : array { - $fieldsToExplode = ['char_filter', 'filter']; + $fieldsToExplode = ['char_filter', 'filter', 'word_list']; foreach ($fieldsToExplode as $fieldToExplode) { if (isset($analyzer[$fieldToExplode])) { diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 4f6d03d..8882696 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -34,6 +34,12 @@ class PagesIndexer extends TcaIndexer */ protected $contentTableService; + /** + * @var \TYPO3\CMS\Core\Resource\FileRepository + * @inject + */ + protected $fileRepository; + /** * @param TcaTableService $tcaTableService * @param TcaTableService $contentTableService @@ -60,28 +66,63 @@ class PagesIndexer extends TcaIndexer } } - $record['content'] = $this->fetchContentForPage($record['uid']); + $record['media'] = $this->fetchMediaForPage($record['uid']); + $content = $this->fetchContentForPage($record['uid']); + if ($content !== []) { + $record['content'] = $content['content']; + $record['media'] = array_values(array_unique(array_merge($record['media'], $content['images']))); + } parent::prepareRecord($record); } - protected function fetchContentForPage(int $uid) : string + protected function fetchContentForPage(int $uid) : array { $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); if ($contentElements === null) { $this->logger->debug('No content for page ' . $uid); - return ''; + return []; } $this->logger->debug('Fetched content for page ' . $uid); + $images = []; $content = []; foreach ($contentElements as $contentElement) { + $images = array_merge( + $images, + $this->getContentElementImages($contentElement['uid']) + ); $content[] = $contentElement['bodytext']; } - // Remove Tags. - // Interpret escaped new lines and special chars. - // Trim, e.g. trailing or leading new lines. - return trim(stripcslashes(strip_tags(implode(' ', $content)))); + return [ + // Remove Tags. + // Interpret escaped new lines and special chars. + // Trim, e.g. trailing or leading new lines. + 'content' => trim(stripcslashes(strip_tags(implode(' ', $content)))), + 'images' => $images, + ]; + } + + protected function getContentElementImages(int $uidOfContentElement) : array + { + return $this->fetchSysFileReferenceUids($uidOfContentElement, 'tt_content', 'image'); + } + + protected function fetchMediaForPage(int $uid) : array + { + return $this->fetchSysFileReferenceUids($uid, 'pages', 'media'); + } + + protected function fetchSysFileReferenceUids(int $uid, string $tablename, string $fieldname) : array + { + $imageRelationUids = []; + $imageRelations = $this->fileRepository->findByRelation($tablename, $fieldname, $uid); + + foreach ($imageRelations as $relation) { + $imageRelationUids[] = $relation->getUid(); + } + + return $imageRelationUids; } } diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 80c5151..98e3324 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -136,13 +136,15 @@ class QueryFactory ]; } - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'query' => [ - 'bool' => [ - 'should' => $boostQueryParts, + if (!empty($boostQueryParts)) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'query' => [ + 'bool' => [ + 'should' => $boostQueryParts, + ], ], - ], - ]); + ]); + } } protected function addFactorBoost(array &$query) From 75335e2bc145657fccbbceec267e23bfab0a7c53 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 11:03:57 +0100 Subject: [PATCH 140/158] TASK: Do not import unnecessary namespace As imported class is on same level as we are, we do not need to import the class. --- Classes/Domain/Index/IndexerFactory.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index 6efe035..b1a431d 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -23,7 +23,6 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Index\IndexerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; From 16bc22aa44282d7e668476336513139e19a59d6f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 11:28:40 +0100 Subject: [PATCH 141/158] TASK: Support tests for TYPO3 CMS 7.6 Check which version to run and switch TYPO3 bootstrapping. Also allow tests to run with TYPO3 CMS 7.6 again. --- .travis.yml | 3 +++ .../source/development/contribution.rst | 6 ++--- Makefile | 11 ++++++++- Tests/Functional/Bootstrap.php | 9 +++++++ Tests/Functional/FunctionalTests.xml | 3 +-- Tests/InstallPatches/composer.json.patch | 14 +++++++++++ Tests/Unit/Bootstrap.php | 9 +++++++ Tests/Unit/UnitTests.xml | 2 +- composer.json | 2 +- ext_emconf.php | 6 ++--- ext_localconf.php | 24 +++++++++++++++---- 11 files changed, 73 insertions(+), 16 deletions(-) create mode 100644 Tests/Functional/Bootstrap.php create mode 100644 Tests/InstallPatches/composer.json.patch create mode 100644 Tests/Unit/Bootstrap.php diff --git a/.travis.yml b/.travis.yml index a765673..2babbf3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ language: php php: - 7.0 - 7.1 + - 7.2 env: global: @@ -24,6 +25,8 @@ env: - typo3DatabaseHost="127.0.0.1" - typo3DatabaseUsername="travis" - typo3DatabasePassword="" + matrix: + - TYPO3_VERSION="~7.6" matrix: fast_finish: true diff --git a/Documentation/source/development/contribution.rst b/Documentation/source/development/contribution.rst index c362c37..f65bb21 100644 --- a/Documentation/source/development/contribution.rst +++ b/Documentation/source/development/contribution.rst @@ -30,7 +30,7 @@ Then setup your system:: git clone git@github.com:codappix/search_core.git \ && cd search_core \ - && export typo3DatabaseName="searchcoretest76" \ + && export typo3DatabaseName="searchcoretest87" \ && export TYPO3_VERSION="~8.7" \ && make install \ && make unitTests \ @@ -41,8 +41,8 @@ If all tests are okay, start your work. If you are working with multiple TYPO3 versions make sure to export `typo3DatabaseName` and `TYPO3_VERSION` in your environment like:: - export typo3DatabaseName="searchcoretest62" - export TYPO3_VERSION="~6.2" + export typo3DatabaseName="searchcoretest76" + export TYPO3_VERSION="~7.6" Also run the install command for each version before running any tests. Only this will make sure you are testing against the actual TYPO3 Version and database scheme. diff --git a/Makefile b/Makefile index b8f4ac5..ac3d7d5 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,18 @@ typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" +sourceOrDist=--prefer-dist +ifeq ($(TYPO3_VERSION),~7.6) + sourceOrDist=--prefer-source +endif + .PHONY: install install: clean - COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev --prefer-dist typo3/cms="$(TYPO3_VERSION)" + if [ $(TYPO3_VERSION) = ~7.6 ]; then \ + patch composer.json Tests/InstallPatches/composer.json.patch; \ + fi + + COMPOSER_PROCESS_TIMEOUT=1000 composer require -vv --dev $(sourceOrDist) typo3/cms="$(TYPO3_VERSION)" git checkout composer.json cgl: diff --git a/Tests/Functional/Bootstrap.php b/Tests/Functional/Bootstrap.php new file mode 100644 index 0000000..66b8eba --- /dev/null +++ b/Tests/Functional/Bootstrap.php @@ -0,0 +1,9 @@ +=7.0.0", - "typo3/cms": "~8.7", + "typo3/cms": ">= 7.6.0 < 9.0.0", "ruflin/elastica": "~3.2" }, "require-dev": { diff --git a/ext_emconf.php b/ext_emconf.php index 4d342a5..7c3a087 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,8 +7,8 @@ $EM_CONF[$_EXTKEY] = [ 'clearCacheOnLoad' => 1, 'constraints' => [ 'depends' => [ - 'typo3' => '8.7.0-8.7.99', - 'php' => '7.1.0-7.99.99' + 'typo3' => '7.6.0-8.7.99', + 'php' => '7.0.0-7.2.99' ], 'conflicts' => [], ], @@ -18,7 +18,7 @@ $EM_CONF[$_EXTKEY] = [ ], ], 'state' => 'beta', - 'version' => '1.0.0', + 'version' => '0.0.1', 'author' => 'Daniel Siepmann', 'author_email' => 'coding@daniel-siepmann.de', ]; diff --git a/ext_localconf.php b/ext_localconf.php index a713d0b..d4d7cef 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -37,17 +37,31 @@ call_user_func( ] ); + // Register different concrete implementations, depending on current TYPO3 version. + // This way we can provide working implementations for multiple TYPO3 versions. + $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); + if (\TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \TYPO3\CMS\Core\TypoScript\TypoScriptService::class + ); + } else { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \TYPO3\CMS\Extbase\Service\TypoScriptService::class + ); + } + // API does make use of object manager, therefore use GLOBALS $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); if ($extensionConfiguration === false || !isset($extensionConfiguration['disable.']['elasticsearch']) || $extensionConfiguration['disable.']['elasticsearch'] !== '1' ) { - \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) - ->registerImplementation( - \Codappix\SearchCore\Connection\ConnectionInterface::class, - \Codappix\SearchCore\Connection\Elasticsearch::class - ); + $container->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); } }, $_EXTKEY From 9e80574361b2791d53876045bdb3f26d4bfc517f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 12:18:00 +0100 Subject: [PATCH 142/158] TASK: Provide compatible TypoScriptService for both TYPO3 versions --- Classes/Compatibility/TypoScriptService.php | 31 +++++++++++++++++++ Classes/Compatibility/TypoScriptService76.php | 31 +++++++++++++++++++ .../TypoScriptServiceInterface.php | 30 ++++++++++++++++++ ...entObjectDataProcessorAdapterProcessor.php | 6 ++-- .../Functional/AbstractFunctionalTestCase.php | 5 +++ ...bjectDataProcessorAdapterProcessorTest.php | 11 +++++-- ext_localconf.php | 4 +-- 7 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 Classes/Compatibility/TypoScriptService.php create mode 100644 Classes/Compatibility/TypoScriptService76.php create mode 100644 Classes/Compatibility/TypoScriptServiceInterface.php diff --git a/Classes/Compatibility/TypoScriptService.php b/Classes/Compatibility/TypoScriptService.php new file mode 100644 index 0000000..e5aa788 --- /dev/null +++ b/Classes/Compatibility/TypoScriptService.php @@ -0,0 +1,31 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\TypoScript\TypoScriptService as CoreTypoScriptService; + +/** + * Used since TYPO3 CMS 8.7. + */ +class TypoScriptService extends CoreTypoScriptService implements TypoScriptServiceInterface +{ + +} diff --git a/Classes/Compatibility/TypoScriptService76.php b/Classes/Compatibility/TypoScriptService76.php new file mode 100644 index 0000000..9df82ea --- /dev/null +++ b/Classes/Compatibility/TypoScriptService76.php @@ -0,0 +1,31 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Extbase\Service\TypoScriptService as CoreTypoScriptService; + +/** + * Used before TYPO3 CMS 8.7. + */ +class TypoScriptService76 extends CoreTypoScriptService implements TypoScriptServiceInterface +{ + +} diff --git a/Classes/Compatibility/TypoScriptServiceInterface.php b/Classes/Compatibility/TypoScriptServiceInterface.php new file mode 100644 index 0000000..8e6a9e4 --- /dev/null +++ b/Classes/Compatibility/TypoScriptServiceInterface.php @@ -0,0 +1,30 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Allows to use DI configuration to switch concrete implementation, depending + * on current TYPO3 Version. + */ +interface TypoScriptServiceInterface +{ + public function convertPlainArrayToTypoScriptArray(array $plainArray); +} diff --git a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php index c3b4c32..2b7f553 100644 --- a/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php +++ b/Classes/DataProcessing/ContentObjectDataProcessorAdapterProcessor.php @@ -20,8 +20,8 @@ namespace Codappix\SearchCore\DataProcessing; * 02110-1301, USA. */ +use Codappix\SearchCore\Compatibility\TypoScriptServiceInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Core\TypoScript\TypoScriptService; use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; /** @@ -30,11 +30,11 @@ use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer; class ContentObjectDataProcessorAdapterProcessor implements ProcessorInterface { /** - * @var TypoScriptService + * @var TypoScriptServiceInterface */ protected $typoScriptService; - public function __construct(TypoScriptService $typoScriptService) + public function __construct(TypoScriptServiceInterface $typoScriptService) { $this->typoScriptService = $typoScriptService; } diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 7f808c7..1287477 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -66,4 +66,9 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase { return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']; } + + protected function useLegacyVersion() : bool + { + return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; + } } diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php index 871b205..029e51d 100644 --- a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -20,9 +20,10 @@ namespace Codappix\SearchCore\Tests\Functional\DataProcessing; * 02110-1301, USA. */ +use Codappix\SearchCore\Compatibility\TypoScriptService76; +use Codappix\SearchCore\Compatibility\TypoScriptService; use Codappix\SearchCore\DataProcessing\ContentObjectDataProcessorAdapterProcessor; use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; -use TYPO3\CMS\Extbase\Service\TypoScriptService; use TYPO3\CMS\Frontend\DataProcessing\SplitProcessor; class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalTestCase @@ -44,7 +45,13 @@ class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalT 'new_content' => ['value1', 'value2'], ]; - $subject = new ContentObjectDataProcessorAdapterProcessor(new TypoScriptService); + if ($this->useLegacyVersion()) { + $typoScriptService = new TypoScriptService76(); + } else { + $typoScriptService = new TypoScriptService(); + } + + $subject = new ContentObjectDataProcessorAdapterProcessor($typoScriptService); $processedData = $subject->processData($record, $configuration); $this->assertSame( $expectedData, diff --git a/ext_localconf.php b/ext_localconf.php index d4d7cef..627f33b 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -43,12 +43,12 @@ call_user_func( if (\TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { $container->registerImplementation( \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \TYPO3\CMS\Core\TypoScript\TypoScriptService::class + \Codappix\SearchCore\Compatibility\TypoScriptService::class ); } else { $container->registerImplementation( \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \TYPO3\CMS\Extbase\Service\TypoScriptService::class + \Codappix\SearchCore\Compatibility\TypoScriptService76::class ); } From 6d7199ccf2ae4504ba0a536c519ba9a3d7293b0d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 12:19:30 +0100 Subject: [PATCH 143/158] TASK: Provide already used logger As we use the logger, we should inject it. --- Classes/Connection/Elasticsearch/IndexFactory.php | 15 +++++++++++++++ .../Connection/Elasticsearch/IndexFactoryTest.php | 1 + 2 files changed, 16 insertions(+) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index c56892a..d274db7 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -37,6 +37,21 @@ class IndexFactory implements Singleton */ protected $configuration; + /** + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * Inject log manager to get concrete logger from it. + * + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + /** * @param ConfigurationContainerInterface $configuration */ diff --git a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php index c73fe36..4ede3cb 100644 --- a/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php +++ b/Tests/Unit/Connection/Elasticsearch/IndexFactoryTest.php @@ -38,6 +38,7 @@ class IndexFactoryTest extends AbstractUnitTestCase $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); $this->subject = new IndexFactory($this->configuration); + $this->subject->injectLogger($this->getMockedLogger()); } /** From 0122dd88e47e0f7272f428936e2228fed5de3531 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 12:19:54 +0100 Subject: [PATCH 144/158] TASK: Fix call to non existing method --- Classes/Connection/Elasticsearch/IndexFactory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index d274db7..6493c3c 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -85,7 +85,7 @@ class IndexFactory implements Singleton foreach (['analyzer', 'filter'] as $optionsToExpand) { if (isset($configuration['analysis'][$optionsToExpand])) { foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { - $configuration['analysis'][$optionsToExpand][$key] = $this->prepareOptions($options); + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration($options); } } } From b2a63e9cb028e6ecce4c35de087cca19ba70d61d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 13 Mar 2018 17:28:50 +0100 Subject: [PATCH 145/158] TASK: Make extension compatible with CMS 7.6 --- .travis.yml | 1 + .../ImplementationRegistrationService.php | 56 +++ .../Connection/Elasticsearch/IndexFactory.php | 4 +- Classes/Domain/Index/IndexerFactory.php | 8 +- Classes/Domain/Index/TcaIndexer.php | 51 +-- .../Domain/Index/TcaIndexer/PagesIndexer.php | 23 +- .../Index/TcaIndexer/RelationResolver.php | 2 +- .../Index/TcaIndexer/TcaTableService.php | 59 ++- .../Index/TcaIndexer/TcaTableService76.php | 379 ++++++++++++++++++ .../TcaIndexer/TcaTableServiceInterface.php | 44 ++ Makefile | 2 +- .../Functional/AbstractFunctionalTestCase.php | 4 +- .../Connection/Elasticsearch/FilterTest.php | 2 +- .../Elasticsearch/IndexTcaTableTest.php | 17 +- ...bjectDataProcessorAdapterProcessorTest.php | 2 +- .../ProcessesAllowedTablesTest.php | 14 + Tests/Functional/Indexing/TcaIndexerTest.php | 4 +- Tests/Unit/AbstractUnitTestCase.php | 5 + .../Index/TcaIndexer/TcaTableServiceTest.php | 62 ++- .../Form/Finisher/DataHandlerFinisherTest.php | 2 + ext_localconf.php | 16 +- 21 files changed, 646 insertions(+), 111 deletions(-) create mode 100644 Classes/Compatibility/ImplementationRegistrationService.php create mode 100644 Classes/Domain/Index/TcaIndexer/TcaTableService76.php create mode 100644 Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php diff --git a/.travis.yml b/.travis.yml index 2babbf3..56f42fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ env: - typo3DatabasePassword="" matrix: - TYPO3_VERSION="~7.6" + - TYPO3_VERSION="~8.7" matrix: fast_finish: true diff --git a/Classes/Compatibility/ImplementationRegistrationService.php b/Classes/Compatibility/ImplementationRegistrationService.php new file mode 100644 index 0000000..fa3faa8 --- /dev/null +++ b/Classes/Compatibility/ImplementationRegistrationService.php @@ -0,0 +1,56 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Core\Utility\VersionNumberUtility; +use TYPO3\CMS\Extbase\Object\Container\Container; + +/** + * Register different concrete implementations, depending on current TYPO3 version. + * This way we can provide working implementations for multiple TYPO3 versions. + */ +class ImplementationRegistrationService +{ + public static function registerImplementations() + { + $container = GeneralUtility::makeInstance(Container::class); + if (VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \Codappix\SearchCore\Compatibility\TypoScriptService::class + ); + $container->registerImplementation( + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class, + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService::class + ); + } else { + $container->registerImplementation( + \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, + \Codappix\SearchCore\Compatibility\TypoScriptService76::class + ); + $container->registerImplementation( + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface::class, + \Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76::class + ); + } + } +} diff --git a/Classes/Connection/Elasticsearch/IndexFactory.php b/Classes/Connection/Elasticsearch/IndexFactory.php index 6493c3c..2ef905f 100644 --- a/Classes/Connection/Elasticsearch/IndexFactory.php +++ b/Classes/Connection/Elasticsearch/IndexFactory.php @@ -85,7 +85,9 @@ class IndexFactory implements Singleton foreach (['analyzer', 'filter'] as $optionsToExpand) { if (isset($configuration['analysis'][$optionsToExpand])) { foreach ($configuration['analysis'][$optionsToExpand] as $key => $options) { - $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration($options); + $configuration['analysis'][$optionsToExpand][$key] = $this->prepareAnalyzerConfiguration( + $options + ); } } } diff --git a/Classes/Domain/Index/IndexerFactory.php b/Classes/Domain/Index/IndexerFactory.php index b1a431d..668111d 100644 --- a/Classes/Domain/Index/IndexerFactory.php +++ b/Classes/Domain/Index/IndexerFactory.php @@ -23,7 +23,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Index\IndexerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -81,13 +81,13 @@ class IndexerFactory implements Singleton ) { $indexer = $this->objectManager->get( $indexerClass, - $this->objectManager->get(TcaTableService::class, $identifier), - $this->objectManager->get(TcaTableService::class, 'tt_content') + $this->objectManager->get(TcaTableServiceInterface::class, $identifier), + $this->objectManager->get(TcaTableServiceInterface::class, 'tt_content') ); } elseif (is_subclass_of($indexerClass, TcaIndexer::class) || $indexerClass === TcaIndexer::class) { $indexer = $this->objectManager->get( $indexerClass, - $this->objectManager->get(TcaTableService::class, $identifier) + $this->objectManager->get(TcaTableServiceInterface::class, $identifier) ); } elseif (class_exists($indexerClass) && in_array(IndexerInterface::class, class_implements($indexerClass))) { $indexer = $this->objectManager->get($indexerClass); diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 477bf18..4f96815 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -22,10 +22,7 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; -use TYPO3\CMS\Core\Database\ConnectionPool; -use TYPO3\CMS\Core\Database\Query\QueryBuilder; -use TYPO3\CMS\Core\Utility\GeneralUtility; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; /** * Will index the given table using configuration from TCA. @@ -33,17 +30,17 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; class TcaIndexer extends AbstractIndexer { /** - * @var TcaIndexer\TcaTableService + * @var TcaIndexer\TcaTableServiceInterface */ protected $tcaTableService; /** - * @param TcaIndexer\TcaTableService $tcaTableService + * @param TcaIndexer\TcaTableServiceInterface $tcaTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ public function __construct( - TcaIndexer\TcaTableService $tcaTableService, + TcaIndexer\TcaTableServiceInterface $tcaTableService, ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { @@ -56,13 +53,8 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecords(int $offset, int $limit) { - $records = $this->getQuery() - ->setFirstResult($offset) - ->setMaxResults($limit) - ->execute() - ->fetchAll(); - - if ($records === null) { + $records = $this->tcaTableService->getRecords($offset, $limit); + if ($records === []) { return null; } @@ -79,11 +71,9 @@ class TcaIndexer extends AbstractIndexer */ protected function getRecord(int $identifier) : array { - $query = $this->getQuery(); - $query = $query->andWhere($this->tcaTableService->getTableName() . '.uid = ' . $identifier); - $record = $query->execute()->fetch(); + $record = $this->tcaTableService->getRecord($identifier); - if ($record === false || $record === null) { + if ($record === []) { throw new NoRecordFoundException( 'Record could not be fetched from database: "' . $identifier . '". Perhaps record is not active.', 1484225364 @@ -98,29 +88,4 @@ class TcaIndexer extends AbstractIndexer { return $this->tcaTableService->getTableName(); } - - protected function getQuery(TcaTableService $tcaTableService = null) : QueryBuilder - { - if ($tcaTableService === null) { - $tcaTableService = $this->tcaTableService; - } - $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($tcaTableService->getTableName()); - $where = $tcaTableService->getWhereClause(); - $query = $queryBuilder->select(... $tcaTableService->getFields()) - ->from($tcaTableService->getTableClause()) - ->where($where->getStatement()) - ->setParameters($where->getParameters()); - - foreach ($tcaTableService->getJoins() as $join) { - $query->from($join->getTable()); - $query->andWhere($join->getCondition()); - } - - return $query; - } - - protected function getDatabaseConnection() : ConnectionPool - { - return GeneralUtility::makeInstance(ConnectionPool::class); - } } diff --git a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php index 8882696..7124ca1 100644 --- a/Classes/Domain/Index/TcaIndexer/PagesIndexer.php +++ b/Classes/Domain/Index/TcaIndexer/PagesIndexer.php @@ -23,6 +23,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Domain\Index\TcaIndexer; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; /** * Specific indexer for Pages, will basically add content of page. @@ -30,7 +31,7 @@ use Codappix\SearchCore\Domain\Index\TcaIndexer; class PagesIndexer extends TcaIndexer { /** - * @var TcaTableService + * @var TcaTableServiceInterface */ protected $contentTableService; @@ -41,14 +42,14 @@ class PagesIndexer extends TcaIndexer protected $fileRepository; /** - * @param TcaTableService $tcaTableService - * @param TcaTableService $contentTableService + * @param TcaTableServiceInterface $tcaTableService + * @param TcaTableServiceInterface $contentTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ public function __construct( - TcaTableService $tcaTableService, - TcaTableService $contentTableService, + TcaTableServiceInterface $tcaTableService, + TcaTableServiceInterface $contentTableService, ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { @@ -77,7 +78,17 @@ class PagesIndexer extends TcaIndexer protected function fetchContentForPage(int $uid) : array { - $contentElements = $this->getQuery($this->contentTableService)->execute()->fetchAll(); + if ($this->contentTableService instanceof TcaTableService) { + $contentElements = $this->contentTableService->getQuery() + ->execute()->fetchAll(); + } else { + $contentElements = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( + $this->contentTableService->getFields(), + $this->contentTableService->getTableClause(), + $this->contentTableService->getWhereClause() . + sprintf(' AND %s.pid = %u', $this->contentTableService->getTableName(), $uid) + ); + } if ($contentElements === null) { $this->logger->debug('No content for page ' . $uid); diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 6af330e..ae47e30 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -33,7 +33,7 @@ use TYPO3\CMS\Core\Utility\GeneralUtility; */ class RelationResolver implements Singleton { - public function resolveRelationsForRecord(TcaTableService $service, array &$record) + public function resolveRelationsForRecord(TcaTableServiceInterface $service, array &$record) { foreach (array_keys($record) as $column) { // TODO: Define / configure fields to exclude?! diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 19f6a55..52e19e5 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,11 +21,13 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; -use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; +use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Core\Database\ConnectionPool; +use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\RootlineUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; @@ -33,7 +35,7 @@ use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** * Encapsulate logik related to TCA configuration. */ -class TcaTableService +class TcaTableService implements TcaTableServiceInterface { /** * TCA for current table. @@ -117,9 +119,26 @@ class TcaTableService return $this->tableName; } - /** - * Filter the given records by root line blacklist settings. - */ + public function getRecords(int $offset, int $limit) : array + { + $records = $this->getQuery() + ->setFirstResult($offset) + ->setMaxResults($limit) + ->execute() + ->fetchAll(); + + return $records ?: []; + } + + public function getRecord(int $identifier) : array + { + $query = $this->getQuery(); + $query = $query->andWhere($this->getTableName() . '.uid = ' . $identifier); + $record = $query->execute()->fetch(); + + return $record ?: []; + } + public function filterRecordsByRootLineBlacklist(array &$records) { $records = array_filter( @@ -142,7 +161,7 @@ class TcaTableService } } - public function getWhereClause() : Where + protected function getWhereClause() : Where { $parameters = []; $whereClause = $this->getSystemWhereClause(); @@ -164,7 +183,7 @@ class TcaTableService return new Where($whereClause, $parameters); } - public function getFields() : array + protected function getFields() : array { $fields = array_merge( ['uid','pid'], @@ -187,7 +206,7 @@ class TcaTableService return $fields; } - public function getJoins() : array + protected function getJoins() : array { if ($this->tableName === 'pages') { return []; @@ -202,7 +221,7 @@ class TcaTableService * Generate SQL for TYPO3 as a system, to make sure only available records * are fetched. */ - public function getSystemWhereClause() : string + protected function getSystemWhereClause() : string { $whereClause = '1=1' . BackendUtility::BEenableFields($this->tableName) @@ -345,4 +364,26 @@ class TcaTableService $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') ); } + + public function getQuery() : QueryBuilder + { + $queryBuilder = $this->getDatabaseConnection()->getQueryBuilderForTable($this->getTableName()); + $where = $this->getWhereClause(); + $query = $queryBuilder->select(... $this->getFields()) + ->from($this->getTableClause()) + ->where($where->getStatement()) + ->setParameters($where->getParameters()); + + foreach ($this->getJoins() as $join) { + $query->from($join->getTable()); + $query->andWhere($join->getCondition()); + } + + return $query; + } + + protected function getDatabaseConnection() : ConnectionPool + { + return GeneralUtility::makeInstance(ConnectionPool::class); + } } diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php new file mode 100644 index 0000000..28922a5 --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php @@ -0,0 +1,379 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\IndexingException; +use Codappix\SearchCore\Domain\Index\TcaIndexer\InvalidArgumentException; +use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Core\Utility\RootlineUtility; +use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; + +/** + * Encapsulate logik related to TCA configuration. + */ +class TcaTableService76 implements TcaTableServiceInterface +{ + /** + * TCA for current table. + * !REFERENCE! To save memory. + * @var array + */ + protected $tca; + + /** + * @var string + */ + protected $tableName; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var RelationResolver + */ + protected $relationResolver; + + /** + * @var \TYPO3\CMS\Core\Log\Logger + */ + protected $logger; + + /** + * @var ObjectManagerInterface + */ + protected $objectManager; + + /** + * Inject log manager to get concrete logger from it. + * + * @param \TYPO3\CMS\Core\Log\LogManager $logManager + */ + public function injectLogger(\TYPO3\CMS\Core\Log\LogManager $logManager) + { + $this->logger = $logManager->getLogger(__CLASS__); + } + + /** + * @param ObjectManagerInterface $objectManager + */ + public function injectObjectManager(ObjectManagerInterface $objectManager) + { + $this->objectManager = $objectManager; + } + + /** + * @param string $tableName + * @param ConfigurationContainerInterface $configuration + */ + public function __construct( + $tableName, + RelationResolver $relationResolver, + ConfigurationContainerInterface $configuration + ) { + if (!isset($GLOBALS['TCA'][$tableName])) { + throw new IndexingException( + 'Table "' . $tableName . '" is not configured in TCA.', + IndexingException::CODE_UNKOWN_TCA_TABLE + ); + } + + $this->tableName = $tableName; + $this->tca = &$GLOBALS['TCA'][$this->tableName]; + $this->configuration = $configuration; + $this->relationResolver = $relationResolver; + } + + public function getTableName() : string + { + return $this->tableName; + } + + public function getTableClause() : string + { + if ($this->tableName === 'pages') { + return $this->tableName; + } + + return $this->tableName . ' LEFT JOIN pages on ' . $this->tableName . '.pid = pages.uid'; + } + + public function getRecords(int $offset, int $limit) : array + { + $records = $this->getConnection()->exec_SELECTgetRows( + $this->getFields(), + $this->getTableClause(), + $this->getWhereClause(), + '', + '', + (int) $offset . ',' . (int) $limit + ); + + return $records ?: []; + } + + public function getRecord(int $identifier) : array + { + $record = $this->getConnection()->exec_SELECTgetSingleRow( + $this->getFields(), + $this->getTableClause(), + $this->getWhereClause() + . ' AND ' . $this->getTableName() . '.uid = ' . (int) $identifier + ); + + return $record ?: []; + } + + public function filterRecordsByRootLineBlacklist(array &$records) + { + $records = array_filter( + $records, + function ($record) { + return ! $this->isRecordBlacklistedByRootline($record); + } + ); + } + + public function prepareRecord(array &$record) + { + $this->relationResolver->resolveRelationsForRecord($this, $record); + + if (isset($record['uid']) && !isset($record['search_identifier'])) { + $record['search_identifier'] = $record['uid']; + } + if (isset($record[$this->tca['ctrl']['label']]) && !isset($record['search_title'])) { + $record['search_title'] = $record[$this->tca['ctrl']['label']]; + } + } + + public function getWhereClause() : string + { + $whereClause = '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + . ' AND pages.no_search = 0' + ; + + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } + + $userDefinedWhere = $this->configuration->getIfExists( + 'indexing.' . $this->getTableName() . '.additionalWhereClause' + ); + if (is_string($userDefinedWhere)) { + $whereClause .= ' AND ' . $userDefinedWhere; + } + if ($this->isBlacklistedRootLineConfigured()) { + $whereClause .= ' AND pages.uid NOT IN (' + . implode(',', $this->getBlacklistedRootLine()) + . ')' + . ' AND pages.pid NOT IN (' + . implode(',', $this->getBlacklistedRootLine()) + . ')'; + } + + $this->logger->debug('Generated where clause.', [$this->tableName, $whereClause]); + return $whereClause; + } + + public function getFields() : string + { + $fields = array_merge( + ['uid','pid'], + array_filter( + array_keys($this->tca['columns']), + function ($columnName) { + return !$this->isSystemField($columnName) + && !$this->isUserField($columnName) + && !$this->isPassthroughField($columnName) + ; + } + ) + ); + + foreach ($fields as $key => $field) { + $fields[$key] = $this->tableName . '.' . $field; + } + + $this->logger->debug('Generated fields.', [$this->tableName, $fields]); + return implode(',', $fields); + } + + + /** + * Generate SQL for TYPO3 as a system, to make sure only available records + * are fetched. + */ + protected function getSystemWhereClause() : string + { + $whereClause = '1=1' + . BackendUtility::BEenableFields($this->tableName) + . BackendUtility::deleteClause($this->tableName) + . ' AND pages.no_search = 0' + ; + + if ($this->tableName !== 'pages') { + $whereClause .= BackendUtility::BEenableFields('pages') + . BackendUtility::deleteClause('pages') + ; + } + + return $whereClause; + } + + protected function isSystemField(string $columnName) : bool + { + $systemFields = [ + // Versioning fields, + // https://docs.typo3.org/typo3cms/TCAReference/Reference/Ctrl/Index.html#versioningws + 't3ver_oid', 't3ver_id', 't3ver_label', 't3ver_wsid', + 't3ver_state', 't3ver_stage', 't3ver_count', 't3ver_tstamp', + 't3ver_move_id', 't3ver_swapmode', + $this->tca['ctrl']['transOrigDiffSourceField'], + $this->tca['ctrl']['cruser_id'], + $this->tca['ctrl']['fe_cruser_id'], + $this->tca['ctrl']['fe_crgroup_id'], + $this->tca['ctrl']['languageField'], + $this->tca['ctrl']['origUid'], + ]; + + return in_array($columnName, $systemFields); + } + + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + + protected function isPassthroughField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'passthrough'; + } + + /** + * @throws InvalidArgumentException + */ + public function getColumnConfig(string $columnName) : array + { + if (!isset($this->tca['columns'][$columnName])) { + throw new InvalidArgumentException( + 'Column does not exist.', + InvalidArgumentException::COLUMN_DOES_NOT_EXIST + ); + } + + return $this->tca['columns'][$columnName]['config']; + } + + /** + * Checks whether the given record was blacklisted by root line. + * This can be configured by typoscript as whole root lines can be black listed. + * + * Also further TYPO3 mechanics are taken into account. Does a valid root + * line exist, is page inside a recycler, is inherited start- endtime + * excluded, etc. + */ + protected function isRecordBlacklistedByRootline(array &$record) : bool + { + $pageUid = $record['pid']; + if ($this->tableName === 'pages') { + $pageUid = $record['uid']; + } + + try { + $rootline = $this->objectManager->get(RootlineUtility::class, $pageUid)->get(); + } catch (\RuntimeException $e) { + $this->logger->notice( + sprintf('Could not fetch rootline for page %u, because: %s', $pageUid, $e->getMessage()), + [$record, $e] + ); + return true; + } + + foreach ($rootline as $pageInRootLine) { + // Check configured black list if present. + if ($this->isBlackListedRootLineConfigured() + && in_array($pageInRootLine['uid'], $this->getBlackListedRootLine()) + ) { + $this->logger->info( + sprintf( + 'Record %u is black listed due to configured root line configuration of page %u.', + $record['uid'], + $pageInRootLine['uid'] + ), + [$record, $pageInRootLine] + ); + return true; + } + + if ($pageInRootLine['extendToSubpages'] && ( + ($pageInRootLine['endtime'] > 0 && $pageInRootLine['endtime'] <= time()) + || ($pageInRootLine['starttime'] > 0 && $pageInRootLine['starttime'] >= time()) + )) { + $this->logger->info( + sprintf( + 'Record %u is black listed due to configured timing of parent page %u.', + $record['uid'], + $pageInRootLine['uid'] + ), + [$record, $pageInRootLine] + ); + return true; + } + } + + return false; + } + + /** + * Checks whether any page uids are black listed. + */ + protected function isBlackListedRootLineConfigured() : bool + { + return (bool) $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist'); + } + + /** + * Get the list of black listed root line page uids. + * + * @return array + */ + protected function getBlackListedRootLine() : array + { + return GeneralUtility::intExplode( + ',', + $this->configuration->getIfExists('indexing.' . $this->getTableName() . '.rootLineBlacklist') + ); + } + + protected function getConnection() : \TYPO3\CMS\Core\Database\DatabaseConnection + { + return $GLOBALS['TYPO3_DB']; + } +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php new file mode 100644 index 0000000..0c9cfc4 --- /dev/null +++ b/Classes/Domain/Index/TcaIndexer/TcaTableServiceInterface.php @@ -0,0 +1,44 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +interface TcaTableServiceInterface +{ + public function getTableName() : string; + + public function getTableClause() : string; + + /** + * Filter the given records by root line blacklist settings. + */ + public function filterRecordsByRootLineBlacklist(array &$records); + + public function prepareRecord(array &$record); + + /** + * @throws InvalidArgumentException + */ + public function getColumnConfig(string $columnName) : array; + + public function getRecords(int $offset, int $limit) : array; + + public function getRecord(int $identifier) : array; +} diff --git a/Makefile b/Makefile index ac3d7d5..b1f4af0 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ TYPO3_WEB_DIR := $(current_dir).Build/web TYPO3_PATH_ROOT := $(current_dir).Build/web # Allow different versions on travis TYPO3_VERSION ?= ~8.7 -typo3DatabaseName ?= "searchcore_test2" +typo3DatabaseName ?= "searchcore_test" typo3DatabaseUsername ?= "dev" typo3DatabasePassword ?= "dev" typo3DatabaseHost ?= "127.0.0.1" diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 1287477..3d47af8 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -41,6 +41,8 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase } $this->setUpFrontendRootPage(1, $this->getTypoScriptFilesForFrontendRootPage()); + + // \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); } /** @@ -67,7 +69,7 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase return ['EXT:search_core/Tests/Functional/Fixtures/BasicSetup.ts']; } - protected function useLegacyVersion() : bool + protected function isLegacyVersion() : bool { return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; } diff --git a/Tests/Functional/Connection/Elasticsearch/FilterTest.php b/Tests/Functional/Connection/Elasticsearch/FilterTest.php index 8bbe0f0..894fb3f 100644 --- a/Tests/Functional/Connection/Elasticsearch/FilterTest.php +++ b/Tests/Functional/Connection/Elasticsearch/FilterTest.php @@ -55,7 +55,7 @@ class FilterTest extends AbstractFunctionalTestCase $searchRequest->setFilter(['CType' => 'HTML']); $result = $searchService->search($searchRequest); - $this->assertSame(5, $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); + $this->assertSame(5, (int) $result->getResults()[0]['uid'], 'Did not get the expected result entry.'); $this->assertSame(1, count($result), 'Did not receive the single filtered element.'); } } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index b1e8191..e9eab37 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -220,12 +220,17 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase $response = $this->client->request('typo3content/_search?q=*:*'); $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); - $this->getConnectionPool()->getConnectionForTable('tt_content') - ->update( - 'tt_content', - ['hidden' => true], - ['uid' => 10] - ); + if ($this->isLegacyVersion()) { + $this->getDatabaseConnection() + ->exec_UPDATEquery('tt_content', 'uid = 10', ['hidden' => 1]); + } else { + $this->getConnectionPool()->getConnectionForTable('tt_content') + ->update( + 'tt_content', + ['hidden' => true], + ['uid' => 10] + ); + } \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) ->get(IndexerFactory::class) diff --git a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php index 029e51d..bb997cf 100644 --- a/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php +++ b/Tests/Functional/DataProcessing/ContentObjectDataProcessorAdapterProcessorTest.php @@ -45,7 +45,7 @@ class ContentObjectDataProcessorAdapterProcessorTest extends AbstractFunctionalT 'new_content' => ['value1', 'value2'], ]; - if ($this->useLegacyVersion()) { + if ($this->isLegacyVersion()) { $typoScriptService = new TypoScriptService76(); } else { $typoScriptService = new TypoScriptService(); diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 29ee14c..7271816 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -72,6 +72,13 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1' + && isset($record['pid']) && $record['pid'] === '1' + && isset($record['colPos']) && $record['colPos'] === '1' + ; + } + return isset($record['uid']) && $record['uid'] === 1 && isset($record['pid']) && $record['pid'] === 1 && isset($record['colPos']) && $record['colPos'] === 1 @@ -100,6 +107,13 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '2' + && isset($record['pid']) && $record['pid'] === '1' + && isset($record['header']) && $record['header'] === 'a new record' + ; + } + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index edf1a74..472cc15 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -24,7 +24,7 @@ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\Elasticsearch; use Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; -use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; use Codappix\SearchCore\Tests\Functional\AbstractFunctionalTestCase; use TYPO3\CMS\Extbase\Object\ObjectManager; @@ -47,7 +47,7 @@ class TcaIndexerTest extends AbstractFunctionalTestCase $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); $tableName = 'tt_content'; $tableService = $objectManager->get( - TcaTableService::class, + TcaTableServiceInterface::class, $tableName, $objectManager->get(RelationResolver::class), $objectManager->get(ConfigurationContainerInterface::class) diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index c21209d..fa29d57 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -95,4 +95,9 @@ abstract class AbstractUnitTestCase extends CoreTestCase ->willReturn($translationService); GeneralUtility::setSingletonInstance(ObjectManager::class, $objectManager); } + + protected function isLegacyVersion() : bool + { + return \TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) < 8000000; + } } diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index edad84a..289e915 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -23,8 +23,10 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\DataProcessing\CopyToProcessor; use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; +use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService76; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Core\Database\DatabaseConnection; class TcaTableServiceTest extends AbstractUnitTestCase { @@ -38,16 +40,30 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ protected $configuration; + /** + * @var DatabaseConnection + */ + protected $databaseConnection; + public function setUp() { parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->databaseConnection = $this->getMockBuilder(DatabaseConnection::class)->getMock(); - $this->subject = $this->getMockBuilder(TcaTableService::class) + $className = TcaTableService::class; + if ($this->isLegacyVersion()) { + $className = TcaTableService76::class; + } + $this->subject = $this->getMockBuilder($className) ->disableOriginalConstructor() - ->setMethodsExcept(['getWhereClause', 'injectLogger', 'getTableName']) + ->setMethods(['getConnection', 'getSystemWhereClause']) ->getMock(); + $this->subject->expects($this->any()) + ->method('getConnection') + ->willReturn($this->databaseConnection); + $this->inject($this->subject, 'configuration', $this->configuration); $this->inject($this->subject, 'logger', $this->getMockedLogger()); $this->inject($this->subject, 'tableName', 'table'); @@ -58,6 +74,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function doUsePlainQueryIfNoAdditionalWhereClauseIsDefined() { + $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) @@ -66,7 +83,6 @@ class TcaTableServiceTest extends AbstractUnitTestCase ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause = $this->subject->getWhereClause(); $this->assertSame( '1=1 AND pages.no_search = 0', $whereClause->getStatement() @@ -82,23 +98,27 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function configuredAdditionalWhereClauseIsAdded() { + $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) ->will($this->onConsecutiveCalls('table.field = "someValue"', false)); + $this->subject->expects($this->once()) ->method('getSystemWhereClause') ->will($this->returnValue('1=1 AND pages.no_search = 0')); - $whereClause = $this->subject->getWhereClause(); - $this->assertSame( - '1=1 AND pages.no_search = 0 AND table.field = "someValue"', - $whereClause->getStatement() - ); - $this->assertSame( - [], - $whereClause->getParameters() - ); + $this->subject->getRecord(10); + + // $whereClause = $this->subject->getWhereClause(); + // $this->assertSame( + // '1=1 AND pages.no_search = 0 AND table.field = "someValue"', + // $whereClause->getStatement() + // ); + // $this->assertSame( + // [], + // $whereClause->getParameters() + // ); } /** @@ -137,15 +157,15 @@ class TcaTableServiceTest extends AbstractUnitTestCase ); $this->inject($subject, 'logger', $this->getMockedLogger()); - $this->assertSame( - [ - 'test_table.uid', - 'test_table.pid', - 'test_table.available_column', - ], - $subject->getFields(), - '' - ); + // $this->assertSame( + // [ + // 'test_table.uid', + // 'test_table.pid', + // 'test_table.available_column', + // ], + // $subject->getFields(), + // '' + // ); unset($GLOBALS['TCA']['test_table']); } } diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php index 24c5059..30745cd 100644 --- a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -61,6 +61,7 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase /** * @test + * @requires function \TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher::setOptions * @dataProvider possibleFinisherSetup */ public function validConfiguration(string $action, array $nonCalledActions, $expectedSecondArgument) @@ -98,6 +99,7 @@ class DataHandlerFinisherTest extends AbstractUnitTestCase /** * @test + * @requires function \TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher::setOptions * @dataProvider invalidFinisherSetup */ public function nothingHappensIfUnknownActionIsConfigured(array $options) diff --git a/ext_localconf.php b/ext_localconf.php index 627f33b..a48f8db 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -37,23 +37,11 @@ call_user_func( ] ); - // Register different concrete implementations, depending on current TYPO3 version. - // This way we can provide working implementations for multiple TYPO3 versions. - $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); - if (\TYPO3\CMS\Core\Utility\VersionNumberUtility::convertVersionNumberToInteger(TYPO3_version) >= 8000000) { - $container->registerImplementation( - \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \Codappix\SearchCore\Compatibility\TypoScriptService::class - ); - } else { - $container->registerImplementation( - \Codappix\SearchCore\Compatibility\TypoScriptServiceInterface::class, - \Codappix\SearchCore\Compatibility\TypoScriptService76::class - ); - } + \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); // API does make use of object manager, therefore use GLOBALS $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); + $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); if ($extensionConfiguration === false || !isset($extensionConfiguration['disable.']['elasticsearch']) || $extensionConfiguration['disable.']['elasticsearch'] !== '1' From 20513400de3eae37b921d0b26290b080bfe6b1ea Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 20:43:30 +0100 Subject: [PATCH 146/158] FEATURE: Index page if cache was cleared We use the cache clear hook to index pages whenever the cache was cleared. This makes it possible to cover some cases like changing content on a page. But also if an integrator configures to clear additional pages. This is limited as we can not handle cache tags at the moment. Resolves: #131 --- Classes/Hook/DataHandler.php | 18 ++++ Documentation/source/changelog.rst | 1 + .../20180408-131-respect-page-cache-clear.rst | 14 +++ .../DataHandler/NonAllowedTablesTest.php | 18 +++- .../ProcessesAllowedTablesTest.php | 57 +++++++---- Tests/Unit/Hook/DataHandlerTest.php | 94 +++++++++++++++++++ ext_localconf.php | 3 + 7 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst create mode 100644 Tests/Unit/Hook/DataHandlerTest.php diff --git a/Classes/Hook/DataHandler.php b/Classes/Hook/DataHandler.php index 71fe44b..e0b8e40 100644 --- a/Classes/Hook/DataHandler.php +++ b/Classes/Hook/DataHandler.php @@ -99,6 +99,24 @@ class DataHandler implements Singleton } } + public function clearCachePostProc(array $parameters, CoreDataHandler $dataHandler) + { + $pageUid = 0; + + // If editor uses "small page blizzard" + if (isset($parameters['cacheCmd']) && is_numeric($parameters['cacheCmd'])) { + $pageUid = $parameters['cacheCmd']; + } + // If records were changed + if (isset($parameters['uid_page']) && is_numeric($parameters['uid_page'])) { + $pageUid = $parameters['uid_page']; + } + + if ($pageUid > 0) { + $this->processRecord('pages', (int) $pageUid); + } + } + protected function processRecord(string $table, int $uid) : bool { if (! $this->shouldProcessHookForTable($table)) { diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index 3b121a3..a09d40d 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,5 +5,6 @@ Changelog :maxdepth: 1 :glob: + changelog/20180408-131-respect-page-cache-clear changelog/20180408-introduce-php70-type-hints changelog/20180406-120-facet-configuration diff --git a/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst b/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst new file mode 100644 index 0000000..09800a0 --- /dev/null +++ b/Documentation/source/changelog/20180408-131-respect-page-cache-clear.rst @@ -0,0 +1,14 @@ +Feature 131 "Pages do not get indexed if content has changed" +============================================================= + +Previously we only used DataHandler hooks triggered when processing records. This way we did not +index a page when content has changed. + +We now also use cache clear hooks of DataHandler to index pages whenever their cache get cleared. +This way we also index a page if an integrator configured to clear further pages if content was +changed. + +Still there are limitations. We do not get informed for pages which got cleared due to attached +caches via TypoScript. + +See :issue:`131`. diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index 509f053..11b4806 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -47,7 +47,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest */ public function deletionWillNotBeTriggeredForSysCategories() { - $this->subject->expects($this->exactly(0))->method('delete'); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; @@ -66,7 +70,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillNotBeTriggeredForExistingSysCategory() { - $this->subject->expects($this->exactly(0))->method('update'); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; @@ -85,7 +93,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillNotBeTriggeredForNewSysCategoy() { - $this->subject->expects($this->exactly(0))->method('update'); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 29ee14c..b1b58fd 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -50,6 +50,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('delete') ->with($this->equalTo('tt_content'), $this->equalTo('1')); + $this->subject->expects($this->exactly(1)) + ->method('update') + ->with('pages', $this->callback(function (array $record) { + return isset($record['uid']) && $record['uid'] === 1; + })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; @@ -68,15 +73,23 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillBeTriggeredForExistingTtContent() { - $this->subject->expects($this->exactly(1))->method('update') - ->with( - $this->equalTo('tt_content'), - $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1 - && isset($record['pid']) && $record['pid'] === 1 - && isset($record['colPos']) && $record['colPos'] === 1 - ; - }) + $this->subject->expects($this->exactly(2))->method('update') + ->withConsecutive( + [ + $this->equalTo('tt_content'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 1 + && isset($record['pid']) && $record['pid'] === 1 + && isset($record['colPos']) && $record['colPos'] === 1 + ; + }) + ], + [ + $this->equalTo('pages'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 1; + }) + ] ); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -96,15 +109,23 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillBeTriggeredForNewTtContent() { - $this->subject->expects($this->exactly(1))->method('update') - ->with( - $this->equalTo('tt_content'), - $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 2 - && isset($record['pid']) && $record['pid'] === 1 - && isset($record['header']) && $record['header'] === 'a new record' - ; - }) + $this->subject->expects($this->exactly(2))->method('update') + ->withConsecutive( + [ + $this->equalTo('tt_content'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 2 + && isset($record['pid']) && $record['pid'] === 1 + && isset($record['header']) && $record['header'] === 'a new record' + ; + }) + ], + [ + $this->equalTo('pages'), + $this->callback(function ($record) { + return isset($record['uid']) && $record['uid'] === 1; + }) + ] ); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); diff --git a/Tests/Unit/Hook/DataHandlerTest.php b/Tests/Unit/Hook/DataHandlerTest.php new file mode 100644 index 0000000..ed68993 --- /dev/null +++ b/Tests/Unit/Hook/DataHandlerTest.php @@ -0,0 +1,94 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Service\DataHandler as OwnDataHandler; +use Codappix\SearchCore\Hook\DataHandler; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Core\DataHandling\DataHandler as CoreDataHandler; + +class DataHandlerToProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleCallCombinations + */ + public function fieldsAreCopiedAsConfigured(array $parameters, bool $expectCall) + { + $coreDataHandlerMock = $this->getMockBuilder(CoreDataHandler::class)->getMock(); + $ownDataHandlerMock = $this->getMockBuilder(OwnDataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + $subject = $this->getMockBuilder(DataHandler::class) + ->setConstructorArgs([$ownDataHandlerMock]) + ->setMethods(['getRecord']) + ->getMock(); + + $ownDataHandlerMock->expects($this->any()) + ->method('supportsTable') + ->willReturn(true); + + if ($expectCall) { + $subject->expects($this->once()) + ->method('getRecord') + ->with('pages', 10) + ->willReturn(['uid' => 10]); + $ownDataHandlerMock->expects($this->once()) + ->method('update') + ->with('pages', ['uid' => 10]); + } else { + $subject->expects($this->never()) + ->method('getRecord'); + $ownDataHandlerMock->expects($this->never()) + ->method('update'); + } + + $subject->clearCachePostProc($parameters, $coreDataHandlerMock); + } + + public function getPossibleCallCombinations() : array + { + return [ + 'Editor triggered cache clear of page manual' => [ + 'parameters' => [ + 'cacheCmd' => '10', + ], + 'expectCall' => true, + ], + 'Editor changed records on a page' => [ + 'parameters' => [ + 'uid_page' =>10, + ], + 'expectCall' => true, + ], + 'Something unexpected' => [ + 'parameters' => [], + 'expectCall' => false, + ], + 'Something unexpected' => [ + 'parameters' => [ + 'cacheCmd' => 'something like a tag?!', + ], + 'expectCall' => false, + ], + ]; + } +} diff --git a/ext_localconf.php b/ext_localconf.php index a713d0b..6a729d1 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -15,6 +15,9 @@ call_user_func( ], ], 't3lib/class.t3lib_tcemain.php' => [ + 'clearCachePostProc' => [ + $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class . '->clearCachePostProc', + ], 'processCmdmapClass' => [ $extensionKey => \Codappix\SearchCore\Hook\DataHandler::class, ], From b5b0acdc621aa8bd2a8e8b34d378502a6a6dbef3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 8 Mar 2018 20:20:24 +0100 Subject: [PATCH 147/158] TASK: Provide sys_language_uid Fetch sys_language_uid from database. Until now we did not fetch system related fields from database. As there is now a RemoveProcessor, integrators are able to remove fields. Also providing this field enables basic multi language support. Relates: #25 --- .../Domain/Index/TcaIndexer/TcaTableService.php | 1 - Documentation/source/changelog.rst | 1 + .../20180409-25-provide-sys-language-uid.rst | 16 ++++++++++++++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 9 +++++++-- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 19f6a55..9a3342c 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -231,7 +231,6 @@ class TcaTableService $this->tca['ctrl']['cruser_id'], $this->tca['ctrl']['fe_cruser_id'], $this->tca['ctrl']['fe_crgroup_id'], - $this->tca['ctrl']['languageField'], $this->tca['ctrl']['origUid'], ]; diff --git a/Documentation/source/changelog.rst b/Documentation/source/changelog.rst index a09d40d..4917e1b 100644 --- a/Documentation/source/changelog.rst +++ b/Documentation/source/changelog.rst @@ -5,6 +5,7 @@ Changelog :maxdepth: 1 :glob: + changelog/20180409-25-provide-sys-language-uid changelog/20180408-131-respect-page-cache-clear changelog/20180408-introduce-php70-type-hints changelog/20180406-120-facet-configuration diff --git a/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst b/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst new file mode 100644 index 0000000..3700ce0 --- /dev/null +++ b/Documentation/source/changelog/20180409-25-provide-sys-language-uid.rst @@ -0,0 +1,16 @@ +FEATURE 25 "Respect multiple languages" - Provide sys_language_uid +================================================================== + +Previously we did not fetch ``sys_language_uid`` field from database. This prevented everyone from +working with multiple languages. +By not removing the field it gets indexed and provides a very basic way of implementing multiple +languages. +At least it's now possible to filter search results by current language for now. Still the records +are not "valid" as we do not add overlays for now. + +This is a first step into full multi language support. + +Martin Hummer already has a basic proof of concept, based on :ref:`concepts_dataprocessing` working, +depending on ``sys_language_uid``. + +See :issue:`25`. diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index edad84a..49ffabd 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -108,10 +108,14 @@ class TcaTableServiceTest extends AbstractUnitTestCase { $GLOBALS['TCA']['test_table'] = [ 'ctrl' => [ - 'languageField' => 'sys_language', + 'languageField' => 'sys_language_uid', ], 'columns' => [ - 'sys_language' => [], + 'sys_language_uid' => [ + 'config' => [ + 'type' => 'select', + ], + ], 't3ver_oid' => [], 'available_column' => [ 'config' => [ @@ -141,6 +145,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase [ 'test_table.uid', 'test_table.pid', + 'test_table.sys_language_uid', 'test_table.available_column', ], $subject->getFields(), From 4d2c8f79caf22b13d46a3c7e4827071d2d4f65f1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:08:53 +0100 Subject: [PATCH 148/158] TASK: Use imported interface --- Classes/Domain/Index/TcaIndexer.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer.php b/Classes/Domain/Index/TcaIndexer.php index 4f96815..071b671 100644 --- a/Classes/Domain/Index/TcaIndexer.php +++ b/Classes/Domain/Index/TcaIndexer.php @@ -30,17 +30,17 @@ use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableServiceInterface; class TcaIndexer extends AbstractIndexer { /** - * @var TcaIndexer\TcaTableServiceInterface + * @var TcaTableServiceInterface */ protected $tcaTableService; /** - * @param TcaIndexer\TcaTableServiceInterface $tcaTableService + * @param TcaTableServiceInterface $tcaTableService * @param ConnectionInterface $connection * @param ConfigurationContainerInterface $configuration */ public function __construct( - TcaIndexer\TcaTableServiceInterface $tcaTableService, + TcaTableServiceInterface $tcaTableService, ConnectionInterface $connection, ConfigurationContainerInterface $configuration ) { From b3390c66dbdb19e4ffbbc822f04fd636617c41f3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:23:04 +0100 Subject: [PATCH 149/158] TASK: Keep old logic to make changes smaller --- ext_localconf.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext_localconf.php b/ext_localconf.php index 2e26cd2..658d683 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -44,15 +44,15 @@ call_user_func( // API does make use of object manager, therefore use GLOBALS $extensionConfiguration = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][$extensionKey]); - $container = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class); if ($extensionConfiguration === false || !isset($extensionConfiguration['disable.']['elasticsearch']) || $extensionConfiguration['disable.']['elasticsearch'] !== '1' ) { - $container->registerImplementation( - \Codappix\SearchCore\Connection\ConnectionInterface::class, - \Codappix\SearchCore\Connection\Elasticsearch::class - ); + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\TYPO3\CMS\Extbase\Object\Container\Container::class) + ->registerImplementation( + \Codappix\SearchCore\Connection\ConnectionInterface::class, + \Codappix\SearchCore\Connection\Elasticsearch::class + ); } }, $_EXTKEY From ec8362b93424fc8ab62996e336eab6a7ad6d7586 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:23:29 +0100 Subject: [PATCH 150/158] TASK: Remove unused comment --- Tests/Functional/AbstractFunctionalTestCase.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tests/Functional/AbstractFunctionalTestCase.php b/Tests/Functional/AbstractFunctionalTestCase.php index 3d47af8..d015457 100644 --- a/Tests/Functional/AbstractFunctionalTestCase.php +++ b/Tests/Functional/AbstractFunctionalTestCase.php @@ -41,8 +41,6 @@ abstract class AbstractFunctionalTestCase extends CoreTestCase } $this->setUpFrontendRootPage(1, $this->getTypoScriptFilesForFrontendRootPage()); - - // \Codappix\SearchCore\Compatibility\ImplementationRegistrationService::registerImplementations(); } /** From 56bfc4f75ad1b10056314218c75050b3fb6bb9ed Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 14 Mar 2018 20:23:40 +0100 Subject: [PATCH 151/158] TASK: Fix merge conflict issue --- .../Hooks/DataHandler/ProcessesAllowedTablesTest.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index dff5f3b..279a113 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -73,7 +73,6 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest */ public function updateWillBeTriggeredForExistingTtContent() { -<<<<<<< HEAD $this->subject->expects($this->exactly(2))->method('update') ->withConsecutive( [ @@ -98,7 +97,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest return isset($record['uid']) && $record['uid'] === 1; }) ] - }); + ); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); $tce->stripslashes_values = 0; From 5a0a4931e4f6c6d8d528b99c14ba218d77e0797b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 08:07:28 +0100 Subject: [PATCH 152/158] TASK: Make tests compatible with TYPO3 CMS 7.6 As there is no doctrine, we receive values as string, also for uid. --- .../Hooks/DataHandler/NonAllowedTablesTest.php | 18 +++++++++++++++--- .../DataHandler/ProcessesAllowedTablesTest.php | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php index 11b4806..a9cbaae 100644 --- a/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/NonAllowedTablesTest.php @@ -50,7 +50,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -73,7 +77,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -96,7 +104,11 @@ class NonAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 279a113..5a571d3 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -53,7 +53,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest $this->subject->expects($this->exactly(1)) ->method('update') ->with('pages', $this->callback(function (array $record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } })); $tce = GeneralUtility::makeInstance(Typo3DataHandler::class); @@ -94,7 +98,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest [ $this->equalTo('pages'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } }) ] ); @@ -137,7 +145,11 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest [ $this->equalTo('pages'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === 1; + if ($this->isLegacyVersion()) { + return isset($record['uid']) && $record['uid'] === '1'; + } else { + return isset($record['uid']) && $record['uid'] === 1; + } }) ] ); From 403fd47df096ebabab6b8bd7fa42d4f0dea6d5c1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 11:29:29 +0100 Subject: [PATCH 153/158] TASK: Add necessary phpdoc for extbase As extbase does not reflect PHP source but phpdoc, we have to define the type to use for mapping as phpdoc. --- Classes/Domain/Model/SearchRequest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 22bd697..3c88871 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -87,6 +87,9 @@ class SearchRequest implements SearchRequestInterface return $this->query; } + /** + * @param array $filter + */ public function setFilter(array $filter) { $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); From f3e8dacd4e0ceb405dac3caa9b8f252ebee767c8 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 13:59:47 +0100 Subject: [PATCH 154/158] TASK: Keep language information for all TYPO3 Versions Keep code consistent. Fetch language field for both TYPO3 versions. To make sure we do not mess up, add test case. --- .../Index/TcaIndexer/TcaTableService76.php | 1 - Tests/Functional/Indexing/TcaIndexerTest.php | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php index 28922a5..f52027a 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService76.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService76.php @@ -257,7 +257,6 @@ class TcaTableService76 implements TcaTableServiceInterface $this->tca['ctrl']['cruser_id'], $this->tca['ctrl']['fe_cruser_id'], $this->tca['ctrl']['fe_crgroup_id'], - $this->tca['ctrl']['languageField'], $this->tca['ctrl']['origUid'], ]; diff --git a/Tests/Functional/Indexing/TcaIndexerTest.php b/Tests/Functional/Indexing/TcaIndexerTest.php index 472cc15..d7ecc3a 100644 --- a/Tests/Functional/Indexing/TcaIndexerTest.php +++ b/Tests/Functional/Indexing/TcaIndexerTest.php @@ -77,4 +77,40 @@ class TcaIndexerTest extends AbstractFunctionalTestCase $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); } + + /** + * @test + */ + public function sysLanguageIsKept() + { + $this->importDataSet('Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml'); + $objectManager = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class); + $tableName = 'tt_content'; + $tableService = $objectManager->get( + TcaTableServiceInterface::class, + $tableName, + $objectManager->get(RelationResolver::class), + $objectManager->get(ConfigurationContainerInterface::class) + ); + + $connection = $this->getMockBuilder(Elasticsearch::class) + ->setMethods(['addDocuments']) + ->disableOriginalConstructor() + ->getMock(); + + $connection->expects($this->once()) + ->method('addDocuments') + ->with( + $this->stringContains('tt_content'), + $this->callback(function ($documents) { + if ($this->isLegacyVersion()) { + return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === '2'; + } else { + return isset($documents[0]['sys_language_uid']) && $documents[0]['sys_language_uid'] === 2; + } + }) + ); + + $objectManager->get(TcaIndexer::class, $tableService, $connection)->indexAllDocuments(); + } } From dea028ed0f862b968a0a57db00c2c1921aeed8f7 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:00:45 +0100 Subject: [PATCH 155/158] TASK: Add missing fixture --- .../TcaIndexer/KeepSysLanguageUid.xml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml diff --git a/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml new file mode 100644 index 0000000..913ec67 --- /dev/null +++ b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml @@ -0,0 +1,29 @@ + + + + 2 + 1 + Content here will be indexed + + + + 1 + 1 + 1480686370 + 1480686370 + 0 + 72 + 2 + header +
indexed content element
+ this is the content of header content element that should get indexed + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + From 22e097ca851bc086e8475fd2afad2284630888e7 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:00:53 +0100 Subject: [PATCH 156/158] TASK: Do not comment out code for non working tests Mark tests as non working for now. --- .../Index/TcaIndexer/TcaTableServiceTest.php | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 70174ec..f470613 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -74,7 +74,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function doUsePlainQueryIfNoAdditionalWhereClauseIsDefined() { - $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); + $this->markTestIncomplete('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) @@ -98,7 +98,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function configuredAdditionalWhereClauseIsAdded() { - $this->markTestSkipped('We have to migrate this test for TYPO3 CMS 8.x'); + $this->markTestIncomplete('We have to migrate this test for TYPO3 CMS 8.x'); $this->configuration->expects($this->exactly(2)) ->method('getIfExists') ->withConsecutive(['indexing.table.additionalWhereClause'], ['indexing.table.rootLineBlacklist']) @@ -110,15 +110,15 @@ class TcaTableServiceTest extends AbstractUnitTestCase $this->subject->getRecord(10); - // $whereClause = $this->subject->getWhereClause(); - // $this->assertSame( - // '1=1 AND pages.no_search = 0 AND table.field = "someValue"', - // $whereClause->getStatement() - // ); - // $this->assertSame( - // [], - // $whereClause->getParameters() - // ); + $whereClause = $this->subject->getWhereClause(); + $this->assertSame( + '1=1 AND pages.no_search = 0 AND table.field = "someValue"', + $whereClause->getStatement() + ); + $this->assertSame( + [], + $whereClause->getParameters() + ); } /** @@ -126,6 +126,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase */ public function allConfiguredAndAllowedTcaColumnsAreReturnedAsFields() { + $this->markTestIncomplete('We have to migrate this test'); $GLOBALS['TCA']['test_table'] = [ 'ctrl' => [ 'languageField' => 'sys_language_uid', @@ -161,16 +162,16 @@ class TcaTableServiceTest extends AbstractUnitTestCase ); $this->inject($subject, 'logger', $this->getMockedLogger()); - // $this->assertSame( - // [ - // 'test_table.uid', - // 'test_table.pid', - // 'test_table.sys_language_uid', - // 'test_table.available_column', - // ], - // $subject->getFields(), - // '' - // ); + $this->assertSame( + [ + 'test_table.uid', + 'test_table.pid', + 'test_table.sys_language_uid', + 'test_table.available_column', + ], + $subject->getFields(), + '' + ); unset($GLOBALS['TCA']['test_table']); } } From 2b78ffbe9668d1f5c8e66ae8927be8420818dd71 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:42:41 +0100 Subject: [PATCH 157/158] BUGFIX: Workaround doctrine limitation Doctrine will not implode our array, so we have to do --- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index a0d5851..ee0a2d8 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -174,7 +174,7 @@ class TcaTableService implements TcaTableServiceInterface } if ($this->isBlackListedRootLineConfigured()) { - $parameters[':blacklistedRootLine'] = $this->getBlackListedRootLine(); + $parameters[':blacklistedRootLine'] = implode(',', $this->getBlackListedRootLine()); $whereClause .= ' AND pages.uid NOT IN (:blacklistedRootLine)' . ' AND pages.pid NOT IN (:blacklistedRootLine)'; } From 5c8cfa9cca65192bf3d367dfdcf2500710aae16e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 15 Mar 2018 14:43:00 +0100 Subject: [PATCH 158/158] TASK: Remove unnecessary fixture --- .../Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml index 913ec67..f17051c 100644 --- a/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml +++ b/Tests/Functional/Fixtures/Indexing/TcaIndexer/KeepSysLanguageUid.xml @@ -1,11 +1,5 @@ - - 2 - 1 - Content here will be indexed - - 1 1