From fafa919f37155aa1a679a9154bcc07fea9c6a99f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 16 Sep 2017 20:50:03 +0200 Subject: [PATCH 01/11] WIP|FEATURE: Basic hardcoded implementation --- Classes/DataProcessing/GeoPointProcessing.php | 34 +++++++++++++++++++ Classes/DataProcessing/ProcessorInterface.php | 34 +++++++++++++++++++ .../Index/TcaIndexer/TcaTableService.php | 20 ++++++++++- Classes/Domain/Model/SearchRequest.php | 17 +++++++++- Classes/Domain/Search/QueryFactory.php | 34 +++++++++++++++---- 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 Classes/DataProcessing/GeoPointProcessing.php create mode 100644 Classes/DataProcessing/ProcessorInterface.php diff --git a/Classes/DataProcessing/GeoPointProcessing.php b/Classes/DataProcessing/GeoPointProcessing.php new file mode 100644 index 0000000..cbf6589 --- /dev/null +++ b/Classes/DataProcessing/GeoPointProcessing.php @@ -0,0 +1,34 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class GeoPointProcessing implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + $record[$configuration['to']] = [ + 'lat' => (float) $record[$configuration['fields']['lat']], + 'lon' => (float) $record[$configuration['fields']['lon']], + ]; + + return $record; + } +} diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php new file mode 100644 index 0000000..4dc0794 --- /dev/null +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -0,0 +1,34 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * All DataProcessing Processors should implement this interface, otherwise they + * will not be executed. + */ +interface ProcessorInterface +{ + /** + * Processes the given record. + * Also retrieves the configuration for this processor instance. + */ + public function processRecord(array $record, array $configuration) : array; +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..6b9024f 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -146,6 +147,17 @@ class TcaTableService { $this->relationResolver->resolveRelationsForRecord($this, $record); + try { + foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { + $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidConfigurationArgumentException $e) { + // Nothing to do. + } + if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } @@ -181,7 +193,7 @@ class TcaTableService array_filter( array_keys($this->tca['columns']), function ($columnName) { - return !$this->isSystemField($columnName); + return !$this->isSystemField($columnName) && !$this->isUserField($columnName); } ) ); @@ -249,6 +261,12 @@ class TcaTableService return in_array($columnName, $systemFields); } + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + /** * @param string $columnName * @return array diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 39e477c..fd76062 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,22 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter(array_map('strval', $filter)); + $this->filter = array_filter($filter); + + $this->mapGeoDistanceFilter(); + } + + public function mapGeoDistanceFilter() + { + if (isset($this->filter['geo_distance'])) { + foreach (array_keys($this->filter['geo_distance']) as $config) { + if (strpos($config, '_') !== false) { + $newKey = str_replace('_', '.', $config); + $this->filter['geo_distance'][$newKey] = $this->filter['geo_distance'][$config]; + unset($this->filter['geo_distance'][$config]); + } + } + } } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9455f80..9f2f68a 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -164,6 +164,17 @@ class QueryFactory */ protected function addFactorBoost(array &$query) { + $query['sort'] = [ + '_geo_distance' => [ + 'location' => [ + 'lat' => 51.2014392, + 'lon' => 6.4302962, + ], + 'order' => 'asc', + 'unit' => 'km', + 'distance_type' => 'plane', + ] + ]; try { $query['query'] = [ 'function_score' => [ @@ -186,24 +197,33 @@ class QueryFactory return; } - $terms = []; + $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $terms[] = [ - 'term' => [ - $name => $value, - ], - ]; + $filter[] = $this->buildFilter($name, $value); } $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ - 'filter' => $terms, + 'filter' => $filter, ], ], ]); } + protected function buildFilter(string $name, $value) : array + { + if ($name === 'geo_distance') { + return [$name => $value]; + } + + return [ + 'term' => [ + $name => $value, + ], + ]; + } + /** * @param SearchRequestInterface $searchRequest * @param array $query From e1764dca133fb9c10ee7f98d045163dbe83bcc67 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 20 Oct 2017 16:36:26 +0200 Subject: [PATCH 02/11] FEATURE: Add GeoPoint Processor Document data processors. Add test for new data processor. --- Classes/DataProcessing/CopyToProcessor.php | 6 +- ...ntProcessing.php => GeoPointProcessor.php} | 31 ++++- .../dataProcessing/CopyToProcessor.rst | 23 ++++ .../dataProcessing/GeoPointProcessor.rst | 27 +++++ .../source/configuration/indexing.rst | 8 +- .../DataProcessing/GeoPointProcessorTest.php | 113 ++++++++++++++++++ 6 files changed, 198 insertions(+), 10 deletions(-) rename Classes/DataProcessing/{GeoPointProcessing.php => GeoPointProcessor.php} (53%) create mode 100644 Documentation/source/configuration/dataProcessing/CopyToProcessor.rst create mode 100644 Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst create mode 100644 Tests/Unit/DataProcessing/GeoPointProcessorTest.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index a252a9a..eea555f 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class CopyToProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) + public function processRecord(array $record, array $configuration) : array { $all = []; @@ -36,10 +36,6 @@ class CopyToProcessor implements ProcessorInterface return $record; } - /** - * @param array &$target - * @param array $from - */ protected function addArray(array &$target, array $from) { foreach ($from as $value) { diff --git a/Classes/DataProcessing/GeoPointProcessing.php b/Classes/DataProcessing/GeoPointProcessor.php similarity index 53% rename from Classes/DataProcessing/GeoPointProcessing.php rename to Classes/DataProcessing/GeoPointProcessor.php index cbf6589..f9256b3 100644 --- a/Classes/DataProcessing/GeoPointProcessing.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -20,15 +20,40 @@ namespace Codappix\SearchCore\DataProcessing; * 02110-1301, USA. */ -class GeoPointProcessing implements ProcessorInterface +/** + * Adds a new fields, ready to use as GeoPoint field for Elasticsearch. + */ +class GeoPointProcessor implements ProcessorInterface { public function processRecord(array $record, array $configuration) : array { + if (! $this->canApply($record, $configuration)) { + return $record; + } + $record[$configuration['to']] = [ - 'lat' => (float) $record[$configuration['fields']['lat']], - 'lon' => (float) $record[$configuration['fields']['lon']], + 'lat' => (float) $record[$configuration['lat']], + 'lon' => (float) $record[$configuration['lon']], ]; return $record; } + + protected function canApply(array $record, array $configuration) : bool + { + if (!isset($record[$configuration['lat']]) + || !is_numeric($record[$configuration['lat']]) + || trim($record[$configuration['lat']]) === '' + ) { + return false; + } + if (!isset($record[$configuration['lon']]) + || !is_numeric($record[$configuration['lon']]) + || trim($record[$configuration['lon']]) === '' + ) { + return false; + } + + return true; + } } diff --git a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst new file mode 100644 index 0000000..6d83d70 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\CopyToProcessor`` +====================================================== + +Will copy contents of fields to other fields. + +Possible Options: + +``to`` + Defines the field to copy the values into. All values not false will be copied at the moment. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = all + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = spellcheck + } + } + diff --git a/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst new file mode 100644 index 0000000..a19928f --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst @@ -0,0 +1,27 @@ +``Codappix\SearchCore\DataProcessing\GeoPointProcessor`` +======================================================== + +Will create a new field, ready to use as GeoPoint field for Elasticsearch. + +Possible Options: + +``to`` + Defines the field to create as GeoPoint. +``lat`` + Defines the field containing the latitude. +``lon`` + Defines the field containing the longitude. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor + 1 { + to = location + lat = lat + lon = lng + } + } + +The above example will create a new field ``location`` as GeoPoint with latitude fetched from field +``lat`` and longitude fetched from field ``lng``. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 24c037f..413da3d 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -175,8 +175,12 @@ dataProcessing The following Processor are available: - ``Codappix\SearchCore\DataProcessing\CopyToProcessor`` - Will copy contents of fields to other fields + .. toctree:: + :maxdepth: 1 + :glob: + + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor The following Processor are planned: diff --git a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php new file mode 100644 index 0000000..99b8eb3 --- /dev/null +++ b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\GeoPointProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class GeoPointProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new GeoPointProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Create new field with existing lat and lng' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + 'location' => [ + 'lat' => 23.232, + 'lon' => 45.43, + ], + ], + ], + 'Do not create new field due to missing configuration' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + ], + 'Do not create new field due to missing lat and lon' => [ + 'record' => [ + 'lat' => '', + 'lng' => '', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => '', + 'lng' => '', + ], + ], + 'Do not create new field due to invalid lat and lon' => [ + 'record' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + ], + ]; + } +} From 636ef78a143e34461d8c43fb74e2ccefbea32598 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 11:23:28 +0200 Subject: [PATCH 03/11] WIP|TASK: Remove sort and add filter As we filter for distance and do not sort. --- Classes/Domain/Model/SearchRequest.php | 15 ---- Classes/Domain/Search/QueryFactory.php | 98 +++++++++----------------- 2 files changed, 35 insertions(+), 78 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index fd76062..1e3e652 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -93,21 +93,6 @@ class SearchRequest implements SearchRequestInterface public function setFilter(array $filter) { $this->filter = array_filter($filter); - - $this->mapGeoDistanceFilter(); - } - - public function mapGeoDistanceFilter() - { - if (isset($this->filter['geo_distance'])) { - foreach (array_keys($this->filter['geo_distance']) as $config) { - if (strpos($config, '_') !== false) { - $newKey = str_replace('_', '.', $config); - $this->filter['geo_distance'][$newKey] = $this->filter['geo_distance'][$config]; - unset($this->filter['geo_distance'][$config]); - } - } - } } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9f2f68a..6511cdd 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -39,10 +39,6 @@ class QueryFactory */ protected $configuration; - /** - * @param \TYPO3\CMS\Core\Log\LogManager $logManager - * @param ConfigurationContainerInterface $configuration - */ public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, ConfigurationContainerInterface $configuration @@ -55,22 +51,13 @@ class QueryFactory * TODO: This is not in scope Elasticsearch, therefore it should not return * \Elastica\Query, but decide to use a more specific QueryFactory like * ElasticaQueryFactory, once the second query is added? - * - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query */ - public function create(SearchRequestInterface $searchRequest) + public function create(SearchRequestInterface $searchRequest) : \Elastica\Query { return $this->createElasticaQuery($searchRequest); } - /** - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query - */ - protected function createElasticaQuery(SearchRequestInterface $searchRequest) + protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query { $query = []; $this->addSize($searchRequest, $query); @@ -83,14 +70,12 @@ class QueryFactory // Better approach would be something like DQL to generate query and build result in the end. $this->addFactorBoost($query); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($query, '$query', 8, false);die; + $this->logger->debug('Generated elasticsearch query.', [$query]); return new \Elastica\Query($query); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSize(SearchRequestInterface $searchRequest, array &$query) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -99,10 +84,6 @@ class QueryFactory ]); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { if (trim($searchRequest->getSearchTerm()) === '') { @@ -125,10 +106,6 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { try { @@ -159,22 +136,8 @@ class QueryFactory ]); } - /** - * @param array $query - */ protected function addFactorBoost(array &$query) { - $query['sort'] = [ - '_geo_distance' => [ - 'location' => [ - 'lat' => 51.2014392, - 'lon' => 6.4302962, - ], - 'order' => 'asc', - 'unit' => 'km', - 'distance_type' => 'plane', - ] - ]; try { $query['query'] = [ 'function_score' => [ @@ -187,10 +150,6 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { @@ -199,7 +158,11 @@ class QueryFactory $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $filter[] = $this->buildFilter($name, $value); + $filter[] = $this->buildFilter( + $name, + $value, + $this->configuration->getIfExists('searching.filter.' . $name) ?: [] + ); } $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -211,23 +174,6 @@ class QueryFactory ]); } - protected function buildFilter(string $name, $value) : array - { - if ($name === 'geo_distance') { - return [$name => $value]; - } - - return [ - 'term' => [ - $name => $value, - ], - ]; - } - - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { @@ -242,4 +188,30 @@ class QueryFactory ]); } } + + protected function buildFilter(string $name, $value, array $config) : array + { + if ($config === []) { + return [ + 'term' => [ + $name => $value, + ], + ]; + } + + $filter = []; + + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($name, '$name', 8, false); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($config, '$config', 8, false);die; + + if (isset($config['fields'])) { + foreach ($config['fields'] as $elasticField => $inputField) { + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false);die; + $filter[$elasticField] = $value[$inputField]; + } + } + + return [$config['field'] => $filter]; + } } From 8d343ee97fd178db5f549bc35345e8ab63bcefd5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 17:34:48 +0200 Subject: [PATCH 04/11] FEATURE: Finish filter configuration for geo_search --- Classes/Domain/Search/QueryFactory.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 6511cdd..2042775 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -70,8 +70,6 @@ class QueryFactory // Better approach would be something like DQL to generate query and build result in the end. $this->addFactorBoost($query); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($query, '$query', 8, false);die; - $this->logger->debug('Generated elasticsearch query.', [$query]); return new \Elastica\Query($query); } @@ -161,7 +159,7 @@ class QueryFactory $filter[] = $this->buildFilter( $name, $value, - $this->configuration->getIfExists('searching.filter.' . $name) ?: [] + $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: [] ); } @@ -201,13 +199,8 @@ class QueryFactory $filter = []; - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($name, '$name', 8, false); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($config, '$config', 8, false);die; - if (isset($config['fields'])) { foreach ($config['fields'] as $elasticField => $inputField) { - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false);die; $filter[$elasticField] = $value[$inputField]; } } From b1f81c0d3be781fbdc6d5fb0050b3dcd4a60a21d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 17:51:04 +0200 Subject: [PATCH 05/11] WIP|FEATURE: Add field and sorting Sort result by distance and provide distance to result items. --- Classes/Domain/Search/QueryFactory.php | 70 ++++++++++++++++++++------ 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 2042775..7fa3cd5 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -65,6 +65,8 @@ class QueryFactory $this->addBoosts($searchRequest, $query); $this->addFilter($searchRequest, $query); $this->addFacets($searchRequest, $query); + $this->addFields($searchRequest, $query); + $this->addSort($searchRequest, $query); // Use last, as it might change structure of query. // Better approach would be something like DQL to generate query and build result in the end. @@ -148,6 +150,44 @@ class QueryFactory } } + protected function addFields(SearchRequestInterface $searchRequest, array &$query) + { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => [ '_source' ], + ]); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'script_fields' => [ + 'distance' => [ + 'script' => [ + 'params' => [ + 'lat' => 51.168098, + 'lon' => 6.381384, + ], + 'lang' => 'painless', + 'inline' => 'doc["location"].arcDistance(params.lat,params.lon) * 0.001', + ], + ], + ], + ]); + } + + protected function addSort(SearchRequestInterface $searchRequest, array &$query) + { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'sort' => [ + '_geo_distance' => [ + 'location' => [ + 'lat' => 51.168098, + 'lon' => 6.381384, + ], + 'order' => 'asc', + 'unit' => 'km', + 'distance_type' => 'plane', + ], + ], + ]); + } + protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { @@ -172,21 +212,6 @@ class QueryFactory ]); } - protected function addFacets(SearchRequestInterface $searchRequest, array &$query) - { - foreach ($searchRequest->getFacets() as $facet) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'aggs' => [ - $facet->getIdentifier() => [ - 'terms' => [ - 'field' => $facet->getField(), - ], - ], - ], - ]); - } - } - protected function buildFilter(string $name, $value, array $config) : array { if ($config === []) { @@ -207,4 +232,19 @@ class QueryFactory return [$config['field'] => $filter]; } + + protected function addFacets(SearchRequestInterface $searchRequest, array &$query) + { + foreach ($searchRequest->getFacets() as $facet) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'aggs' => [ + $facet->getIdentifier() => [ + 'terms' => [ + 'field' => $facet->getField(), + ], + ], + ], + ]); + } + } } From 07a4fec622795ed959b4056c43944ad2ff01647d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 23 Oct 2017 16:35:38 +0200 Subject: [PATCH 06/11] WIP|FEATURE: Allow fields and sorting to be configurable --- Classes/Domain/Model/SearchRequest.php | 3 +- Classes/Domain/Search/QueryFactory.php | 67 ++++++++++++++------------ 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 1e3e652..d751ce8 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,8 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter($filter); + $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); + $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7fa3cd5..56d4dd3 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; +use TYPO3\CMS\Fluid\View\StandaloneView; class QueryFactory { @@ -152,40 +154,28 @@ class QueryFactory protected function addFields(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'stored_fields' => [ '_source' ], - ]); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'script_fields' => [ - 'distance' => [ - 'script' => [ - 'params' => [ - 'lat' => 51.168098, - 'lon' => 6.381384, - ], - 'lang' => 'painless', - 'inline' => 'doc["location"].arcDistance(params.lat,params.lon) * 0.001', - ], - ], - ], - ]); + try { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), + ]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } + + try { + $scriptFields = $this->configuration->get('searching.fields.script_fields'); + $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } } protected function addSort(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'sort' => [ - '_geo_distance' => [ - 'location' => [ - 'lat' => 51.168098, - 'lon' => 6.381384, - ], - 'order' => 'asc', - 'unit' => 'km', - 'distance_type' => 'plane', - ], - ], - ]); + $sorting = $this->configuration->getIfExists('searching.sort') ?: []; + $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -247,4 +237,21 @@ class QueryFactory ]); } } + + protected function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be end as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } } From bf91c4a5ba2005bf78d5497661e001f47e26af4a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 12:25:25 +0100 Subject: [PATCH 07/11] TASK: Allow fields and sorting to contain a condition This way integrators can configure when the sorting and fields should be added. --- Classes/Domain/Search/QueryFactory.php | 25 ++- .../source/configuration/connections.rst | 18 +- .../source/configuration/indexing.rst | 194 +++++++++--------- .../source/configuration/searching.rst | 192 ++++++++++++----- 4 files changed, 266 insertions(+), 163 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 56d4dd3..d5f3669 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -165,7 +165,10 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + $scriptFields = $this->filterByCondition($scriptFields); + if ($scriptFields !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } } catch (InvalidArgumentException $e) { // Nothing configured } @@ -175,7 +178,10 @@ class QueryFactory { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + $sorting = $this->filterByCondition($sorting); + if ($sorting !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + } } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -254,4 +260,19 @@ class QueryFactory return $array; } + + protected function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; + }); + + foreach ($entries as $key => $entry) { + if (array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } } diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst index 841d878..5819730 100644 --- a/Documentation/source/configuration/connections.rst +++ b/Documentation/source/configuration/connections.rst @@ -29,27 +29,27 @@ The following settings are available. For each setting its documented which conn ``host`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. +The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 +installation. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost .. _port: ``port`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. +The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 413da3d..93eff19 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -29,18 +29,18 @@ The following settings are available. For each setting its documented which inde rootLineBlacklist ----------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. +Defines a blacklist of page uids. Records below any of these pages, or subpages, are not +indexed. This allows you to define areas that should not be indexed. +The page attribute *No Search* is also taken into account to prevent indexing records from only one +page without recursion. - Contains a comma separated list of page uids. Spaces are trimmed. +Contains a comma separated list of page uids. Spaces are trimmed. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 Also it's possible to define some behaviour for the different document types. In context of TYPO3 tables are used as document types 1:1. It's possible to configure different tables. The following @@ -51,156 +51,156 @@ options are available: additionalWhereClause --------------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. +Add additional SQL to where clauses to determine indexable records from the table. This way you +can exclude specific records like ``tt_content`` records with specific ``CType`` values or +something else. E.g. you can add a new field to the table to exclude records from indexing. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - .. attention:: +.. attention:: - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. .. _abstractFields: abstractFields -------------- - Used by: :ref:`PagesIndexer`. +Used by: :ref:`PagesIndexer`. - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. +Define which field should be used to provide the auto generated field "search_abstract". +The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also +possible. - Example:: +Example:: - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - Default:: +Default:: - abstract, description, bodytext + abstract, description, bodytext .. _mapping: mapping ------- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. +Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html +You are able to define the mapping for each property / columns. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword } + } - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. +The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This +makes building a facet possible. .. _index: index ----- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html +Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding } + } - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 } } } + } - ``char_filter`` and ``filter`` are a comma separated list of options. +``char_filter`` and ``filter`` are a comma separated list of options. .. _dataProcessing: dataProcessing -------------- - Used by: All connections while indexing. +Used by: All connections while indexing. - Configure modifications on each document before sending it to the configured connection. Same as - provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. +Configure modifications on each document before sending it to the configured connection. Same as +provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - All processors are applied in configured order. Allowing to work with already processed data. +All processors are applied in configured order. Allowing to work with already processed data. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck } - The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards - all fields, including ``search_spellcheck`` will be copied to ``search_all``. - E.g. used to index all information into a field for :ref:`spellchecking` or searching with - different :ref:`mapping`. + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } - The following Processor are available: +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. +E.g. used to index all information into a field for :ref:`spellchecking` or searching with +different :ref:`mapping`. - .. toctree:: - :maxdepth: 1 - :glob: +The following Processor are available: - dataProcessing/CopyToProcessor - dataProcessing/GeoPointProcessor +.. toctree:: + :maxdepth: 1 + :glob: - The following Processor are planned: + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. +The following Processor are planned: - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". - Of course you are able to provide further processors. Just implement - ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified - class name) as done in the examples above. + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. - By implementing also the same interface as necessary for TYPO3 - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code - also for Fluid to prepare the same record fetched from DB for your fluid. +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index 2143364..e953dcb 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -8,118 +8,200 @@ Searching size ---- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Defined how many search results should be fetched to be available in search result. +Defined how many search results should be fetched to be available in search result. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.size = 50 + plugin.tx_searchcore.settings.searching.size = 50 - Default if not configured is 10. +Default if not configured is 10. .. _facets: facets ------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. +Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html +Currently only the term facet is provided. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType } + } - The above example will provide a facet with options for all found ``CType`` results together - with a count. +The above example will provide a facet with options for all found ``CType`` results together +with a count. .. _filter: -``filter`` -""""""""""" +filter +------ - Used by: While building search request. +Used by: While building search request. - Define filter that should be set for all requests. +Define filter that should be set for all requests. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.filter { - property = value - } + plugin.tx_searchcore.settings.searching.filter { + property = value + } - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. +For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. .. _minimumShouldMatch: minimumShouldMatch ------------------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html +Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - Example:: +Example:: - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% .. _boost: boost ----- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define fields that should boost the score for results. +Define fields that should boost the score for results. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html +For further information take a look at +https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html .. _fieldValueFactor: fieldValueFactor ---------------- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor +Define a field to use as a factor for scoring. The configuration is passed through to elastic +search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - Example:: +Example:: - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mapping.filter: + +mapping.filter +-------------- + +Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will +be added. + +E.g. you submit a filter in form of: + +.. code-block:: html + + + + + +This will create a ``distance`` filter with subproperties. To make this filter actually work, you +can add the following TypoScript, which will be added to the filter:: + + mapping { + filter { + distance { + field = geo_distance + fields { + distance = distance + location = location + } + } } + } + +``fields`` has a special meaning here. This will actually map the properties of the filter to fields +in elasticsearch. In above example they do match, but you can also use different names in your form. +On the left hand side is the elasticsearch field name, on the right side the one submitted as a +filter. + +The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. + +.. _fields: + +fields +------ + +Defines the fields to fetch from elasticsearch. Two sub entries exist: + +First ``stored_fields`` which is a list of comma separated fields which actually exist and will be +added. Typically you will use ``_source`` to fetch the whole indexed fields. + +Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +An example might look like the following:: + + fields { + script_fields { + distance { + condition = {request.filter.distance.location} + script { + params { + lat = {request.filter.distance.location.lat -> f:format.number()} + lon = {request.filter.distance.location.lon -> f:format.number()} + } + lang = painless + inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001 + } + } + } + } + +In above example we add a single ``script_field`` called ``distance``. We add a condition when this +field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. +If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the +query. The ``condition`` will be removed and everything else is submitted one to one to +elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +from search request, e.g. to insert latitude and longitude from a filter, like in the above example. + +.. _sort: + +sort +---- + +Sort is handled like :ref:`fields`. .. _mode: -``mode`` -"""""""" +mode +---- - Used by: Controller while preparing action. +Used by: Controller while preparing action. - Define to switch from search to filter mode. +Define to switch from search to filter mode. - Example:: +Example:: - plugin.tx_searchcore.settings.searching { - mode = filter - } + plugin.tx_searchcore.settings.searching { + mode = filter + } - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. +Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. From 7a5bea687e32f3f247beb9bac80b874564a517ee Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:03:42 +0100 Subject: [PATCH 08/11] TASK: Add new tests for filter setting on model --- Tests/Unit/Domain/Model/SearchRequestTest.php | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Tests/Unit/Domain/Model/SearchRequestTest.php diff --git a/Tests/Unit/Domain/Model/SearchRequestTest.php b/Tests/Unit/Domain/Model/SearchRequestTest.php new file mode 100644 index 0000000..127baf2 --- /dev/null +++ b/Tests/Unit/Domain/Model/SearchRequestTest.php @@ -0,0 +1,80 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class SearchRequestTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleEmptyFilter + */ + public function emptyFilterWillNotBeSet(array $filter) + { + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + [], + $searchRequest->getFilter(), + 'Empty filter were set, even if they should not.' + ); + } + + public function possibleEmptyFilter() + { + return [ + 'Complete empty Filter' => [ + 'filter' => [], + ], + 'Single filter with empty value' => [ + 'filter' => [ + 'someFilter' => '', + ], + ], + 'Single filter with empty recursive values' => [ + 'filter' => [ + 'someFilter' => [ + 'someKey' => '', + ], + ], + ], + ]; + } + + /** + * @test + */ + public function filterIsSet() + { + $filter = ['someField' => 'someValue']; + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + $filter, + $searchRequest->getFilter(), + 'Filter was not set.' + ); + } +} From 85bfb86f5fc7ac37b8766a467426151687901e72 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:03:58 +0100 Subject: [PATCH 09/11] TASK: Fix broken tests for query factory --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index f7d0dd2..baf239f 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -100,8 +100,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ 'field' => '', - 'field1' => 0, - 'field2' => false, ]); $this->assertFalse( @@ -209,10 +207,16 @@ class QueryFactoryTest extends AbstractUnitTestCase public function minimumShouldMatchIsAddedToQuery() { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->once()) + $this->configuration->expects($this->any()) ->method('getIfExists') - ->with('searching.minimumShouldMatch') - ->willReturn('50%'); + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + '50%', + null + )); $this->configuration->expects($this->any()) ->method('get') ->will($this->throwException(new InvalidArgumentException)); @@ -244,14 +248,21 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( [ 'search_title' => 3, 'search_abstract' => 1.5, ], + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException) )); @@ -292,10 +303,17 @@ class QueryFactoryTest extends AbstractUnitTestCase 'factor' => '2', 'missing' => '1', ]; - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $fieldConfig )); From c38f7b9d6ab6b4bcbd6dd915bf2515cb52813db1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:44:17 +0100 Subject: [PATCH 10/11] TASK: Move configuration logic into own class Also add tests for new code. --- .../Configuration/ConfigurationUtility.php | 69 +++++++++ Classes/Domain/Search/QueryFactory.php | 51 ++----- Tests/Unit/AbstractUnitTestCase.php | 15 ++ .../ConfigurationUtilityTest.php | 143 ++++++++++++++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 4 +- 5 files changed, 243 insertions(+), 39 deletions(-) create mode 100644 Classes/Configuration/ConfigurationUtility.php create mode 100644 Tests/Unit/Configuration/ConfigurationUtilityTest.php diff --git a/Classes/Configuration/ConfigurationUtility.php b/Classes/Configuration/ConfigurationUtility.php new file mode 100644 index 0000000..29cb721 --- /dev/null +++ b/Classes/Configuration/ConfigurationUtility.php @@ -0,0 +1,69 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Fluid\View\StandaloneView; + +class ConfigurationUtility +{ + /** + * Will parse all entries, recursive as fluid template, with request variable set to $searchRequest. + */ + public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be send as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } + + /** + * Will check all entries, whether they have a condition and filter entries out, where condition is false. + * Also will remove condition in the end. + */ + public function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !is_array($entry) + || !array_key_exists('condition', $entry) + || (bool) $entry['condition'] === true + ; + }); + + foreach ($entries as $key => $entry) { + if (is_array($entry) && array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } +} diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index d5f3669..f73372d 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -21,13 +21,13 @@ namespace Codappix\SearchCore\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; -use TYPO3\CMS\Fluid\View\StandaloneView; class QueryFactory { @@ -41,12 +41,19 @@ class QueryFactory */ protected $configuration; + /** + * @var ConfigurationUtility + */ + protected $configurationUtility; + public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, - ConfigurationContainerInterface $configuration + ConfigurationContainerInterface $configuration, + ConfigurationUtility $configurationUtility ) { $this->logger = $logManager->getLogger(__CLASS__); $this->configuration = $configuration; + $this->configurationUtility = $configurationUtility; } /** @@ -164,8 +171,8 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); - $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $scriptFields = $this->filterByCondition($scriptFields); + $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); if ($scriptFields !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); } @@ -177,8 +184,8 @@ class QueryFactory protected function addSort(SearchRequestInterface $searchRequest, array &$query) { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; - $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $sorting = $this->filterByCondition($sorting); + $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $sorting = $this->configurationUtility->filterByCondition($sorting); if ($sorting !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); } @@ -243,36 +250,4 @@ class QueryFactory ]); } } - - protected function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array - { - array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { - $template = new StandaloneView(); - $template->assign('request', $searchRequest); - $template->setTemplateSource($value); - $value = $template->render(); - - // As elasticsearch does need some doubles to be end as doubles. - if (is_numeric($value)) { - $value = (float) $value; - } - }, $searchRequest); - - return $array; - } - - protected function filterByCondition(array $entries) : array - { - $entries = array_filter($entries, function ($entry) { - return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; - }); - - foreach ($entries as $key => $entry) { - if (array_key_exists('condition', $entry)) { - unset($entries[$key]['condition']); - } - } - - return $entries; - } } diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index c11e74e..a8ac167 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -24,6 +24,21 @@ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; abstract class AbstractUnitTestCase extends CoreTestCase { + public function setUp() + { + parent::setUp(); + + // Disable caching backends to make TYPO3 parts work in unit test mode. + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( + \TYPO3\CMS\Core\Cache\CacheManager::class + )->setCacheConfigurations([ + 'extbase_object' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + ]); + } + /** * @return \TYPO3\CMS\Core\Log\LogManager */ diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php new file mode 100644 index 0000000..b2dc5f8 --- /dev/null +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -0,0 +1,143 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationUtility; +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class ConfigurationUtilityTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleRequestAndConfigurationForFluidtemplate + */ + public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->replaceArrayValuesWithRequestContent($searchRequest, $array), + 'Entries in array were not parsed as fluid template with search request.' + ); + } + + public function possibleRequestAndConfigurationForFluidtemplate() : array + { + return [ + 'Nothing in array' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [], + 'expected' => [], + ], + 'Small array with nothing to replace' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Rescursive array with replacements' => [ + 'searchRequest' => call_user_func(function () { + $request = new SearchRequest(); + $request->setFilter([ + 'distance' => [ + 'location' => '10', + ], + ]); + return $request; + }), + 'array' => [ + 'sub1' => [ + 'sub1.1' => '{request.filter.distance.location}', + 'sub1.2' => '{request.nonExisting}', + ], + ], + 'expected' => [ + 'sub1' => [ + // Numberics are casted to double + 'sub1.1' => 10.0, + 'sub1.2' => null, + ], + ], + ], + ]; + } + + /** + * @test + * @dataProvider possibleConditionEntries + */ + public function conditionsAreHandledAsExpected(array $entries, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->filterByCondition($entries), + 'Conditions were not processed as expected.' + ); + } + + public function possibleConditionEntries() : array + { + return [ + 'Nothing in array' => [ + 'entries' => [], + 'expected' => [], + ], + 'Entries without condition' => [ + 'entries' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Entry with matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => true, + 'sub1.2' => 'something', + ], + ], + 'expected' => [ + 'sub1' => [ + 'sub1.2' => 'something', + ], + ], + ], + 'Entry with non matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => false, + 'sub1.2' => 'something', + ], + ], + 'expected' => [], + ], + ]; + } +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index baf239f..3b70679 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; @@ -44,7 +45,8 @@ class QueryFactoryTest extends AbstractUnitTestCase parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); - $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); + $configurationUtility = new ConfigurationUtility(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility); } /** From b5225b943c3efd21d90c319677226eaebfb0bd07 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 14:16:16 +0100 Subject: [PATCH 11/11] TASK: Add tests for new query factory code --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 3b70679..6301cdd 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -363,4 +363,210 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Empty search request does not create expected query.' ); } + + /** + * @test + */ + public function storedFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + '_source, something,nothing', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + ['_source', 'something', 'nothing'], + $query->toArray()['stored_fields'], + 'Stored fields were not added to query as expected.' + ); + } + + /** + * @test + */ + public function storedFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertFalse( + isset($query->toArray()['stored_fields']), + 'Stored fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function scriptFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ], + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['script_fields'], + 'Script fields were not added to query as expected.' + ); + + } + + /** + * @test + */ + public function scriptFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['script_fields']), + 'Script fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function sortIsAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ] + )); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['sort'], + 'Sort was not added to query as expected.' + ); + } + + /** + * @test + */ + public function sortIsNotAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + null + )); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['sort']), + 'Sort was added to query even if not configured.' + ); + } }