From fafa919f37155aa1a679a9154bcc07fea9c6a99f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 16 Sep 2017 20:50:03 +0200 Subject: [PATCH 01/33] WIP|FEATURE: Basic hardcoded implementation --- Classes/DataProcessing/GeoPointProcessing.php | 34 +++++++++++++++++++ Classes/DataProcessing/ProcessorInterface.php | 34 +++++++++++++++++++ .../Index/TcaIndexer/TcaTableService.php | 20 ++++++++++- Classes/Domain/Model/SearchRequest.php | 17 +++++++++- Classes/Domain/Search/QueryFactory.php | 34 +++++++++++++++---- 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 Classes/DataProcessing/GeoPointProcessing.php create mode 100644 Classes/DataProcessing/ProcessorInterface.php diff --git a/Classes/DataProcessing/GeoPointProcessing.php b/Classes/DataProcessing/GeoPointProcessing.php new file mode 100644 index 0000000..cbf6589 --- /dev/null +++ b/Classes/DataProcessing/GeoPointProcessing.php @@ -0,0 +1,34 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +class GeoPointProcessing implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + $record[$configuration['to']] = [ + 'lat' => (float) $record[$configuration['fields']['lat']], + 'lon' => (float) $record[$configuration['fields']['lon']], + ]; + + return $record; + } +} diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php new file mode 100644 index 0000000..4dc0794 --- /dev/null +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -0,0 +1,34 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * All DataProcessing Processors should implement this interface, otherwise they + * will not be executed. + */ +interface ProcessorInterface +{ + /** + * Processes the given record. + * Also retrieves the configuration for this processor instance. + */ + public function processRecord(array $record, array $configuration) : array; +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..6b9024f 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -146,6 +147,17 @@ class TcaTableService { $this->relationResolver->resolveRelationsForRecord($this, $record); + try { + foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { + $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidConfigurationArgumentException $e) { + // Nothing to do. + } + if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } @@ -181,7 +193,7 @@ class TcaTableService array_filter( array_keys($this->tca['columns']), function ($columnName) { - return !$this->isSystemField($columnName); + return !$this->isSystemField($columnName) && !$this->isUserField($columnName); } ) ); @@ -249,6 +261,12 @@ class TcaTableService return in_array($columnName, $systemFields); } + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + /** * @param string $columnName * @return array diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 39e477c..fd76062 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,22 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter(array_map('strval', $filter)); + $this->filter = array_filter($filter); + + $this->mapGeoDistanceFilter(); + } + + public function mapGeoDistanceFilter() + { + if (isset($this->filter['geo_distance'])) { + foreach (array_keys($this->filter['geo_distance']) as $config) { + if (strpos($config, '_') !== false) { + $newKey = str_replace('_', '.', $config); + $this->filter['geo_distance'][$newKey] = $this->filter['geo_distance'][$config]; + unset($this->filter['geo_distance'][$config]); + } + } + } } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9455f80..9f2f68a 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -164,6 +164,17 @@ class QueryFactory */ protected function addFactorBoost(array &$query) { + $query['sort'] = [ + '_geo_distance' => [ + 'location' => [ + 'lat' => 51.2014392, + 'lon' => 6.4302962, + ], + 'order' => 'asc', + 'unit' => 'km', + 'distance_type' => 'plane', + ] + ]; try { $query['query'] = [ 'function_score' => [ @@ -186,24 +197,33 @@ class QueryFactory return; } - $terms = []; + $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $terms[] = [ - 'term' => [ - $name => $value, - ], - ]; + $filter[] = $this->buildFilter($name, $value); } $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ - 'filter' => $terms, + 'filter' => $filter, ], ], ]); } + protected function buildFilter(string $name, $value) : array + { + if ($name === 'geo_distance') { + return [$name => $value]; + } + + return [ + 'term' => [ + $name => $value, + ], + ]; + } + /** * @param SearchRequestInterface $searchRequest * @param array $query From efeb5d1e07bdc17884c670382e757b65192b7d2d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 13:02:48 +0200 Subject: [PATCH 02/33] FEATURE: Add data processing to extension Allow integrators / developer to apply data processing concept known from FLUIDTEMPLATE to indexing. --- Classes/DataProcessing/CopyToProcessor.php | 54 +++ Classes/DataProcessing/ProcessorInterface.php | 39 ++ .../Index/TcaIndexer/TcaTableService.php | 13 + Documentation/source/concepts.rst | 10 + Documentation/source/conf.py | 1 + Documentation/source/configuration.rst | 334 +----------------- .../source/configuration/connections.rst | 55 +++ .../source/configuration/indexing.rst | 202 +++++++++++ .../source/configuration/searching.rst | 125 +++++++ Documentation/source/features.rst | 2 +- Documentation/source/indexer.rst | 18 +- .../ProcessesAllowedTablesTest.php | 2 +- .../DataProcessing/CopyToProcessorTest.php | 85 +++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 43 +++ 14 files changed, 652 insertions(+), 331 deletions(-) create mode 100644 Classes/DataProcessing/CopyToProcessor.php create mode 100644 Classes/DataProcessing/ProcessorInterface.php create mode 100644 Documentation/source/configuration/connections.rst create mode 100644 Documentation/source/configuration/indexing.rst create mode 100644 Documentation/source/configuration/searching.rst create mode 100644 Tests/Unit/DataProcessing/CopyToProcessorTest.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php new file mode 100644 index 0000000..a252a9a --- /dev/null +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -0,0 +1,54 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Copies values from one field to another one. + */ +class CopyToProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) + { + $all = []; + + $this->addArray($all, $record); + $all = array_filter($all); + $record[$configuration['to']] = implode(PHP_EOL, $all); + + return $record; + } + + /** + * @param array &$target + * @param array $from + */ + protected function addArray(array &$target, array $from) + { + foreach ($from as $value) { + if (is_array($value)) { + $this->addArray($target, $value); + continue; + } + + $target[] = (string) $value; + } + } +} diff --git a/Classes/DataProcessing/ProcessorInterface.php b/Classes/DataProcessing/ProcessorInterface.php new file mode 100644 index 0000000..1c78be3 --- /dev/null +++ b/Classes/DataProcessing/ProcessorInterface.php @@ -0,0 +1,39 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * All DataProcessing Processors should implement this interface, otherwise they + * will not be executed. + */ +interface ProcessorInterface +{ + /** + * Processes the given record. + * Also retrieves the configuration for this processor instance. + * + * @param array $record + * @param array $configuration + * + * @return array + */ + public function processRecord(array $record, array $configuration); +} diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..a0a9182 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -146,6 +148,17 @@ class TcaTableService { $this->relationResolver->resolveRelationsForRecord($this, $record); + try { + foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { + $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidConfigurationArgumentException $e) { + // Nothing to do. + } + if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } diff --git a/Documentation/source/concepts.rst b/Documentation/source/concepts.rst index 0ce3a0e..0fe6c5d 100644 --- a/Documentation/source/concepts.rst +++ b/Documentation/source/concepts.rst @@ -28,3 +28,13 @@ The indexing is done by one of the available indexer. For each identifier it's p the indexer to use. Also it's possible to write custom indexer to use. Currently only the :ref:`TcaIndexer` is provided. + +.. _concepts_indexing_dataprocessing: + +DataProcessing +^^^^^^^^^^^^^^ + +Before data is transfered to search service, it can be processed by "DataProcessors" like already +known by :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing` of :ref:`t3tsref:cobj-fluidtemplate`. + +Configuration is done through TypoScript, see :ref:`dataProcessing`. diff --git a/Documentation/source/conf.py b/Documentation/source/conf.py index a708209..1419689 100644 --- a/Documentation/source/conf.py +++ b/Documentation/source/conf.py @@ -304,6 +304,7 @@ texinfo_documents = [ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { 't3tcaref': ('https://docs.typo3.org/typo3cms/TCAReference/', None), + 't3tsref': ('https://docs.typo3.org/typo3cms/TyposcriptReference/', None), } extlinks = { 'project': ('https://github.com/Codappix/search_core/projects/%s', 'Github project: '), diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 549ca68..eca5ba4 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -36,330 +36,14 @@ Here is the example default configuration that's provided through static include Options ------- -The following section contains the different options, e.g. for :ref:`connections` and -:ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or -``plugin.tx_searchcore.settings.indexing``. +The following sections contains the different options grouped by their applied area, e.g. for +:ref:`connections` and :ref:`indexer`: ``plugin.tx_searchcore.settings.connection`` or +``plugin.tx_searchcore.settings.indexing``: -.. _configuration_options_connection: +.. toctree:: + :maxdepth: 1 + :glob: -connections -^^^^^^^^^^^ - -Holds settings regarding the different possible connections for search services like Elasticsearch -or Solr. - -Configured as:: - - plugin { - tx_searchcore { - settings { - connections { - connectionName { - // the settings - } - } - } - } - } - -Where ``connectionName`` is one of the available :ref:`connections`. - -The following settings are available. For each setting its documented which connection consumes it. - -.. _host: - -``host`` -"""""""" - - Used by: :ref:`Elasticsearch`. - - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. - - Example:: - - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost - -.. _port: - -``port`` -"""""""" - - Used by: :ref:`Elasticsearch`. - - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - - Example:: - - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 - - -.. _configuration_options_index: - -Indexing -^^^^^^^^ - -Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. - -Configured as:: - - plugin { - tx_searchcore { - settings { - indexing { - identifier { - indexer = FullyQualifiedClassname - // the settings - } - } - } - } - } - -Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. - -The following settings are available. For each setting its documented which indexer consumes it. - -.. _rootLineBlacklist: - -``rootLineBlacklist`` -""""""""""""""""""""" - - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. - - Contains a comma separated list of page uids. Spaces are trimmed. - - Example:: - - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 - -Also it's possible to define some behaviour for the different document types. In context of TYPO3 -tables are used as document types 1:1. It's possible to configure different tables. The following -options are available: - -.. _additionalWhereClause: - -``additionalWhereClause`` -""""""""""""""""""""""""" - - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. - - Example:: - - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - - .. attention:: - - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. - -.. _abstractFields: - -``abstractFields`` -""""""""""""""""""""""""" - - Used by: :ref:`PagesIndexer`. - - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. - - Example:: - - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - - Default:: - - abstract, description, bodytext - -.. _mapping: - -``mapping`` -""""""""""" - - Used by: Elasticsearch connection while indexing. - - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. - - Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } - } - - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. - - -.. _index: - -``index`` -""""""""" - - Used by: Elasticsearch connection while indexing. - - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - - Example:: - - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } - } - - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } - } - } - } - - ``char_filter`` and ``filter`` are a comma separated list of options. - -.. _configuration_options_search: - -Searching -^^^^^^^^^ - -.. _size: - -``size`` -"""""""" - - Used by: Elasticsearch connection while building search query. - - Defined how many search results should be fetched to be available in search result. - - Example:: - - plugin.tx_searchcore.settings.searching.size = 50 - - Default if not configured is 10. - -.. _facets: - -``facets`` -""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. - - Example:: - - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } - } - - The above example will provide a facet with options for all found ``CType`` results together - with a count. - -.. _filter: - -``filter`` -""""""""""" - - Used by: While building search request. - - Define filter that should be set for all requests. - - Example:: - - plugin.tx_searchcore.settings.searching.filter { - property = value - } - - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. - -.. _minimumShouldMatch: - -``minimumShouldMatch`` -"""""""""""""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - - Example:: - - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% - -.. _boost: - -``boost`` -""""""""" - - Used by: Elasticsearch connection while building search query. - - Define fields that should boost the score for results. - - Example:: - - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } - - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html - -.. _fieldValueFactor: - -``fieldValueFactor`` -"""""""""""""""""""" - - Used by: Elasticsearch connection while building search query. - - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - - Example:: - - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 - } - -.. _mode: - -``mode`` -"""""""" - - Used by: Controller while preparing action. - - Define to switch from search to filter mode. - - Example:: - - plugin.tx_searchcore.settings.searching { - mode = filter - } - - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. + configuration/connections + configuration/indexing + configuration/searching diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst new file mode 100644 index 0000000..841d878 --- /dev/null +++ b/Documentation/source/configuration/connections.rst @@ -0,0 +1,55 @@ +.. _configuration_options_connection: + +Connections +=========== + +Holds settings regarding the different possible connections for search services like Elasticsearch +or Solr. + +Configured as:: + + plugin { + tx_searchcore { + settings { + connections { + connectionName { + // the settings + } + } + } + } + } + +Where ``connectionName`` is one of the available :ref:`connections`. + +The following settings are available. For each setting its documented which connection consumes it. + +.. _host: + +``host`` +-------- + + Used by: :ref:`Elasticsearch`. + + The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 + installation. + + Example:: + + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + +.. _port: + +``port`` +-------- + + Used by: :ref:`Elasticsearch`. + + The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. + + Example:: + + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + + + diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst new file mode 100644 index 0000000..24c037f --- /dev/null +++ b/Documentation/source/configuration/indexing.rst @@ -0,0 +1,202 @@ +.. _configuration_options_index: + +Indexing +======== + +Holds settings regarding the indexing, e.g. of TYPO3 records, to search services. + +Configured as:: + + plugin { + tx_searchcore { + settings { + indexing { + identifier { + indexer = FullyQualifiedClassname + // the settings + } + } + } + } + } + +Where ``identifier`` is up to you, but should match table names to make :ref:`TcaIndexer` work. + +The following settings are available. For each setting its documented which indexer consumes it. + +.. _rootLineBlacklist: + +rootLineBlacklist +----------------- + + Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. + + Defines a blacklist of page uids. Records below any of these pages, or subpages, are not + indexed. This allows you to define areas that should not be indexed. + The page attribute *No Search* is also taken into account to prevent indexing records from only one + page without recursion. + + Contains a comma separated list of page uids. Spaces are trimmed. + + Example:: + + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + +Also it's possible to define some behaviour for the different document types. In context of TYPO3 +tables are used as document types 1:1. It's possible to configure different tables. The following +options are available: + +.. _additionalWhereClause: + +additionalWhereClause +--------------------- + + Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. + + Add additional SQL to where clauses to determine indexable records from the table. This way you + can exclude specific records like ``tt_content`` records with specific ``CType`` values or + something else. E.g. you can add a new field to the table to exclude records from indexing. + + Example:: + + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + + .. attention:: + + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. + +.. _abstractFields: + +abstractFields +-------------- + + Used by: :ref:`PagesIndexer`. + + Define which field should be used to provide the auto generated field "search_abstract". + The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also + possible. + + Example:: + + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + + Default:: + + abstract, description, bodytext + +.. _mapping: + +mapping +------- + + Used by: Elasticsearch connection while indexing. + + Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html + You are able to define the mapping for each property / columns. + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword + } + } + + The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This + makes building a facet possible. + +.. _index: + +index +----- + + Used by: Elasticsearch connection while indexing. + + Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding + } + } + + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 + } + } + } + } + + ``char_filter`` and ``filter`` are a comma separated list of options. + +.. _dataProcessing: + +dataProcessing +-------------- + + Used by: All connections while indexing. + + Configure modifications on each document before sending it to the configured connection. Same as + provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through + :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. + + All processors are applied in configured order. Allowing to work with already processed data. + + Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck + } + + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } + + The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards + all fields, including ``search_spellcheck`` will be copied to ``search_all``. + E.g. used to index all information into a field for :ref:`spellchecking` or searching with + different :ref:`mapping`. + + The following Processor are available: + + ``Codappix\SearchCore\DataProcessing\CopyToProcessor`` + Will copy contents of fields to other fields + + The following Processor are planned: + + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. + + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. + + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". + + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. + + Of course you are able to provide further processors. Just implement + ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified + class name) as done in the examples above. + + By implementing also the same interface as necessary for TYPO3 + :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code + also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst new file mode 100644 index 0000000..2143364 --- /dev/null +++ b/Documentation/source/configuration/searching.rst @@ -0,0 +1,125 @@ +.. _configuration_options_search: + +Searching +========= + +.. _size: + +size +---- + + Used by: Elasticsearch connection while building search query. + + Defined how many search results should be fetched to be available in search result. + + Example:: + + plugin.tx_searchcore.settings.searching.size = 50 + + Default if not configured is 10. + +.. _facets: + +facets +------ + + Used by: Elasticsearch connection while building search query. + + Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html + Currently only the term facet is provided. + + Example:: + + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType + } + } + + The above example will provide a facet with options for all found ``CType`` results together + with a count. + +.. _filter: + +``filter`` +""""""""""" + + Used by: While building search request. + + Define filter that should be set for all requests. + + Example:: + + plugin.tx_searchcore.settings.searching.filter { + property = value + } + + For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. + +.. _minimumShouldMatch: + +minimumShouldMatch +------------------ + + Used by: Elasticsearch connection while building search query. + + Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html + + Example:: + + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + +.. _boost: + +boost +----- + + Used by: Elasticsearch connection while building search query. + + Define fields that should boost the score for results. + + Example:: + + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } + + For further information take a look at + https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html + +.. _fieldValueFactor: + +fieldValueFactor +---------------- + + Used by: Elasticsearch connection while building search query. + + Define a field to use as a factor for scoring. The configuration is passed through to elastic + search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor + + Example:: + + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mode: + +``mode`` +"""""""" + + Used by: Controller while preparing action. + + Define to switch from search to filter mode. + + Example:: + + plugin.tx_searchcore.settings.searching { + mode = filter + } + + Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 669e632..199192a 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -24,7 +24,7 @@ Currently all fields are searched for a single search input. Also multiple filter are supported. Filtering results by fields for string contents. -Even facets / aggregates are now possible. Therefore a mapping has to be defined in TypoScript for +Facets / aggregates are also possible. Therefore a mapping has to be defined in TypoScript for indexing, and the facets itself while searching. .. _features_planned: diff --git a/Documentation/source/indexer.rst b/Documentation/source/indexer.rst index ddc6772..3bb10bc 100644 --- a/Documentation/source/indexer.rst +++ b/Documentation/source/indexer.rst @@ -21,12 +21,18 @@ further stuff. The indexer is configurable through the following options: -* :ref:`allowedTables` - * :ref:`rootLineBlacklist` * :ref:`additionalWhereClause` +* :ref:`abstractFields` + +* :ref:`mapping` + +* :ref:`index` + +* :ref:`dataProcessing` + .. _PagesIndexer: PagesIndexer @@ -42,14 +48,18 @@ improve search. The indexer is configurable through the following options: -* :ref:`allowedTables` - * :ref:`rootLineBlacklist` * :ref:`additionalWhereClause` * :ref:`abstractFields` +* :ref:`mapping` + +* :ref:`index` + +* :ref:`dataProcessing` + .. note:: Not all relations are resolved yet, see :issue:`17` and :pr:`20`. diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 7fa1cc5..6a027cf 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -100,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Unit/DataProcessing/CopyToProcessorTest.php b/Tests/Unit/DataProcessing/CopyToProcessorTest.php new file mode 100644 index 0000000..2f9e498 --- /dev/null +++ b/Tests/Unit/DataProcessing/CopyToProcessorTest.php @@ -0,0 +1,85 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class CopyToProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new CopyToProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Copy all fields to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field 2' => 'Some more content like ipsum', + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field 2' => 'Some more content like ipsum', + 'new_field' => 'Some content like lorem' . PHP_EOL . 'Some more content like ipsum', + ], + ], + 'Copy all fields with sub array to new field' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'to' => 'new_field', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'new_field' => 'Some content like lorem' . PHP_EOL . 'Tag 1' . PHP_EOL . 'Tag 2', + ], + ], + ]; + } +} diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 7fb0280..3088db6 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -21,6 +21,8 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -98,4 +100,45 @@ class TcaTableServiceTest extends AbstractUnitTestCase $whereClause->getParameters() ); } + + /** + * @test + */ + public function executesConfiguredDataProcessing() + { + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('indexing.testTable.dataProcessing') + ->will($this->returnValue([ + '1' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + '2' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + ])); + + $subject = $this->getMockBuilder(TcaTableService::class) + ->disableOriginalConstructor() + ->setMethodsExcept(['prepareRecord']) + ->getMock(); + $this->inject($subject, 'configuration', $this->configuration); + $this->inject($subject, 'tableName', 'testTable'); + $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); + + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['new_test_field'] = 'test'; + $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; + + $subject->prepareRecord($record); + + $this->assertSame( + $expectedRecord, + $record, + 'Dataprocessing is not executed by TcaTableService as expected.' + ); + } } From 6c01abe5a546d77a96600cda62c6e484d94b7dfa Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 13:35:44 +0200 Subject: [PATCH 03/33] BUGFIX: Also handle data processor without configuration --- .../Index/TcaIndexer/TcaTableService.php | 9 ++++- .../Index/TcaIndexer/TcaTableServiceTest.php | 34 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index a0a9182..c46b0e8 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -150,7 +150,14 @@ class TcaTableService try { foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { - $dataProcessor = GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); + $className = ''; + if (is_string($configuration)) { + $className = $configuration; + $configuration = []; + } else { + $className = $configuration['_typoScriptNodeValue']; + } + $dataProcessor = GeneralUtility::makeInstance($className); if ($dataProcessor instanceof ProcessorInterface) { $record = $dataProcessor->processRecord($record, $configuration); } diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 3088db6..e12a4af 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -104,7 +104,7 @@ class TcaTableServiceTest extends AbstractUnitTestCase /** * @test */ - public function executesConfiguredDataProcessing() + public function executesConfiguredDataProcessingWithConfiguration() { $this->configuration->expects($this->exactly(1)) ->method('get') @@ -141,4 +141,36 @@ class TcaTableServiceTest extends AbstractUnitTestCase 'Dataprocessing is not executed by TcaTableService as expected.' ); } + + /** + * @test + */ + public function executesConfiguredDataProcessingWithoutConfiguration() + { + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('indexing.testTable.dataProcessing') + ->will($this->returnValue([CopyToProcessor::class])); + + $subject = $this->getMockBuilder(TcaTableService::class) + ->disableOriginalConstructor() + ->setMethodsExcept(['prepareRecord']) + ->getMock(); + $this->inject($subject, 'configuration', $this->configuration); + $this->inject($subject, 'tableName', 'testTable'); + $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); + + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord[''] = 'test'; + $expectedRecord['search_title'] = 'test'; + + $subject->prepareRecord($record); + + $this->assertSame( + $expectedRecord, + $record, + 'Dataprocessing is not executed by TcaTableService as expected.' + ); + } } From 2c466854b2b76b9f2a02091925f754f570159c2e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 12:27:48 +0200 Subject: [PATCH 04/33] BUGFIX: Do not add non existing db columns to fields array As TCA might contain columns which do not exist in DB, filter them out. --- .../Index/TcaIndexer/TcaTableService.php | 17 ++++++- .../ProcessesAllowedTablesTest.php | 2 +- .../Index/TcaIndexer/TcaTableServiceTest.php | 49 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index dae86aa..c00172d 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -181,7 +181,10 @@ class TcaTableService array_filter( array_keys($this->tca['columns']), function ($columnName) { - return !$this->isSystemField($columnName); + return !$this->isSystemField($columnName) + && !$this->isUserField($columnName) + && !$this->isPassthroughField($columnName) + ; } ) ); @@ -249,6 +252,18 @@ class TcaTableService return in_array($columnName, $systemFields); } + protected function isUserField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'user'; + } + + protected function isPassthroughField(string $columnName) : bool + { + $config = $this->getColumnConfig($columnName); + return isset($config['type']) && $config['type'] === 'passthrough'; + } + /** * @param string $columnName * @return array diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 7fa1cc5..6a027cf 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -100,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index 7fb0280..27a53ae 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Index\TcaIndexer; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Domain\Index\TcaIndexer\RelationResolver; use Codappix\SearchCore\Domain\Index\TcaIndexer\TcaTableService; use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; @@ -98,4 +99,52 @@ class TcaTableServiceTest extends AbstractUnitTestCase $whereClause->getParameters() ); } + + /** + * @test + */ + public function allConfiguredAndAllowedTcaColumnsAreReturnedAsFields() + { + $GLOBALS['TCA']['test_table'] = [ + 'ctrl' => [ + 'languageField' => 'sys_language', + ], + 'columns' => [ + 'sys_language' => [], + 't3ver_oid' => [], + 'available_column' => [ + 'config' => [ + 'type' => 'input', + ], + ], + 'user_column' => [ + 'config' => [ + 'type' => 'user', + ], + ], + 'passthrough_column' => [ + 'config' => [ + 'type' => 'passthrough', + ], + ], + ], + ]; + $subject = new TcaTableService( + 'test_table', + $this->getMockBuilder(RelationResolver::class)->getMock(), + $this->configuration + ); + $this->inject($subject, 'logger', $this->getMockedLogger()); + + $this->assertSame( + [ + 'test_table.uid', + 'test_table.pid', + 'test_table.available_column', + ], + $subject->getFields(), + '' + ); + unset($GLOBALS['TCA']['test_table']); + } } From 67a43e64a5b892344fa62a66ab9626238d979c8e Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sat, 14 Oct 2017 13:06:22 +0200 Subject: [PATCH 05/33] BUGFIX: Fix typos in method name and php doc --- Classes/Connection/Elasticsearch/SearchResult.php | 2 +- Classes/Connection/SearchResultInterface.php | 2 +- .../Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php | 2 +- Tests/Unit/Domain/Search/QueryFactoryTest.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Classes/Connection/Elasticsearch/SearchResult.php b/Classes/Connection/Elasticsearch/SearchResult.php index f45f02f..3919722 100644 --- a/Classes/Connection/Elasticsearch/SearchResult.php +++ b/Classes/Connection/Elasticsearch/SearchResult.php @@ -83,7 +83,7 @@ class SearchResult implements SearchResultInterface /** * Return all facets, if any. * - * @return array + * @return array */ public function getFacets() { diff --git a/Classes/Connection/SearchResultInterface.php b/Classes/Connection/SearchResultInterface.php index d52a8ba..60718cd 100644 --- a/Classes/Connection/SearchResultInterface.php +++ b/Classes/Connection/SearchResultInterface.php @@ -35,7 +35,7 @@ interface SearchResultInterface extends \Iterator, \Countable, QueryResultInterf /** * Return all facets, if any. * - * @return array + * @return array */ public function getFacets(); diff --git a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php index 7fa1cc5..6a027cf 100644 --- a/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php +++ b/Tests/Functional/Hooks/DataHandler/ProcessesAllowedTablesTest.php @@ -100,7 +100,7 @@ class ProcessesAllowedTablesTest extends AbstractDataHandlerTest ->with( $this->equalTo('tt_content'), $this->callback(function ($record) { - return isset($record['uid']) && $record['uid'] === '2' + return isset($record['uid']) && $record['uid'] === 2 && isset($record['pid']) && $record['pid'] === 1 && isset($record['header']) && $record['header'] === 'a new record' ; diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 54fdc2a..f7d0dd2 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -50,7 +50,7 @@ class QueryFactoryTest extends AbstractUnitTestCase /** * @test */ - public function creatonOfQueryWorksInGeneral() + public function creationOfQueryWorksInGeneral() { $searchRequest = new SearchRequest('SearchWord'); From e1764dca133fb9c10ee7f98d045163dbe83bcc67 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 20 Oct 2017 16:36:26 +0200 Subject: [PATCH 06/33] FEATURE: Add GeoPoint Processor Document data processors. Add test for new data processor. --- Classes/DataProcessing/CopyToProcessor.php | 6 +- ...ntProcessing.php => GeoPointProcessor.php} | 31 ++++- .../dataProcessing/CopyToProcessor.rst | 23 ++++ .../dataProcessing/GeoPointProcessor.rst | 27 +++++ .../source/configuration/indexing.rst | 8 +- .../DataProcessing/GeoPointProcessorTest.php | 113 ++++++++++++++++++ 6 files changed, 198 insertions(+), 10 deletions(-) rename Classes/DataProcessing/{GeoPointProcessing.php => GeoPointProcessor.php} (53%) create mode 100644 Documentation/source/configuration/dataProcessing/CopyToProcessor.rst create mode 100644 Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst create mode 100644 Tests/Unit/DataProcessing/GeoPointProcessorTest.php diff --git a/Classes/DataProcessing/CopyToProcessor.php b/Classes/DataProcessing/CopyToProcessor.php index a252a9a..eea555f 100644 --- a/Classes/DataProcessing/CopyToProcessor.php +++ b/Classes/DataProcessing/CopyToProcessor.php @@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing; */ class CopyToProcessor implements ProcessorInterface { - public function processRecord(array $record, array $configuration) + public function processRecord(array $record, array $configuration) : array { $all = []; @@ -36,10 +36,6 @@ class CopyToProcessor implements ProcessorInterface return $record; } - /** - * @param array &$target - * @param array $from - */ protected function addArray(array &$target, array $from) { foreach ($from as $value) { diff --git a/Classes/DataProcessing/GeoPointProcessing.php b/Classes/DataProcessing/GeoPointProcessor.php similarity index 53% rename from Classes/DataProcessing/GeoPointProcessing.php rename to Classes/DataProcessing/GeoPointProcessor.php index cbf6589..f9256b3 100644 --- a/Classes/DataProcessing/GeoPointProcessing.php +++ b/Classes/DataProcessing/GeoPointProcessor.php @@ -20,15 +20,40 @@ namespace Codappix\SearchCore\DataProcessing; * 02110-1301, USA. */ -class GeoPointProcessing implements ProcessorInterface +/** + * Adds a new fields, ready to use as GeoPoint field for Elasticsearch. + */ +class GeoPointProcessor implements ProcessorInterface { public function processRecord(array $record, array $configuration) : array { + if (! $this->canApply($record, $configuration)) { + return $record; + } + $record[$configuration['to']] = [ - 'lat' => (float) $record[$configuration['fields']['lat']], - 'lon' => (float) $record[$configuration['fields']['lon']], + 'lat' => (float) $record[$configuration['lat']], + 'lon' => (float) $record[$configuration['lon']], ]; return $record; } + + protected function canApply(array $record, array $configuration) : bool + { + if (!isset($record[$configuration['lat']]) + || !is_numeric($record[$configuration['lat']]) + || trim($record[$configuration['lat']]) === '' + ) { + return false; + } + if (!isset($record[$configuration['lon']]) + || !is_numeric($record[$configuration['lon']]) + || trim($record[$configuration['lon']]) === '' + ) { + return false; + } + + return true; + } } diff --git a/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst new file mode 100644 index 0000000..6d83d70 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/CopyToProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\CopyToProcessor`` +====================================================== + +Will copy contents of fields to other fields. + +Possible Options: + +``to`` + Defines the field to copy the values into. All values not false will be copied at the moment. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = all + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = spellcheck + } + } + diff --git a/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst new file mode 100644 index 0000000..a19928f --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst @@ -0,0 +1,27 @@ +``Codappix\SearchCore\DataProcessing\GeoPointProcessor`` +======================================================== + +Will create a new field, ready to use as GeoPoint field for Elasticsearch. + +Possible Options: + +``to`` + Defines the field to create as GeoPoint. +``lat`` + Defines the field containing the latitude. +``lon`` + Defines the field containing the longitude. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor + 1 { + to = location + lat = lat + lon = lng + } + } + +The above example will create a new field ``location`` as GeoPoint with latitude fetched from field +``lat`` and longitude fetched from field ``lng``. diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 24c037f..413da3d 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -175,8 +175,12 @@ dataProcessing The following Processor are available: - ``Codappix\SearchCore\DataProcessing\CopyToProcessor`` - Will copy contents of fields to other fields + .. toctree:: + :maxdepth: 1 + :glob: + + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor The following Processor are planned: diff --git a/Tests/Unit/DataProcessing/GeoPointProcessorTest.php b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php new file mode 100644 index 0000000..99b8eb3 --- /dev/null +++ b/Tests/Unit/DataProcessing/GeoPointProcessorTest.php @@ -0,0 +1,113 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\GeoPointProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class GeoPointProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new GeoPointProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Create new field with existing lat and lng' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + 'location' => [ + 'lat' => 23.232, + 'lon' => 45.43, + ], + ], + ], + 'Do not create new field due to missing configuration' => [ + 'record' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + 'configuration' => [ + 'to' => 'location', + ], + 'expectedRecord' => [ + 'lat' => 23.232, + 'lng' => 45.43, + ], + ], + 'Do not create new field due to missing lat and lon' => [ + 'record' => [ + 'lat' => '', + 'lng' => '', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => '', + 'lng' => '', + ], + ], + 'Do not create new field due to invalid lat and lon' => [ + 'record' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + 'configuration' => [ + 'to' => 'location', + 'lat' => 'lat', + 'lon' => 'lng', + ], + 'expectedRecord' => [ + 'lat' => 'av', + 'lng' => 'dsf', + ], + ], + ]; + } +} From 636ef78a143e34461d8c43fb74e2ccefbea32598 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 11:23:28 +0200 Subject: [PATCH 07/33] WIP|TASK: Remove sort and add filter As we filter for distance and do not sort. --- Classes/Domain/Model/SearchRequest.php | 15 ---- Classes/Domain/Search/QueryFactory.php | 98 +++++++++----------------- 2 files changed, 35 insertions(+), 78 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index fd76062..1e3e652 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -93,21 +93,6 @@ class SearchRequest implements SearchRequestInterface public function setFilter(array $filter) { $this->filter = array_filter($filter); - - $this->mapGeoDistanceFilter(); - } - - public function mapGeoDistanceFilter() - { - if (isset($this->filter['geo_distance'])) { - foreach (array_keys($this->filter['geo_distance']) as $config) { - if (strpos($config, '_') !== false) { - $newKey = str_replace('_', '.', $config); - $this->filter['geo_distance'][$newKey] = $this->filter['geo_distance'][$config]; - unset($this->filter['geo_distance'][$config]); - } - } - } } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 9f2f68a..6511cdd 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -39,10 +39,6 @@ class QueryFactory */ protected $configuration; - /** - * @param \TYPO3\CMS\Core\Log\LogManager $logManager - * @param ConfigurationContainerInterface $configuration - */ public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, ConfigurationContainerInterface $configuration @@ -55,22 +51,13 @@ class QueryFactory * TODO: This is not in scope Elasticsearch, therefore it should not return * \Elastica\Query, but decide to use a more specific QueryFactory like * ElasticaQueryFactory, once the second query is added? - * - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query */ - public function create(SearchRequestInterface $searchRequest) + public function create(SearchRequestInterface $searchRequest) : \Elastica\Query { return $this->createElasticaQuery($searchRequest); } - /** - * @param SearchRequestInterface $searchRequest - * - * @return \Elastica\Query - */ - protected function createElasticaQuery(SearchRequestInterface $searchRequest) + protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query { $query = []; $this->addSize($searchRequest, $query); @@ -83,14 +70,12 @@ class QueryFactory // Better approach would be something like DQL to generate query and build result in the end. $this->addFactorBoost($query); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($query, '$query', 8, false);die; + $this->logger->debug('Generated elasticsearch query.', [$query]); return new \Elastica\Query($query); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSize(SearchRequestInterface $searchRequest, array &$query) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -99,10 +84,6 @@ class QueryFactory ]); } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { if (trim($searchRequest->getSearchTerm()) === '') { @@ -125,10 +106,6 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { try { @@ -159,22 +136,8 @@ class QueryFactory ]); } - /** - * @param array $query - */ protected function addFactorBoost(array &$query) { - $query['sort'] = [ - '_geo_distance' => [ - 'location' => [ - 'lat' => 51.2014392, - 'lon' => 6.4302962, - ], - 'order' => 'asc', - 'unit' => 'km', - 'distance_type' => 'plane', - ] - ]; try { $query['query'] = [ 'function_score' => [ @@ -187,10 +150,6 @@ class QueryFactory } } - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { @@ -199,7 +158,11 @@ class QueryFactory $filter = []; foreach ($searchRequest->getFilter() as $name => $value) { - $filter[] = $this->buildFilter($name, $value); + $filter[] = $this->buildFilter( + $name, + $value, + $this->configuration->getIfExists('searching.filter.' . $name) ?: [] + ); } $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ @@ -211,23 +174,6 @@ class QueryFactory ]); } - protected function buildFilter(string $name, $value) : array - { - if ($name === 'geo_distance') { - return [$name => $value]; - } - - return [ - 'term' => [ - $name => $value, - ], - ]; - } - - /** - * @param SearchRequestInterface $searchRequest - * @param array $query - */ protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { @@ -242,4 +188,30 @@ class QueryFactory ]); } } + + protected function buildFilter(string $name, $value, array $config) : array + { + if ($config === []) { + return [ + 'term' => [ + $name => $value, + ], + ]; + } + + $filter = []; + + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($name, '$name', 8, false); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false); + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($config, '$config', 8, false);die; + + if (isset($config['fields'])) { + foreach ($config['fields'] as $elasticField => $inputField) { + \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false);die; + $filter[$elasticField] = $value[$inputField]; + } + } + + return [$config['field'] => $filter]; + } } From 8d343ee97fd178db5f549bc35345e8ab63bcefd5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 17:34:48 +0200 Subject: [PATCH 08/33] FEATURE: Finish filter configuration for geo_search --- Classes/Domain/Search/QueryFactory.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 6511cdd..2042775 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -70,8 +70,6 @@ class QueryFactory // Better approach would be something like DQL to generate query and build result in the end. $this->addFactorBoost($query); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($query, '$query', 8, false);die; - $this->logger->debug('Generated elasticsearch query.', [$query]); return new \Elastica\Query($query); } @@ -161,7 +159,7 @@ class QueryFactory $filter[] = $this->buildFilter( $name, $value, - $this->configuration->getIfExists('searching.filter.' . $name) ?: [] + $this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: [] ); } @@ -201,13 +199,8 @@ class QueryFactory $filter = []; - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($name, '$name', 8, false); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false); - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($config, '$config', 8, false);die; - if (isset($config['fields'])) { foreach ($config['fields'] as $elasticField => $inputField) { - \TYPO3\CMS\Extbase\Utility\DebuggerUtility::var_dump($value, '$value', 8, false);die; $filter[$elasticField] = $value[$inputField]; } } From b1f81c0d3be781fbdc6d5fb0050b3dcd4a60a21d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 22 Oct 2017 17:51:04 +0200 Subject: [PATCH 09/33] WIP|FEATURE: Add field and sorting Sort result by distance and provide distance to result items. --- Classes/Domain/Search/QueryFactory.php | 70 ++++++++++++++++++++------ 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 2042775..7fa3cd5 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -65,6 +65,8 @@ class QueryFactory $this->addBoosts($searchRequest, $query); $this->addFilter($searchRequest, $query); $this->addFacets($searchRequest, $query); + $this->addFields($searchRequest, $query); + $this->addSort($searchRequest, $query); // Use last, as it might change structure of query. // Better approach would be something like DQL to generate query and build result in the end. @@ -148,6 +150,44 @@ class QueryFactory } } + protected function addFields(SearchRequestInterface $searchRequest, array &$query) + { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => [ '_source' ], + ]); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'script_fields' => [ + 'distance' => [ + 'script' => [ + 'params' => [ + 'lat' => 51.168098, + 'lon' => 6.381384, + ], + 'lang' => 'painless', + 'inline' => 'doc["location"].arcDistance(params.lat,params.lon) * 0.001', + ], + ], + ], + ]); + } + + protected function addSort(SearchRequestInterface $searchRequest, array &$query) + { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'sort' => [ + '_geo_distance' => [ + 'location' => [ + 'lat' => 51.168098, + 'lon' => 6.381384, + ], + 'order' => 'asc', + 'unit' => 'km', + 'distance_type' => 'plane', + ], + ], + ]); + } + protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { @@ -172,21 +212,6 @@ class QueryFactory ]); } - protected function addFacets(SearchRequestInterface $searchRequest, array &$query) - { - foreach ($searchRequest->getFacets() as $facet) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'aggs' => [ - $facet->getIdentifier() => [ - 'terms' => [ - 'field' => $facet->getField(), - ], - ], - ], - ]); - } - } - protected function buildFilter(string $name, $value, array $config) : array { if ($config === []) { @@ -207,4 +232,19 @@ class QueryFactory return [$config['field'] => $filter]; } + + protected function addFacets(SearchRequestInterface $searchRequest, array &$query) + { + foreach ($searchRequest->getFacets() as $facet) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'aggs' => [ + $facet->getIdentifier() => [ + 'terms' => [ + 'field' => $facet->getField(), + ], + ], + ], + ]); + } + } } From 07a4fec622795ed959b4056c43944ad2ff01647d Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 23 Oct 2017 16:35:38 +0200 Subject: [PATCH 10/33] WIP|FEATURE: Allow fields and sorting to be configurable --- Classes/Domain/Model/SearchRequest.php | 3 +- Classes/Domain/Search/QueryFactory.php | 67 ++++++++++++++------------ 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/Classes/Domain/Model/SearchRequest.php b/Classes/Domain/Model/SearchRequest.php index 1e3e652..d751ce8 100644 --- a/Classes/Domain/Model/SearchRequest.php +++ b/Classes/Domain/Model/SearchRequest.php @@ -92,7 +92,8 @@ class SearchRequest implements SearchRequestInterface */ public function setFilter(array $filter) { - $this->filter = array_filter($filter); + $filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, ''); + $this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter); } /** diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7fa3cd5..56d4dd3 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -25,7 +25,9 @@ use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; +use TYPO3\CMS\Fluid\View\StandaloneView; class QueryFactory { @@ -152,40 +154,28 @@ class QueryFactory protected function addFields(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'stored_fields' => [ '_source' ], - ]); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'script_fields' => [ - 'distance' => [ - 'script' => [ - 'params' => [ - 'lat' => 51.168098, - 'lon' => 6.381384, - ], - 'lang' => 'painless', - 'inline' => 'doc["location"].arcDistance(params.lat,params.lon) * 0.001', - ], - ], - ], - ]); + try { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ + 'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true), + ]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } + + try { + $scriptFields = $this->configuration->get('searching.fields.script_fields'); + $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } catch (InvalidArgumentException $e) { + // Nothing configured + } } protected function addSort(SearchRequestInterface $searchRequest, array &$query) { - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ - 'sort' => [ - '_geo_distance' => [ - 'location' => [ - 'lat' => 51.168098, - 'lon' => 6.381384, - ], - 'order' => 'asc', - 'unit' => 'km', - 'distance_type' => 'plane', - ], - ], - ]); + $sorting = $this->configuration->getIfExists('searching.sort') ?: []; + $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -247,4 +237,21 @@ class QueryFactory ]); } } + + protected function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be end as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } } From 86d02f7b8d6930e32affd1058398d33756f91e45 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 26 Oct 2017 10:05:32 +0200 Subject: [PATCH 11/33] TASK: Allow integrators to use GET with minimal overhead Allow to map search request even if no trusted properties exist. Also cache initial call to plugin. This allows to use GET as submit for forms with minimal arguments in URL. --- Classes/Controller/SearchController.php | 6 ++++++ ext_localconf.php | 3 --- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Classes/Controller/SearchController.php b/Classes/Controller/SearchController.php index 0fa7f73..e484fcd 100644 --- a/Classes/Controller/SearchController.php +++ b/Classes/Controller/SearchController.php @@ -56,6 +56,12 @@ class SearchController extends ActionController ] )); } + + if ($this->arguments->hasArgument('searchRequest')) { + $this->arguments->getArgument('searchRequest')->getPropertyMappingConfiguration() + ->allowAllProperties() + ; + } } /** diff --git a/ext_localconf.php b/ext_localconf.php index 54bf43b..5c42c9c 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -31,9 +31,6 @@ call_user_func( 'search', [ 'Search' => 'search' - ], - [ - 'Search' => 'search' // TODO: Enable caching. But submitting form results in previous result?! ] ); From 769bdd1003a94b06ad97579b51126d6f67109352 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 11:32:21 +0100 Subject: [PATCH 12/33] BUGFIX: Do not cache search action --- ext_localconf.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext_localconf.php b/ext_localconf.php index 5c42c9c..c52d05d 100644 --- a/ext_localconf.php +++ b/ext_localconf.php @@ -29,6 +29,9 @@ call_user_func( TYPO3\CMS\Extbase\Utility\ExtensionUtility::configurePlugin( 'Codappix.' . $extensionKey, 'search', + [ + 'Search' => 'search' + ], [ 'Search' => 'search' ] From bf91c4a5ba2005bf78d5497661e001f47e26af4a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 12:25:25 +0100 Subject: [PATCH 13/33] TASK: Allow fields and sorting to contain a condition This way integrators can configure when the sorting and fields should be added. --- Classes/Domain/Search/QueryFactory.php | 25 ++- .../source/configuration/connections.rst | 18 +- .../source/configuration/indexing.rst | 194 +++++++++--------- .../source/configuration/searching.rst | 192 ++++++++++++----- 4 files changed, 266 insertions(+), 163 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 56d4dd3..d5f3669 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -165,7 +165,10 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + $scriptFields = $this->filterByCondition($scriptFields); + if ($scriptFields !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } } catch (InvalidArgumentException $e) { // Nothing configured } @@ -175,7 +178,10 @@ class QueryFactory { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + $sorting = $this->filterByCondition($sorting); + if ($sorting !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + } } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -254,4 +260,19 @@ class QueryFactory return $array; } + + protected function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; + }); + + foreach ($entries as $key => $entry) { + if (array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } } diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst index 841d878..5819730 100644 --- a/Documentation/source/configuration/connections.rst +++ b/Documentation/source/configuration/connections.rst @@ -29,27 +29,27 @@ The following settings are available. For each setting its documented which conn ``host`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. +The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 +installation. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost .. _port: ``port`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. +The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 413da3d..93eff19 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -29,18 +29,18 @@ The following settings are available. For each setting its documented which inde rootLineBlacklist ----------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. +Defines a blacklist of page uids. Records below any of these pages, or subpages, are not +indexed. This allows you to define areas that should not be indexed. +The page attribute *No Search* is also taken into account to prevent indexing records from only one +page without recursion. - Contains a comma separated list of page uids. Spaces are trimmed. +Contains a comma separated list of page uids. Spaces are trimmed. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 Also it's possible to define some behaviour for the different document types. In context of TYPO3 tables are used as document types 1:1. It's possible to configure different tables. The following @@ -51,156 +51,156 @@ options are available: additionalWhereClause --------------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. +Add additional SQL to where clauses to determine indexable records from the table. This way you +can exclude specific records like ``tt_content`` records with specific ``CType`` values or +something else. E.g. you can add a new field to the table to exclude records from indexing. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - .. attention:: +.. attention:: - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. .. _abstractFields: abstractFields -------------- - Used by: :ref:`PagesIndexer`. +Used by: :ref:`PagesIndexer`. - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. +Define which field should be used to provide the auto generated field "search_abstract". +The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also +possible. - Example:: +Example:: - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - Default:: +Default:: - abstract, description, bodytext + abstract, description, bodytext .. _mapping: mapping ------- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. +Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html +You are able to define the mapping for each property / columns. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword } + } - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. +The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This +makes building a facet possible. .. _index: index ----- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html +Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding } + } - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 } } } + } - ``char_filter`` and ``filter`` are a comma separated list of options. +``char_filter`` and ``filter`` are a comma separated list of options. .. _dataProcessing: dataProcessing -------------- - Used by: All connections while indexing. +Used by: All connections while indexing. - Configure modifications on each document before sending it to the configured connection. Same as - provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. +Configure modifications on each document before sending it to the configured connection. Same as +provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - All processors are applied in configured order. Allowing to work with already processed data. +All processors are applied in configured order. Allowing to work with already processed data. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck } - The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards - all fields, including ``search_spellcheck`` will be copied to ``search_all``. - E.g. used to index all information into a field for :ref:`spellchecking` or searching with - different :ref:`mapping`. + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } - The following Processor are available: +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. +E.g. used to index all information into a field for :ref:`spellchecking` or searching with +different :ref:`mapping`. - .. toctree:: - :maxdepth: 1 - :glob: +The following Processor are available: - dataProcessing/CopyToProcessor - dataProcessing/GeoPointProcessor +.. toctree:: + :maxdepth: 1 + :glob: - The following Processor are planned: + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. +The following Processor are planned: - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". - Of course you are able to provide further processors. Just implement - ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified - class name) as done in the examples above. + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. - By implementing also the same interface as necessary for TYPO3 - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code - also for Fluid to prepare the same record fetched from DB for your fluid. +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index 2143364..e953dcb 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -8,118 +8,200 @@ Searching size ---- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Defined how many search results should be fetched to be available in search result. +Defined how many search results should be fetched to be available in search result. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.size = 50 + plugin.tx_searchcore.settings.searching.size = 50 - Default if not configured is 10. +Default if not configured is 10. .. _facets: facets ------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. +Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html +Currently only the term facet is provided. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType } + } - The above example will provide a facet with options for all found ``CType`` results together - with a count. +The above example will provide a facet with options for all found ``CType`` results together +with a count. .. _filter: -``filter`` -""""""""""" +filter +------ - Used by: While building search request. +Used by: While building search request. - Define filter that should be set for all requests. +Define filter that should be set for all requests. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.filter { - property = value - } + plugin.tx_searchcore.settings.searching.filter { + property = value + } - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. +For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. .. _minimumShouldMatch: minimumShouldMatch ------------------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html +Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - Example:: +Example:: - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% .. _boost: boost ----- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define fields that should boost the score for results. +Define fields that should boost the score for results. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html +For further information take a look at +https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html .. _fieldValueFactor: fieldValueFactor ---------------- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor +Define a field to use as a factor for scoring. The configuration is passed through to elastic +search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - Example:: +Example:: - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mapping.filter: + +mapping.filter +-------------- + +Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will +be added. + +E.g. you submit a filter in form of: + +.. code-block:: html + + + + + +This will create a ``distance`` filter with subproperties. To make this filter actually work, you +can add the following TypoScript, which will be added to the filter:: + + mapping { + filter { + distance { + field = geo_distance + fields { + distance = distance + location = location + } + } } + } + +``fields`` has a special meaning here. This will actually map the properties of the filter to fields +in elasticsearch. In above example they do match, but you can also use different names in your form. +On the left hand side is the elasticsearch field name, on the right side the one submitted as a +filter. + +The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. + +.. _fields: + +fields +------ + +Defines the fields to fetch from elasticsearch. Two sub entries exist: + +First ``stored_fields`` which is a list of comma separated fields which actually exist and will be +added. Typically you will use ``_source`` to fetch the whole indexed fields. + +Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +An example might look like the following:: + + fields { + script_fields { + distance { + condition = {request.filter.distance.location} + script { + params { + lat = {request.filter.distance.location.lat -> f:format.number()} + lon = {request.filter.distance.location.lon -> f:format.number()} + } + lang = painless + inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001 + } + } + } + } + +In above example we add a single ``script_field`` called ``distance``. We add a condition when this +field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. +If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the +query. The ``condition`` will be removed and everything else is submitted one to one to +elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +from search request, e.g. to insert latitude and longitude from a filter, like in the above example. + +.. _sort: + +sort +---- + +Sort is handled like :ref:`fields`. .. _mode: -``mode`` -"""""""" +mode +---- - Used by: Controller while preparing action. +Used by: Controller while preparing action. - Define to switch from search to filter mode. +Define to switch from search to filter mode. - Example:: +Example:: - plugin.tx_searchcore.settings.searching { - mode = filter - } + plugin.tx_searchcore.settings.searching { + mode = filter + } - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. +Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. From 7a5bea687e32f3f247beb9bac80b874564a517ee Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:03:42 +0100 Subject: [PATCH 14/33] TASK: Add new tests for filter setting on model --- Tests/Unit/Domain/Model/SearchRequestTest.php | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Tests/Unit/Domain/Model/SearchRequestTest.php diff --git a/Tests/Unit/Domain/Model/SearchRequestTest.php b/Tests/Unit/Domain/Model/SearchRequestTest.php new file mode 100644 index 0000000..127baf2 --- /dev/null +++ b/Tests/Unit/Domain/Model/SearchRequestTest.php @@ -0,0 +1,80 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class SearchRequestTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleEmptyFilter + */ + public function emptyFilterWillNotBeSet(array $filter) + { + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + [], + $searchRequest->getFilter(), + 'Empty filter were set, even if they should not.' + ); + } + + public function possibleEmptyFilter() + { + return [ + 'Complete empty Filter' => [ + 'filter' => [], + ], + 'Single filter with empty value' => [ + 'filter' => [ + 'someFilter' => '', + ], + ], + 'Single filter with empty recursive values' => [ + 'filter' => [ + 'someFilter' => [ + 'someKey' => '', + ], + ], + ], + ]; + } + + /** + * @test + */ + public function filterIsSet() + { + $filter = ['someField' => 'someValue']; + $searchRequest = new SearchRequest(); + $searchRequest->setFilter($filter); + + $this->assertSame( + $filter, + $searchRequest->getFilter(), + 'Filter was not set.' + ); + } +} From 85bfb86f5fc7ac37b8766a467426151687901e72 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:03:58 +0100 Subject: [PATCH 15/33] TASK: Fix broken tests for query factory --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index f7d0dd2..baf239f 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -100,8 +100,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ 'field' => '', - 'field1' => 0, - 'field2' => false, ]); $this->assertFalse( @@ -209,10 +207,16 @@ class QueryFactoryTest extends AbstractUnitTestCase public function minimumShouldMatchIsAddedToQuery() { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->once()) + $this->configuration->expects($this->any()) ->method('getIfExists') - ->with('searching.minimumShouldMatch') - ->willReturn('50%'); + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + '50%', + null + )); $this->configuration->expects($this->any()) ->method('get') ->will($this->throwException(new InvalidArgumentException)); @@ -244,14 +248,21 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( [ 'search_title' => 3, 'search_abstract' => 1.5, ], + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException) )); @@ -292,10 +303,17 @@ class QueryFactoryTest extends AbstractUnitTestCase 'factor' => '2', 'missing' => '1', ]; - $this->configuration->expects($this->exactly(2)) + $this->configuration->expects($this->any()) ->method('get') - ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $fieldConfig )); From c38f7b9d6ab6b4bcbd6dd915bf2515cb52813db1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 13:44:17 +0100 Subject: [PATCH 16/33] TASK: Move configuration logic into own class Also add tests for new code. --- .../Configuration/ConfigurationUtility.php | 69 +++++++++ Classes/Domain/Search/QueryFactory.php | 51 ++----- Tests/Unit/AbstractUnitTestCase.php | 15 ++ .../ConfigurationUtilityTest.php | 143 ++++++++++++++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 4 +- 5 files changed, 243 insertions(+), 39 deletions(-) create mode 100644 Classes/Configuration/ConfigurationUtility.php create mode 100644 Tests/Unit/Configuration/ConfigurationUtilityTest.php diff --git a/Classes/Configuration/ConfigurationUtility.php b/Classes/Configuration/ConfigurationUtility.php new file mode 100644 index 0000000..29cb721 --- /dev/null +++ b/Classes/Configuration/ConfigurationUtility.php @@ -0,0 +1,69 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Connection\SearchRequestInterface; +use TYPO3\CMS\Fluid\View\StandaloneView; + +class ConfigurationUtility +{ + /** + * Will parse all entries, recursive as fluid template, with request variable set to $searchRequest. + */ + public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array + { + array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { + $template = new StandaloneView(); + $template->assign('request', $searchRequest); + $template->setTemplateSource($value); + $value = $template->render(); + + // As elasticsearch does need some doubles to be send as doubles. + if (is_numeric($value)) { + $value = (float) $value; + } + }, $searchRequest); + + return $array; + } + + /** + * Will check all entries, whether they have a condition and filter entries out, where condition is false. + * Also will remove condition in the end. + */ + public function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !is_array($entry) + || !array_key_exists('condition', $entry) + || (bool) $entry['condition'] === true + ; + }); + + foreach ($entries as $key => $entry) { + if (is_array($entry) && array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } +} diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index d5f3669..f73372d 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -21,13 +21,13 @@ namespace Codappix\SearchCore\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Utility\ArrayUtility; -use TYPO3\CMS\Fluid\View\StandaloneView; class QueryFactory { @@ -41,12 +41,19 @@ class QueryFactory */ protected $configuration; + /** + * @var ConfigurationUtility + */ + protected $configurationUtility; + public function __construct( \TYPO3\CMS\Core\Log\LogManager $logManager, - ConfigurationContainerInterface $configuration + ConfigurationContainerInterface $configuration, + ConfigurationUtility $configurationUtility ) { $this->logger = $logManager->getLogger(__CLASS__); $this->configuration = $configuration; + $this->configurationUtility = $configurationUtility; } /** @@ -164,8 +171,8 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); - $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $scriptFields = $this->filterByCondition($scriptFields); + $scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); + $scriptFields = $this->configurationUtility->filterByCondition($scriptFields); if ($scriptFields !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); } @@ -177,8 +184,8 @@ class QueryFactory protected function addSort(SearchRequestInterface $searchRequest, array &$query) { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; - $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $sorting = $this->filterByCondition($sorting); + $sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting); + $sorting = $this->configurationUtility->filterByCondition($sorting); if ($sorting !== []) { $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); } @@ -243,36 +250,4 @@ class QueryFactory ]); } } - - protected function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array - { - array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) { - $template = new StandaloneView(); - $template->assign('request', $searchRequest); - $template->setTemplateSource($value); - $value = $template->render(); - - // As elasticsearch does need some doubles to be end as doubles. - if (is_numeric($value)) { - $value = (float) $value; - } - }, $searchRequest); - - return $array; - } - - protected function filterByCondition(array $entries) : array - { - $entries = array_filter($entries, function ($entry) { - return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; - }); - - foreach ($entries as $key => $entry) { - if (array_key_exists('condition', $entry)) { - unset($entries[$key]['condition']); - } - } - - return $entries; - } } diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index c11e74e..a8ac167 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -24,6 +24,21 @@ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; abstract class AbstractUnitTestCase extends CoreTestCase { + public function setUp() + { + parent::setUp(); + + // Disable caching backends to make TYPO3 parts work in unit test mode. + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( + \TYPO3\CMS\Core\Cache\CacheManager::class + )->setCacheConfigurations([ + 'extbase_object' => [ + 'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class, + ], + ]); + } + /** * @return \TYPO3\CMS\Core\Log\LogManager */ diff --git a/Tests/Unit/Configuration/ConfigurationUtilityTest.php b/Tests/Unit/Configuration/ConfigurationUtilityTest.php new file mode 100644 index 0000000..b2dc5f8 --- /dev/null +++ b/Tests/Unit/Configuration/ConfigurationUtilityTest.php @@ -0,0 +1,143 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationUtility; +use Codappix\SearchCore\Connection\SearchRequestInterface; +use Codappix\SearchCore\Domain\Model\SearchRequest; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class ConfigurationUtilityTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider possibleRequestAndConfigurationForFluidtemplate + */ + public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->replaceArrayValuesWithRequestContent($searchRequest, $array), + 'Entries in array were not parsed as fluid template with search request.' + ); + } + + public function possibleRequestAndConfigurationForFluidtemplate() : array + { + return [ + 'Nothing in array' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [], + 'expected' => [], + ], + 'Small array with nothing to replace' => [ + 'searchRequest' => new SearchRequest(), + 'array' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Rescursive array with replacements' => [ + 'searchRequest' => call_user_func(function () { + $request = new SearchRequest(); + $request->setFilter([ + 'distance' => [ + 'location' => '10', + ], + ]); + return $request; + }), + 'array' => [ + 'sub1' => [ + 'sub1.1' => '{request.filter.distance.location}', + 'sub1.2' => '{request.nonExisting}', + ], + ], + 'expected' => [ + 'sub1' => [ + // Numberics are casted to double + 'sub1.1' => 10.0, + 'sub1.2' => null, + ], + ], + ], + ]; + } + + /** + * @test + * @dataProvider possibleConditionEntries + */ + public function conditionsAreHandledAsExpected(array $entries, array $expected) + { + $subject = new ConfigurationUtility(); + + $this->assertSame( + $expected, + $subject->filterByCondition($entries), + 'Conditions were not processed as expected.' + ); + } + + public function possibleConditionEntries() : array + { + return [ + 'Nothing in array' => [ + 'entries' => [], + 'expected' => [], + ], + 'Entries without condition' => [ + 'entries' => [ + 'key1' => 'value1', + ], + 'expected' => [ + 'key1' => 'value1', + ], + ], + 'Entry with matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => true, + 'sub1.2' => 'something', + ], + ], + 'expected' => [ + 'sub1' => [ + 'sub1.2' => 'something', + ], + ], + ], + 'Entry with non matching condition' => [ + 'entries' => [ + 'sub1' => [ + 'condition' => false, + 'sub1.2' => 'something', + ], + ], + 'expected' => [], + ], + ]; + } +} diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index baf239f..3b70679 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\ConfigurationUtility; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; @@ -44,7 +45,8 @@ class QueryFactoryTest extends AbstractUnitTestCase parent::setUp(); $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); - $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); + $configurationUtility = new ConfigurationUtility(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility); } /** From b5225b943c3efd21d90c319677226eaebfb0bd07 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 14:16:16 +0100 Subject: [PATCH 17/33] TASK: Add tests for new query factory code --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 3b70679..6301cdd 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -363,4 +363,210 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Empty search request does not create expected query.' ); } + + /** + * @test + */ + public function storedFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + '_source, something,nothing', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + ['_source', 'something', 'nothing'], + $query->toArray()['stored_fields'], + 'Stored fields were not added to query as expected.' + ); + } + + /** + * @test + */ + public function storedFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertFalse( + isset($query->toArray()['stored_fields']), + 'Stored fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function scriptFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ], + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['script_fields'], + 'Script fields were not added to query as expected.' + ); + + } + + /** + * @test + */ + public function scriptFieldsAreNotAddedToQuery() + { + $searchRequest = new SearchRequest(); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['script_fields']), + 'Script fields were added to query even if not configured.' + ); + } + + /** + * @test + */ + public function sortIsAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => '{request.query}', + ], + ] + )); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'field1' => [ + 'config' => 'something', + ], + 'field2' => [ + 'config' => 'query value', + ], + ], + $query->toArray()['sort'], + 'Sort was not added to query as expected.' + ); + } + + /** + * @test + */ + public function sortIsNotAddedToQuery() + { + $searchRequest = new SearchRequest('query value'); + + $this->configuration->expects($this->any()) + ->method('getIfExists') + ->withConsecutive( + ['searching.minimumShouldMatch'], + ['searching.sort'] + ) + ->will($this->onConsecutiveCalls( + null, + null + )); + + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + + $query = $this->subject->create($searchRequest); + $this->assertTrue( + !isset($query->toArray()['sort']), + 'Sort was added to query even if not configured.' + ); + } } From b0eccc241d6d5bb89446b16f4701607f4051ae43 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 5 Sep 2017 20:30:34 +0200 Subject: [PATCH 18/33] TASK: Allow dev to be required as 1.0 Until we release first stable version. --- composer.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/composer.json b/composer.json index 0061751..985be01 100644 --- a/composer.json +++ b/composer.json @@ -36,6 +36,9 @@ ] }, "extra": { + "branch-alias": { + "dev-develop": "1.0.x-dev" + }, "typo3/cms": { "cms-package-dir": "{$vendor-dir}/typo3/cms", "web-dir": ".Build/web" From 8206a1ec59272d54f3d68f14d3323c7a4bfe541c Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 17:08:33 +0100 Subject: [PATCH 19/33] BUGFIX: Do not remove submitted filter if configured one is empty This will be the case if you add a flexform to the plugin with no value. Then an empty filter is configured and you will not be able to submit a value for this filter. --- Classes/Domain/Search/SearchService.php | 15 ++++++++--- .../Unit/Domain/Search/SearchServiceTest.php | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/Classes/Domain/Search/SearchService.php b/Classes/Domain/Search/SearchService.php index 384f6aa..120ab3b 100644 --- a/Classes/Domain/Search/SearchService.php +++ b/Classes/Domain/Search/SearchService.php @@ -26,6 +26,7 @@ use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\SearchRequestInterface; use Codappix\SearchCore\Connection\SearchResultInterface; use Codappix\SearchCore\Domain\Model\FacetRequest; +use TYPO3\CMS\Core\Utility\ArrayUtility; use TYPO3\CMS\Extbase\Object\ObjectManagerInterface; /** @@ -123,10 +124,16 @@ class SearchService protected function addConfiguredFilters(SearchRequestInterface $searchRequest) { try { - $searchRequest->setFilter(array_merge( - $searchRequest->getFilter(), - $this->configuration->get('searching.filter') - )); + $filter = $searchRequest->getFilter(); + + ArrayUtility::mergeRecursiveWithOverrule( + $filter, + $this->configuration->get('searching.filter'), + true, + false + ); + + $searchRequest->setFilter($filter); } catch (InvalidArgumentException $e) { // Nothing todo, no filter configured. } diff --git a/Tests/Unit/Domain/Search/SearchServiceTest.php b/Tests/Unit/Domain/Search/SearchServiceTest.php index 9149e51..c63e29e 100644 --- a/Tests/Unit/Domain/Search/SearchServiceTest.php +++ b/Tests/Unit/Domain/Search/SearchServiceTest.php @@ -178,4 +178,29 @@ class SearchServiceTest extends AbstractUnitTestCase $searchRequest->setFilter(['anotherProperty' => 'anything']); $this->subject->search($searchRequest); } + + /** + * @test + */ + public function emptyConfiguredFilterIsNotChangingRequestWithExistingFilter() + { + $this->configuration->expects($this->exactly(2)) + ->method('getIfExists') + ->withConsecutive(['searching.size'], ['searching.facets']) + ->will($this->onConsecutiveCalls(null, null)); + $this->configuration->expects($this->exactly(1)) + ->method('get') + ->with('searching.filter') + ->willReturn(['anotherProperty' => '']); + + $this->connection->expects($this->once()) + ->method('search') + ->with($this->callback(function ($searchRequest) { + return $searchRequest->getFilter() === ['anotherProperty' => 'anything']; + })); + + $searchRequest = new SearchRequest('SearchWord'); + $searchRequest->setFilter(['anotherProperty' => 'anything']); + $this->subject->search($searchRequest); + } } From 015931518380b2b3b7a429a3b99ddcd453f75f61 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:20:37 +0100 Subject: [PATCH 20/33] FEATURE: Add data processor to remove fields for indexing Add a new processor, with docs and tests, to allow removal of fields before sending them to search service like elasticsearch. E.g. remove sensitive information that should not be available. --- Classes/DataProcessing/RemoveProcessor.php | 44 +++++++ .../dataProcessing/RemoveProcessor.rst | 23 ++++ .../source/configuration/indexing.rst | 1 + .../DataProcessing/RemoveProcessorTest.php | 122 ++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 Classes/DataProcessing/RemoveProcessor.php create mode 100644 Documentation/source/configuration/dataProcessing/RemoveProcessor.rst create mode 100644 Tests/Unit/DataProcessing/RemoveProcessorTest.php diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php new file mode 100644 index 0000000..b5ab788 --- /dev/null +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -0,0 +1,44 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Core\Utility\GeneralUtility; + +/** + * Removes fields from record. + */ +class RemoveProcessor implements ProcessorInterface +{ + public function processRecord(array $record, array $configuration) : array + { + if (!isset($configuration['fields'])) { + return $record; + } + + foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { + if (isset($record[$field])) { + unset($record[$field]); + } + } + + return $record; + } +} diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst new file mode 100644 index 0000000..197f3c1 --- /dev/null +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -0,0 +1,23 @@ +``Codappix\SearchCore\DataProcessing\RemoveProcessor`` +====================================================== + +Will remove fields from record, e.g. if you do not want to sent them to elasticsearch at all. + +Possible Options: + +``fields`` + Comma separated list of fields to remove from record. + +Example:: + + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\RemoveProcessor + 1 { + fields = description + } + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + fields = description, another_field + } + } + diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 93eff19..b605008 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -180,6 +180,7 @@ The following Processor are available: :glob: dataProcessing/CopyToProcessor + dataProcessing/RemoveProcessor dataProcessing/GeoPointProcessor The following Processor are planned: diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php new file mode 100644 index 0000000..57600f0 --- /dev/null +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -0,0 +1,122 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\DataProcessing\RemoveProcessor; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class RemoveProcessorTest extends AbstractUnitTestCase +{ + /** + * @test + * @dataProvider getPossibleRecordConfigurationCombinations + */ + public function fieldsAreCopiedAsConfigured(array $record, array $configuration, array $expectedRecord) + { + $subject = new RemoveProcessor(); + $processedRecord = $subject->processRecord($record, $configuration); + $this->assertSame( + $expectedRecord, + $processedRecord, + 'The processor did not return the expected processed record.' + ); + } + + /** + * @return array + */ + public function getPossibleRecordConfigurationCombinations() + { + return [ + 'Nothing configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Single field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + 'Non existing field configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + 'configuration' => [ + 'fields' => 'non existing', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + ], + ], + 'Multiple fields configured' => [ + 'record' => [ + 'field 1' => 'Some content like lorem', + 'field with sub2' => [ + 'Tag 1', + 'Tag 2', + ], + 'field 3' => 'Some more like lorem', + ], + 'configuration' => [ + 'fields' => 'field 3, field with sub2', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + 'field 1' => 'Some content like lorem', + ], + ], + ]; + } +} From 379dddf8ac6f7e4bcb526f9f6627524398ecd5ff Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:36:04 +0100 Subject: [PATCH 21/33] BUGFIX: Also remove fields containing "null" --- Classes/DataProcessing/RemoveProcessor.php | 2 +- Tests/Unit/DataProcessing/RemoveProcessorTest.php | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Classes/DataProcessing/RemoveProcessor.php b/Classes/DataProcessing/RemoveProcessor.php index b5ab788..6e74237 100644 --- a/Classes/DataProcessing/RemoveProcessor.php +++ b/Classes/DataProcessing/RemoveProcessor.php @@ -34,7 +34,7 @@ class RemoveProcessor implements ProcessorInterface } foreach (GeneralUtility::trimExplode(',', $configuration['fields'], true) as $field) { - if (isset($record[$field])) { + if (array_key_exists($field, $record)) { unset($record[$field]); } } diff --git a/Tests/Unit/DataProcessing/RemoveProcessorTest.php b/Tests/Unit/DataProcessing/RemoveProcessorTest.php index 57600f0..dc55f73 100644 --- a/Tests/Unit/DataProcessing/RemoveProcessorTest.php +++ b/Tests/Unit/DataProcessing/RemoveProcessorTest.php @@ -117,6 +117,17 @@ class RemoveProcessorTest extends AbstractUnitTestCase 'field 1' => 'Some content like lorem', ], ], + 'Fields with "null" san be removed' => [ + 'record' => [ + 'field 1' => null, + ], + 'configuration' => [ + 'fields' => 'field 1', + '_typoScriptNodeValue' => 'Codappix\SearchCore\DataProcessing\RemoveProcessor', + ], + 'expectedRecord' => [ + ], + ], ]; } } From ddb95e8c9111bcec5679b130af4b2db8c5a7a24b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 20:37:28 +0100 Subject: [PATCH 22/33] BUGFIX: Fix documentation for remove processor --- .../source/configuration/dataProcessing/RemoveProcessor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst index 197f3c1..c8653a5 100644 --- a/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst +++ b/Documentation/source/configuration/dataProcessing/RemoveProcessor.rst @@ -15,7 +15,7 @@ Example:: 1 { fields = description } - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 = Codappix\SearchCore\DataProcessing\RemoveProcessor 2 { fields = description, another_field } From e1a14b2f049f016f915d3417021c7a81b15d2a0f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 8 Nov 2017 21:05:53 +0100 Subject: [PATCH 23/33] !!!|FEATURE: Make data processing available to all indexer Before data processing was applied for TCA only, through tca table service. Now it's applied much later in process and in abstract indexer. Therefore all indexer will run data processing if configured. --- Classes/Domain/Index/AbstractIndexer.php | 29 ++++- .../Index/TcaIndexer/TcaTableService.php | 20 --- .../Unit/Domain/Index/AbstractIndexerTest.php | 121 ++++++++++++++++++ .../Index/TcaIndexer/TcaTableServiceTest.php | 73 ----------- 4 files changed, 149 insertions(+), 94 deletions(-) create mode 100644 Tests/Unit/Domain/Index/AbstractIndexerTest.php diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..3b5d077 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -23,7 +23,8 @@ namespace Codappix\SearchCore\Domain\Index; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; -use \TYPO3\CMS\Core\Utility\GeneralUtility; +use Codappix\SearchCore\DataProcessing\ProcessorInterface; +use TYPO3\CMS\Core\Utility\GeneralUtility; abstract class AbstractIndexer implements IndexerInterface { @@ -122,6 +123,32 @@ abstract class AbstractIndexer implements IndexerInterface * @param array &$record */ protected function prepareRecord(array &$record) + { + try { + foreach ($this->configuration->get('indexing.' . $this->identifier . '.dataProcessing') as $configuration) { + $className = ''; + if (is_string($configuration)) { + $className = $configuration; + $configuration = []; + } else { + $className = $configuration['_typoScriptNodeValue']; + } + $dataProcessor = GeneralUtility::makeInstance($className); + if ($dataProcessor instanceof ProcessorInterface) { + $record = $dataProcessor->processRecord($record, $configuration); + } + } + } catch (InvalidArgumentException $e) { + // Nothing to do. + } + + $this->handleAbstract($record); + } + + /** + * @param array &$record + */ + protected function handleAbstract(array &$record) { $record['search_abstract'] = ''; diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index f4fbd96..7a322a0 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; -use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -141,31 +140,12 @@ class TcaTableService } /** - * Adjust record accordingly to configuration. * @param array &$record */ public function prepareRecord(array &$record) : void { $this->relationResolver->resolveRelationsForRecord($this, $record); - try { - foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { - $className = ''; - if (is_string($configuration)) { - $className = $configuration; - $configuration = []; - } else { - $className = $configuration['_typoScriptNodeValue']; - } - $dataProcessor = GeneralUtility::makeInstance($className); - if ($dataProcessor instanceof ProcessorInterface) { - $record = $dataProcessor->processRecord($record, $configuration); - } - } - } catch (InvalidConfigurationArgumentException $e) { - // Nothing to do. - } - if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; } diff --git a/Tests/Unit/Domain/Index/AbstractIndexerTest.php b/Tests/Unit/Domain/Index/AbstractIndexerTest.php new file mode 100644 index 0000000..3bbc97d --- /dev/null +++ b/Tests/Unit/Domain/Index/AbstractIndexerTest.php @@ -0,0 +1,121 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; +use Codappix\SearchCore\Connection\ConnectionInterface; +use Codappix\SearchCore\DataProcessing\CopyToProcessor; +use Codappix\SearchCore\Domain\Index\AbstractIndexer; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; + +class AbstractIndexerTest extends AbstractUnitTestCase +{ + /** + * @var TcaTableService + */ + protected $subject; + + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + + /** + * @var ConnectionInterface + */ + protected $connection; + + public function setUp() + { + parent::setUp(); + + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->connection = $this->getMockBuilder(ConnectionInterface::class)->getMock(); + + $this->subject = $this->getMockForAbstractClass(AbstractIndexer::class, [ + $this->connection, + $this->configuration + ]); + $this->subject->injectLogger($this->getMockedLogger()); + $this->subject->setIdentifier('testTable'); + $this->subject->expects($this->any()) + ->method('getDocumentName') + ->willReturn('testTable'); + } + + /** + * @test + */ + public function executesConfiguredDataProcessingWithConfiguration() + { + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['new_test_field'] = 'test'; + $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; + $expectedRecord['search_abstract'] = ''; + + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) + ->will($this->onConsecutiveCalls([ + '1' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field', + ], + '2' => [ + '_typoScriptNodeValue' => CopyToProcessor::class, + 'to' => 'new_test_field2', + ], + ], $this->throwException(new InvalidArgumentException))); + $this->subject->expects($this->once()) + ->method('getRecord') + ->with(1) + ->willReturn($record) + ; + + $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); + $this->subject->indexDocument(1); + } + + /** + * @test + */ + public function executesNoDataProcessingForMissingConfiguration() + { + $record = ['field 1' => 'test']; + $expectedRecord = $record; + $expectedRecord['search_abstract'] = ''; + + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['indexing.testTable.dataProcessing'], ['indexing.testTable.abstractFields']) + ->will($this->throwException(new InvalidArgumentException)); + $this->subject->expects($this->once()) + ->method('getRecord') + ->with(1) + ->willReturn($record) + ; + + $this->connection->expects($this->once())->method('addDocument')->with('testTable', $expectedRecord); + $this->subject->indexDocument(1); + } +} diff --git a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php index dd0a389..edad84a 100644 --- a/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php +++ b/Tests/Unit/Domain/Index/TcaIndexer/TcaTableServiceTest.php @@ -148,77 +148,4 @@ class TcaTableServiceTest extends AbstractUnitTestCase ); unset($GLOBALS['TCA']['test_table']); } - - /** - * @test - */ - public function executesConfiguredDataProcessingWithConfiguration() - { - $this->configuration->expects($this->exactly(1)) - ->method('get') - ->with('indexing.testTable.dataProcessing') - ->will($this->returnValue([ - '1' => [ - '_typoScriptNodeValue' => CopyToProcessor::class, - 'to' => 'new_test_field', - ], - '2' => [ - '_typoScriptNodeValue' => CopyToProcessor::class, - 'to' => 'new_test_field2', - ], - ])); - - $subject = $this->getMockBuilder(TcaTableService::class) - ->disableOriginalConstructor() - ->setMethodsExcept(['prepareRecord']) - ->getMock(); - $this->inject($subject, 'configuration', $this->configuration); - $this->inject($subject, 'tableName', 'testTable'); - $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); - - $record = ['field 1' => 'test']; - $expectedRecord = $record; - $expectedRecord['new_test_field'] = 'test'; - $expectedRecord['new_test_field2'] = 'test' . PHP_EOL . 'test'; - - $subject->prepareRecord($record); - - $this->assertSame( - $expectedRecord, - $record, - 'Dataprocessing is not executed by TcaTableService as expected.' - ); - } - - /** - * @test - */ - public function executesConfiguredDataProcessingWithoutConfiguration() - { - $this->configuration->expects($this->exactly(1)) - ->method('get') - ->with('indexing.testTable.dataProcessing') - ->will($this->returnValue([CopyToProcessor::class])); - - $subject = $this->getMockBuilder(TcaTableService::class) - ->disableOriginalConstructor() - ->setMethodsExcept(['prepareRecord']) - ->getMock(); - $this->inject($subject, 'configuration', $this->configuration); - $this->inject($subject, 'tableName', 'testTable'); - $this->inject($subject, 'relationResolver', $this->getMockBuilder(RelationResolver::class)->getMock()); - - $record = ['field 1' => 'test']; - $expectedRecord = $record; - $expectedRecord[''] = 'test'; - $expectedRecord['search_title'] = 'test'; - - $subject->prepareRecord($record); - - $this->assertSame( - $expectedRecord, - $record, - 'Dataprocessing is not executed by TcaTableService as expected.' - ); - } } From 31202f88821d1691d8886d50461ad55ad188edee Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 10 Nov 2017 12:31:06 +0100 Subject: [PATCH 24/33] FEATURE: Provide form finisher for integration into form extension Provide a finisher, working as a proxy, to internal data handler, which is already used for Hooks in TYPO3 backend. --- .../Form/Finisher/DataHandlerFinisher.php | 71 +++++++++ Documentation/source/features.rst | 2 + Documentation/source/usage.rst | 28 ++++ Tests/Unit/AbstractUnitTestCase.php | 37 +++++ .../Form/Finisher/DataHandlerFinisherTest.php | 140 ++++++++++++++++++ 5 files changed, 278 insertions(+) create mode 100644 Classes/Integration/Form/Finisher/DataHandlerFinisher.php create mode 100644 Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php diff --git a/Classes/Integration/Form/Finisher/DataHandlerFinisher.php b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php new file mode 100644 index 0000000..02d6bde --- /dev/null +++ b/Classes/Integration/Form/Finisher/DataHandlerFinisher.php @@ -0,0 +1,71 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Form\Domain\Finishers\AbstractFinisher; +use TYPO3\CMS\Form\Domain\Finishers\Exception\FinisherException; + +/** + * Integrates search_core indexing into TYPO3 Form extension. + * + * Add this finisher AFTER all database operations, as search_core will fetch + * information from database. + */ +class DataHandlerFinisher extends AbstractFinisher +{ + /** + * @var \Codappix\SearchCore\Domain\Service\DataHandler + * @inject + */ + protected $dataHandler; + + /** + * @var array + */ + protected $defaultOptions = [ + 'indexIdentifier' => null, + 'recordUid' => null, + 'action' => '', + ]; + + protected function executeInternal() + { + $action = $this->parseOption('action'); + $record = ['uid' => (int) $this->parseOption('recordUid')]; + $tableName = $this->parseOption('indexIdentifier'); + + if ($action === '' || $tableName === '' || !is_string($tableName) || $record['uid'] === 0) { + throw new FinisherException('Not all necessary options were set.', 1510313095); + } + + switch ($action) { + case 'update': + $this->dataHandler->update($tableName, $record); + break; + case 'add': + $this->dataHandler->add($tableName, $record); + break; + case 'delete': + $this->dataHandler->delete($tableName, $record['uid']); + break; + } + } +} diff --git a/Documentation/source/features.rst b/Documentation/source/features.rst index 199192a..8baf453 100644 --- a/Documentation/source/features.rst +++ b/Documentation/source/features.rst @@ -15,6 +15,8 @@ configuration needs. Still it's possible to configure the indexer. Also custom classes can be used as indexers. +Furthermore a finisher for TYPO3 Form-Extension is provided to integrate indexing. + .. _features_search: Searching diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 1ca50d2..fd94615 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -30,6 +30,34 @@ The tables have to be configured via :ref:`configuration_options_index`. Not all hook operations are supported yet, see :issue:`27`. +.. _usage_form_finisher: + +Form finisher +------------- + +A form finisher is provided to integrate indexing into form extension. + +Add form finisher to your available finishers and configure it like: + +.. code-block:: yaml + :linenos: + + - + identifier: SearchCoreIndexer + options: + action: 'delete' + indexIdentifier: 'fe_users' + recordUid: '{FeUser.user.uid}' + +All three options are necessary, where + +``action`` + Is one of ``delete``, ``update`` or ``add``. +``indexIdentifier`` + Is a configured index identifier. +``recordUid`` + Has to be the uid of the record to index. + .. _usage_searching: Searching / Frontend Plugin diff --git a/Tests/Unit/AbstractUnitTestCase.php b/Tests/Unit/AbstractUnitTestCase.php index a8ac167..c21209d 100644 --- a/Tests/Unit/AbstractUnitTestCase.php +++ b/Tests/Unit/AbstractUnitTestCase.php @@ -21,13 +21,23 @@ namespace Codappix\SearchCore\Tests\Unit; */ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase; +use TYPO3\CMS\Core\Utility\GeneralUtility; +use TYPO3\CMS\Extbase\Object\ObjectManager; +use TYPO3\CMS\Form\Service\TranslationService; abstract class AbstractUnitTestCase extends CoreTestCase { + /** + * @var array A backup of registered singleton instances + */ + protected $singletonInstances = []; + public function setUp() { parent::setUp(); + $this->singletonInstances = GeneralUtility::getSingletonInstances(); + // Disable caching backends to make TYPO3 parts work in unit test mode. \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance( @@ -39,6 +49,12 @@ abstract class AbstractUnitTestCase extends CoreTestCase ]); } + public function tearDown() + { + GeneralUtility::resetSingletonInstances($this->singletonInstances); + parent::tearDown(); + } + /** * @return \TYPO3\CMS\Core\Log\LogManager */ @@ -58,4 +74,25 @@ abstract class AbstractUnitTestCase extends CoreTestCase return $logger; } + + /** + * Configure translation service mock for Form Finisher. + * + * This way parseOption will always return the configured value. + */ + protected function configureMockedTranslationService() + { + $translationService = $this->getMockBuilder(TranslationService::class)->getMock(); + $translationService->expects($this->any()) + ->method('translateFinisherOption') + ->willReturnCallback(function ($formRuntime, $finisherIdentifier, $optionKey, $optionValue) { + return $optionValue; + }); + $objectManager = $this->getMockBuilder(ObjectManager::class)->getMock(); + $objectManager->expects($this->any()) + ->method('get') + ->with(TranslationService::class) + ->willReturn($translationService); + GeneralUtility::setSingletonInstance(ObjectManager::class, $objectManager); + } } diff --git a/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php new file mode 100644 index 0000000..2211cc2 --- /dev/null +++ b/Tests/Unit/Integration/Form/Finisher/DataHandlerFinisherTest.php @@ -0,0 +1,140 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Service\DataHandler; +use Codappix\SearchCore\Integration\Form\Finisher\DataHandlerFinisher; +use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase; +use TYPO3\CMS\Form\Domain\Finishers\Exception\FinisherException; +use TYPO3\CMS\Form\Domain\Finishers\FinisherContext; + +class DataHandlerFinisherTest extends AbstractUnitTestCase +{ + /** + * @var DataHandlerFinisher + */ + protected $subject; + + /** + * @var DataHandler + */ + protected $dataHandlerMock; + + /** + * @var FinisherContext + */ + protected $finisherContextMock; + + public function setUp() + { + parent::setUp(); + + $this->configureMockedTranslationService(); + $this->dataHandlerMock = $this->getMockBuilder(DataHandler::class) + ->disableOriginalConstructor() + ->getMock(); + $this->finisherContextMock = $this->getMockBuilder(FinisherContext::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->subject = new DataHandlerFinisher(); + $this->inject($this->subject, 'dataHandler', $this->dataHandlerMock); + } + + /** + * @test + * @dataProvider possibleFinisherSetup + */ + public function validConfiguration(string $action, array $nonCalledActions, $expectedSecondArgument) + { + $this->subject->setOptions([ + 'indexIdentifier' => 'test_identifier', + 'recordUid' => '23', + 'action' => $action, + ]); + + foreach ($nonCalledActions as $nonCalledAction) { + $this->dataHandlerMock->expects($this->never())->method($nonCalledAction); + } + $this->dataHandlerMock->expects($this->once())->method($action) + ->with('test_identifier', $expectedSecondArgument); + + $this->subject->execute($this->finisherContextMock); + } + + public function possibleFinisherSetup() : array + { + return [ + 'valid add configuration' => [ + 'action' => 'add', + 'nonCalledActions' => ['delete', 'update'], + 'expectedSecondArgument' => ['uid' => 23], + ], + 'valid update configuration' => [ + 'action' => 'update', + 'nonCalledActions' => ['delete', 'add'], + 'expectedSecondArgument' => ['uid' => 23], + ], + 'valid delete configuration' => [ + 'action' => 'delete', + 'nonCalledActions' => ['update', 'add'], + 'expectedSecondArgument' => 23, + ], + ]; + } + + /** + * @test + * @dataProvider invalidFinisherSetup + */ + public function nothingHappensIfUnknownActionIsConfigured(array $options) + { + $this->subject->setOptions($options); + + foreach (['add', 'update', 'delete'] as $nonCalledAction) { + $this->dataHandlerMock->expects($this->never())->method($nonCalledAction); + } + + $this->expectException(FinisherException::class); + $this->subject->execute($this->finisherContextMock); + } + + public function invalidFinisherSetup() : array + { + return [ + 'missing options' => [ + 'options' => [], + ], + 'missing action option' => [ + 'options' => [ + 'indexIdentifier' => 'identifier', + 'recordUid' => '20', + ], + ], + 'missing record uid option' => [ + 'options' => [ + 'indexIdentifier' => 'identifier', + 'action' => 'update', + ], + ], + ]; + } +} From a3a46f5cb5b942ae53356ac3092ab781039cb815 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 10 Nov 2017 13:22:15 +0100 Subject: [PATCH 25/33] FEATURE: Provide command to delete whole index This is necessary, e.g. for complete re-indexing. --- Classes/Command/IndexCommandController.php | 16 +++++- Classes/Connection/ConnectionInterface.php | 9 ++++ Classes/Connection/Elasticsearch.php | 12 +++++ Classes/Domain/Index/AbstractIndexer.php | 7 +++ Classes/Domain/Index/IndexerInterface.php | 11 +++- Documentation/source/usage.rst | 13 +++++ .../Elasticsearch/IndexDeletionTest.php | 50 +++++++++++++++++++ .../Command/IndexCommandControllerTest.php | 37 ++++++++++++++ 8 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php diff --git a/Classes/Command/IndexCommandController.php b/Classes/Command/IndexCommandController.php index b14d4b3..70816ed 100644 --- a/Classes/Command/IndexCommandController.php +++ b/Classes/Command/IndexCommandController.php @@ -50,7 +50,6 @@ class IndexCommandController extends CommandController */ public function indexCommand($identifier) { - // TODO: Also allow to index everything? try { $this->indexerFactory->getIndexer($identifier)->indexAllDocuments(); $this->outputLine($identifier . ' was indexed.'); @@ -58,4 +57,19 @@ class IndexCommandController extends CommandController $this->outputLine('No indexer found for: ' . $identifier); } } + + /** + * Will delete the given identifier. + * + * @param string $identifier + */ + public function deleteCommand($identifier) + { + try { + $this->indexerFactory->getIndexer($identifier)->delete(); + $this->outputLine($identifier . ' was deleted.'); + } catch (NoMatchingIndexerException $e) { + $this->outputLine('No indexer found for: ' . $identifier); + } + } } diff --git a/Classes/Connection/ConnectionInterface.php b/Classes/Connection/ConnectionInterface.php index a130f9e..59cf9f8 100644 --- a/Classes/Connection/ConnectionInterface.php +++ b/Classes/Connection/ConnectionInterface.php @@ -77,4 +77,13 @@ interface ConnectionInterface * @return SearchResultInterface */ public function search(SearchRequestInterface $searchRequest); + + /** + * Will delete the whole index / db. + * + * @param string $documentType + * + * @return void + */ + public function deleteIndex($documentType); } diff --git a/Classes/Connection/Elasticsearch.php b/Classes/Connection/Elasticsearch.php index 4c66b6a..8a3cb2b 100644 --- a/Classes/Connection/Elasticsearch.php +++ b/Classes/Connection/Elasticsearch.php @@ -156,6 +156,18 @@ class Elasticsearch implements Singleton, ConnectionInterface ); } + public function deleteIndex($documentType) + { + $index = $this->connection->getClient()->getIndex('typo3content'); + + if (! $index->exists()) { + $this->logger->notice('Index did not exist, therefore was not deleted.', [$documentType, 'typo3content']); + return; + } + + $index->delete(); + } + /** * Execute given callback with Elastica Type based on provided documentType * diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..0b4d675 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -104,6 +104,13 @@ abstract class AbstractIndexer implements IndexerInterface $this->logger->info('Finish indexing'); } + public function delete() + { + $this->logger->info('Start deletion of index.'); + $this->connection->deleteIndex($this->getDocumentName()); + $this->logger->info('Finish deletion.'); + } + /** * @return \Generator */ diff --git a/Classes/Domain/Index/IndexerInterface.php b/Classes/Domain/Index/IndexerInterface.php index 5a4ca6c..72ebb9d 100644 --- a/Classes/Domain/Index/IndexerInterface.php +++ b/Classes/Domain/Index/IndexerInterface.php @@ -33,9 +33,9 @@ interface IndexerInterface public function indexAllDocuments(); /** - * Fetches a single document from the indexerService and pushes it to the connection. + * Fetches a single document and pushes it to the connection. * - * @param string $identifier identifier, the indexer needs to identify a single document + * @param string $identifier * * @return void */ @@ -49,4 +49,11 @@ interface IndexerInterface * @return void */ public function setIdentifier($identifier); + + /** + * Delete the whole index. + * + * @return void + */ + public function delete(); } diff --git a/Documentation/source/usage.rst b/Documentation/source/usage.rst index 1ca50d2..fb98a99 100644 --- a/Documentation/source/usage.rst +++ b/Documentation/source/usage.rst @@ -18,6 +18,19 @@ This will index the table ``tt_content`` using the :ref:`TcaIndexer`. Only one index per call is available, to run multiple indexers, just make multiple calls. The indexers have to be defined in TypoScript via :ref:`configuration_options_index`. +.. _usage_manual_deletion: + +Manual deletion +--------------- + +You can trigger deletion for a single index from CLI:: + + ./typo3/cli_dispatch.phpsh extbase index:delete --identifier 'tt_content' + +This will delete the index for the table ``tt_content``. + +Only one delete per call is available, to run multiple deletions, just make multiple calls. + .. _usage_auto_indexing: Auto indexing diff --git a/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php new file mode 100644 index 0000000..8297a2a --- /dev/null +++ b/Tests/Functional/Connection/Elasticsearch/IndexDeletionTest.php @@ -0,0 +1,50 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use Codappix\SearchCore\Domain\Index\IndexerFactory; +use TYPO3\CMS\Extbase\Object\ObjectManager; + +class IndexDeletionTest extends AbstractFunctionalTestCase +{ + /** + * @test + */ + public function indexIsDeleted() + { + $this->client->getIndex('typo3content')->create(); + $this->assertTrue( + $this->client->getIndex('typo3content')->exists(), + 'Could not create index for test.' + ); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->delete() + ; + + $this->assertFalse( + $this->client->getIndex('typo3content')->exists(), + 'Index could not be deleted through command controller.' + ); + } +} diff --git a/Tests/Unit/Command/IndexCommandControllerTest.php b/Tests/Unit/Command/IndexCommandControllerTest.php index 4c81672..9dcc7f3 100644 --- a/Tests/Unit/Command/IndexCommandControllerTest.php +++ b/Tests/Unit/Command/IndexCommandControllerTest.php @@ -91,4 +91,41 @@ class IndexCommandControllerTest extends AbstractUnitTestCase $this->subject->indexCommand('allowedTable'); } + + /** + * @test + */ + public function deletionIsPossible() + { + $indexerMock = $this->getMockBuilder(TcaIndexer::class) + ->disableOriginalConstructor() + ->getMock(); + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('allowedTable was deleted.'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('allowedTable') + ->will($this->returnValue($indexerMock)); + + $indexerMock->expects($this->once()) + ->method('delete'); + $this->subject->deleteCommand('allowedTable'); + } + + /** + * @test + */ + public function deletionForNonExistingIndexerDoesNotWork() + { + $this->subject->expects($this->once()) + ->method('outputLine') + ->with('No indexer found for: nonAllowedTable'); + $this->indexerFactory->expects($this->once()) + ->method('getIndexer') + ->with('nonAllowedTable') + ->will($this->throwException(new NoMatchingIndexerException)); + + $this->subject->deleteCommand('nonAllowedTable'); + } } From 0815eaff6bb0a3deaadbcae51950df73075e036f Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 10 Nov 2017 13:47:26 +0100 Subject: [PATCH 26/33] BUGFIX: Remove records during update if no longer available E.g. update is to deactivate a record. In this case we will not be able to update the record but should delete him instead. --- Classes/Domain/Index/AbstractIndexer.php | 3 +- .../Elasticsearch/IndexTcaTableTest.php | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Classes/Domain/Index/AbstractIndexer.php b/Classes/Domain/Index/AbstractIndexer.php index 143a219..de8072d 100644 --- a/Classes/Domain/Index/AbstractIndexer.php +++ b/Classes/Domain/Index/AbstractIndexer.php @@ -99,7 +99,8 @@ abstract class AbstractIndexer implements IndexerInterface $this->connection->addDocument($this->getDocumentName(), $record); } catch (NoRecordFoundException $e) { - $this->logger->info('Could not index document.', [$e->getMessage()]); + $this->logger->info('Could not index document. Try to delete it therefore.', [$e->getMessage()]); + $this->connection->deleteDocument($this->getDocumentName(), $identifier); } $this->logger->info('Finish indexing'); } diff --git a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php index 022fb3b..b1e8191 100644 --- a/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php +++ b/Tests/Functional/Connection/Elasticsearch/IndexTcaTableTest.php @@ -205,4 +205,35 @@ class IndexTcaTableTest extends AbstractFunctionalTestCase 'Record was indexed with resolved category relation, but should not have any.' ); } + + /** + * @test + */ + public function indexingDeltedRecordIfRecordShouldBeIndexedButIsNoLongerAvailableAndWasAlreadyIndexed() + { + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexAllDocuments() + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 2, 'Not exactly 2 documents were indexed.'); + + $this->getConnectionPool()->getConnectionForTable('tt_content') + ->update( + 'tt_content', + ['hidden' => true], + ['uid' => 10] + ); + + \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(ObjectManager::class) + ->get(IndexerFactory::class) + ->getIndexer('tt_content') + ->indexDocument(10) + ; + + $response = $this->client->request('typo3content/_search?q=*:*'); + $this->assertSame($response->getData()['hits']['total'], 1, 'Not exactly 1 document is in index.'); + } } From 02ef86b67b8f2eab59528496b977ec5fe35c72f1 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 18:57:09 +0100 Subject: [PATCH 27/33] FEATURE: Provide new feature to configure fields to search in This enables you to search only in some fields. Also if some fields contain mapping, you can add them in addition to e.g. `_all`. --- Classes/Domain/Search/QueryFactory.php | 18 ++++++++---------- Configuration/TypoScript/setup.txt | 6 ++++++ .../source/configuration/searching.rst | 13 ++++++++++++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index f73372d..980b763 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -99,20 +99,18 @@ class QueryFactory return; } - $query = ArrayUtility::setValueByPath( - $query, - 'query.bool.must.0.match._all.query', - $searchRequest->getSearchTerm() - ); + $matchExpression = [ + 'type' => 'most_fields', + 'query' => $searchRequest->getSearchTerm(), + 'fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.query')), + ]; $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); if ($minimumShouldMatch) { - $query = ArrayUtility::setValueByPath( - $query, - 'query.bool.must.0.match._all.minimum_should_match', - $minimumShouldMatch - ); + $matchExpression['minimum_should_match'] = $minimumShouldMatch; } + + $query = ArrayUtility::setValueByPath($query, 'query.bool.must.0.multi_match', $matchExpression); } protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index 67612e1..1a1577f 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -21,6 +21,12 @@ plugin { abstractFields = {$plugin.tx_searchcore.settings.indexing.pages.abstractFields} } } + + searching { + fields { + query = _all + } + } } } } diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index e953dcb..ae73fad 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -151,7 +151,18 @@ filtering. This way you can use arbitrary filter names and map them to existing fields ------ -Defines the fields to fetch from elasticsearch. Two sub entries exist: +Defines the fields to fetch and search from elasticsearch. With the following sub keys: + +``query`` defines the fields to search in. Default is ``_all`` from 5.x times of elasticsearch. +Configure a comma separated list of fields to search in. This is necessary if you have configured +special mapping for some fields, or just want to search some fields. +The most hits get ranked highest. The following is an example configuration:: + + fields { + query = _all, city + } + +The following sub properties configure the fields to fetch from elasticsearch: First ``stored_fields`` which is a list of comma separated fields which actually exist and will be added. Typically you will use ``_source`` to fetch the whole indexed fields. From 0006148a525e38d5af3dc94fda56c16431b0d994 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 19:43:16 +0100 Subject: [PATCH 28/33] TASK: Fix broken functional tests Add new default TypoScript to not break tests. --- Tests/Functional/Fixtures/BasicSetup.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tests/Functional/Fixtures/BasicSetup.ts b/Tests/Functional/Fixtures/BasicSetup.ts index 1e2b3a9..b7b0c6a 100644 --- a/Tests/Functional/Fixtures/BasicSetup.ts +++ b/Tests/Functional/Fixtures/BasicSetup.ts @@ -42,6 +42,10 @@ plugin { field = CType } } + + fields { + query = _all + } } } } From e3151e802c55aec78d7ff99fa7dffc04fe5720a6 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 19:52:10 +0100 Subject: [PATCH 29/33] TASK: Fix broken unit tests Adjust tests to match new queries built with multiple fields. --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 84 ++++++++++--------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 6301cdd..ba845d4 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -56,9 +56,7 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertInstanceOf( @@ -73,9 +71,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); @@ -95,9 +91,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptyFilterIsNotAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ @@ -122,9 +116,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function facetsAreAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); @@ -153,9 +145,7 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function sizeIsAddedToQuery() { - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setLimit(45); $searchRequest->setOffset(35); @@ -179,9 +169,7 @@ class QueryFactoryTest extends AbstractUnitTestCase public function searchTermIsAddedToQuery() { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -189,9 +177,11 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'query' => 'SearchWord', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', ], ], ], @@ -219,9 +209,7 @@ class QueryFactoryTest extends AbstractUnitTestCase '50%', null )); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertArraySubset( @@ -229,10 +217,13 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'minimum_should_match' => '50%', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', ], + 'minimum_should_match' => '50%', ], ], ], @@ -253,12 +244,14 @@ class QueryFactoryTest extends AbstractUnitTestCase $this->configuration->expects($this->any()) ->method('get') ->withConsecutive( + ['searching.fields.query'], ['searching.boost'], ['searching.fields.stored_fields'], ['searching.fields.script_fields'], ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( + '_all', [ 'search_title' => 3, 'search_abstract' => 1.5, @@ -308,12 +301,14 @@ class QueryFactoryTest extends AbstractUnitTestCase $this->configuration->expects($this->any()) ->method('get') ->withConsecutive( + ['searching.fields.query'], ['searching.boost'], ['searching.fields.stored_fields'], ['searching.fields.script_fields'], ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( + '_all', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), @@ -328,9 +323,11 @@ class QueryFactoryTest extends AbstractUnitTestCase 'bool' => [ 'must' => [ [ - 'match' => [ - '_all' => [ - 'query' => 'SearchWord', + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', ], ], ], @@ -352,9 +349,7 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest(); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertInstanceOf( @@ -433,12 +428,14 @@ class QueryFactoryTest extends AbstractUnitTestCase $this->configuration->expects($this->any()) ->method('get') ->withConsecutive( + ['searching.fields.query'], ['searching.boost'], ['searching.fields.stored_fields'], ['searching.fields.script_fields'], ['searching.fieldValueFactor'] ) ->will($this->onConsecutiveCalls( + '_all', $this->throwException(new InvalidArgumentException), $this->throwException(new InvalidArgumentException), [ @@ -522,9 +519,7 @@ class QueryFactoryTest extends AbstractUnitTestCase ] )); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -559,9 +554,7 @@ class QueryFactoryTest extends AbstractUnitTestCase null )); - $this->configuration->expects($this->any()) - ->method('get') - ->will($this->throwException(new InvalidArgumentException)); + $this->configureConfigurationMockWithDefault(); $query = $this->subject->create($searchRequest); $this->assertTrue( @@ -569,4 +562,17 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Sort was added to query even if not configured.' ); } + + protected function configureConfigurationMockWithDefault() + { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->returnCallback(function ($configName) { + if ($configName === 'searching.fields.query') { + return '_all'; + } + + throw new InvalidArgumentException(); + })); + } } From 5ba860b8de7854ecb4ed76c6ae492259645bae37 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Wed, 29 Nov 2017 20:00:10 +0100 Subject: [PATCH 30/33] TASK: Add new test covering new feature --- Tests/Unit/Domain/Search/QueryFactoryTest.php | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index ba845d4..e0fc86d 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -359,6 +359,54 @@ class QueryFactoryTest extends AbstractUnitTestCase ); } + /** + * @test + */ + public function configuredQueryFieldsAreAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + + $this->configuration->expects($this->any()) + ->method('get') + ->withConsecutive( + ['searching.fields.query'], + ['searching.boost'], + ['searching.fields.stored_fields'], + ['searching.fields.script_fields'], + ['searching.fieldValueFactor'] + ) + ->will($this->onConsecutiveCalls( + '_all, field1, field2', + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException), + $this->throwException(new InvalidArgumentException) + )); + + $query = $this->subject->create($searchRequest); + $this->assertArraySubset( + [ + 'bool' => [ + 'must' => [ + [ + 'multi_match' => [ + 'type' => 'most_fields', + 'query' => 'SearchWord', + 'fields' => [ + '_all', + 'field1', + 'field2', + ], + ], + ], + ], + ], + ], + $query->toArray()['query'], + 'Configured fields were not added to query as configured.' + ); + } + /** * @test */ @@ -462,7 +510,6 @@ class QueryFactoryTest extends AbstractUnitTestCase $query->toArray()['script_fields'], 'Script fields were not added to query as expected.' ); - } /** From bdecbf9699d207cdaa387738c835837c8b14afdc Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 29 Jan 2018 22:45:53 +0100 Subject: [PATCH 31/33] TASK: Fix license for packagist --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 985be01..b704e77 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,7 @@ "type": "typo3-cms-extension", "description": "Codappix Search Core.", "homepage": "https://github.com/Codappix/search_core", - "license": ["GPL-2.0+"], + "license": ["GPL-2.0-or-later"], "autoload": { "psr-4": { "Codappix\\SearchCore\\": "Classes" From c994a32ac1f4d62ef76cdac79125d17a5e31af94 Mon Sep 17 00:00:00 2001 From: Justus Moroni Date: Mon, 29 Jan 2018 22:11:53 +0100 Subject: [PATCH 32/33] BUGFIX: Make BackendUtility usable in frontend BackendUtility used LanguageService which only works in the backend. Extend BackendUtility and use TSFE instead. --- .../Index/TcaIndexer/RelationResolver.php | 26 +++++++++---- Classes/Utility/FrontendUtility.php | 39 +++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 Classes/Utility/FrontendUtility.php diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index 88aa982..a2302ae 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ +use Codappix\SearchCore\Utility\FrontendUtility; use TYPO3\CMS\Backend\Utility\BackendUtility; use TYPO3\CMS\Core\SingletonInterface as Singleton; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -39,13 +40,15 @@ class RelationResolver implements Singleton if ($column === 'pid') { continue; } - $record[$column] = BackendUtility::getProcessedValueExtra( - $service->getTableName(), - $column, - $record[$column], - 0, - $record['uid'] - ); + + $record[$column] = GeneralUtility::makeInstance($this->getUtilityForMode()) + ::getProcessedValueExtra( + $service->getTableName(), + $column, + $record[$column], + 0, + $record['uid'] + ); try { $config = $service->getColumnConfig($column); @@ -93,4 +96,13 @@ class RelationResolver implements Singleton { return array_map('trim', explode(',', $value)); } + + protected function getUtilityForMode(): string + { + if (TYPO3_MODE === 'BE') { + return BackendUtility::class; + } + + return FrontendUtility::class; + } } diff --git a/Classes/Utility/FrontendUtility.php b/Classes/Utility/FrontendUtility.php new file mode 100644 index 0000000..1282421 --- /dev/null +++ b/Classes/Utility/FrontendUtility.php @@ -0,0 +1,39 @@ + + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use TYPO3\CMS\Backend\Utility\BackendUtility; +use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; + +/** + * Overwrite BackendUtility to use in frontend. + * LanguageService was only usable in backend. + */ +class FrontendUtility extends BackendUtility +{ + /** + * @return TypoScriptFrontendController + */ + protected static function getLanguageService() + { + return $GLOBALS['TSFE']; + } +} From a5b35c54d913c4ee308a7218d79fc3b44628e764 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 30 Jan 2018 20:54:41 +0100 Subject: [PATCH 33/33] TASK: Remove data processing from tca table service This is not the place to be. Data processing during indexing happens in abstract indexer for all indexers that make use of it. It's more generic then to TCA. --- Classes/Domain/Index/TcaIndexer/TcaTableService.php | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/TcaTableService.php b/Classes/Domain/Index/TcaIndexer/TcaTableService.php index 7faac04..d0f3031 100644 --- a/Classes/Domain/Index/TcaIndexer/TcaTableService.php +++ b/Classes/Domain/Index/TcaIndexer/TcaTableService.php @@ -22,7 +22,6 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Configuration\InvalidArgumentException as InvalidConfigurationArgumentException; -use Codappix\SearchCore\DataProcessing\ProcessorInterface; use Codappix\SearchCore\Database\Doctrine\Join; use Codappix\SearchCore\Database\Doctrine\Where; use Codappix\SearchCore\Domain\Index\IndexingException; @@ -144,17 +143,6 @@ class TcaTableService { $this->relationResolver->resolveRelationsForRecord($this, $record); - try { - foreach ($this->configuration->get('indexing.' . $this->tableName . '.dataProcessing') as $configuration) { - $dataProcessor = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance($configuration['_typoScriptNodeValue']); - if ($dataProcessor instanceof ProcessorInterface) { - $record = $dataProcessor->processRecord($record, $configuration); - } - } - } catch (InvalidConfigurationArgumentException $e) { - // Nothing to do. - } - if (isset($record['uid']) && !isset($record['search_identifier'])) { $record['search_identifier'] = $record['uid']; }