From bf91c4a5ba2005bf78d5497661e001f47e26af4a Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Sun, 29 Oct 2017 12:25:25 +0100 Subject: [PATCH] TASK: Allow fields and sorting to contain a condition This way integrators can configure when the sorting and fields should be added. --- Classes/Domain/Search/QueryFactory.php | 25 ++- .../source/configuration/connections.rst | 18 +- .../source/configuration/indexing.rst | 194 +++++++++--------- .../source/configuration/searching.rst | 192 ++++++++++++----- 4 files changed, 266 insertions(+), 163 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 56d4dd3..d5f3669 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -165,7 +165,10 @@ class QueryFactory try { $scriptFields = $this->configuration->get('searching.fields.script_fields'); $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + $scriptFields = $this->filterByCondition($scriptFields); + if ($scriptFields !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); + } } catch (InvalidArgumentException $e) { // Nothing configured } @@ -175,7 +178,10 @@ class QueryFactory { $sorting = $this->configuration->getIfExists('searching.sort') ?: []; $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); - $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + $sorting = $this->filterByCondition($sorting); + if ($sorting !== []) { + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); + } } protected function addFilter(SearchRequestInterface $searchRequest, array &$query) @@ -254,4 +260,19 @@ class QueryFactory return $array; } + + protected function filterByCondition(array $entries) : array + { + $entries = array_filter($entries, function ($entry) { + return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true; + }); + + foreach ($entries as $key => $entry) { + if (array_key_exists('condition', $entry)) { + unset($entries[$key]['condition']); + } + } + + return $entries; + } } diff --git a/Documentation/source/configuration/connections.rst b/Documentation/source/configuration/connections.rst index 841d878..5819730 100644 --- a/Documentation/source/configuration/connections.rst +++ b/Documentation/source/configuration/connections.rst @@ -29,27 +29,27 @@ The following settings are available. For each setting its documented which conn ``host`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 - installation. +The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 +installation. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost + plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost .. _port: ``port`` -------- - Used by: :ref:`Elasticsearch`. +Used by: :ref:`Elasticsearch`. - The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. +The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. - Example:: +Example:: - plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 + plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 diff --git a/Documentation/source/configuration/indexing.rst b/Documentation/source/configuration/indexing.rst index 413da3d..93eff19 100644 --- a/Documentation/source/configuration/indexing.rst +++ b/Documentation/source/configuration/indexing.rst @@ -29,18 +29,18 @@ The following settings are available. For each setting its documented which inde rootLineBlacklist ----------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Defines a blacklist of page uids. Records below any of these pages, or subpages, are not - indexed. This allows you to define areas that should not be indexed. - The page attribute *No Search* is also taken into account to prevent indexing records from only one - page without recursion. +Defines a blacklist of page uids. Records below any of these pages, or subpages, are not +indexed. This allows you to define areas that should not be indexed. +The page attribute *No Search* is also taken into account to prevent indexing records from only one +page without recursion. - Contains a comma separated list of page uids. Spaces are trimmed. +Contains a comma separated list of page uids. Spaces are trimmed. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 + plugin.tx_searchcore.settings.indexing..rootLineBlacklist = 3, 10, 100 Also it's possible to define some behaviour for the different document types. In context of TYPO3 tables are used as document types 1:1. It's possible to configure different tables. The following @@ -51,156 +51,156 @@ options are available: additionalWhereClause --------------------- - Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. +Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. - Add additional SQL to where clauses to determine indexable records from the table. This way you - can exclude specific records like ``tt_content`` records with specific ``CType`` values or - something else. E.g. you can add a new field to the table to exclude records from indexing. +Add additional SQL to where clauses to determine indexable records from the table. This way you +can exclude specific records like ``tt_content`` records with specific ``CType`` values or +something else. E.g. you can add a new field to the table to exclude records from indexing. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') + plugin.tx_searchcore.settings.indexing..additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') - .. attention:: +.. attention:: - Make sure to prefix all fields with the corresponding table name. The selection from - database will contain joins and can lead to SQL errors if a field exists in multiple tables. + Make sure to prefix all fields with the corresponding table name. The selection from + database will contain joins and can lead to SQL errors if a field exists in multiple tables. .. _abstractFields: abstractFields -------------- - Used by: :ref:`PagesIndexer`. +Used by: :ref:`PagesIndexer`. - Define which field should be used to provide the auto generated field "search_abstract". - The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also - possible. +Define which field should be used to provide the auto generated field "search_abstract". +The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also +possible. - Example:: +Example:: - # As last fallback we use the content of the page - plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) + # As last fallback we use the content of the page + plugin.tx_searchcore.settings.indexing..abstractFields := addToList(content) - Default:: +Default:: - abstract, description, bodytext + abstract, description, bodytext .. _mapping: mapping ------- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html - You are able to define the mapping for each property / columns. +Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html +You are able to define the mapping for each property / columns. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.mapping { - CType { - type = keyword - } + plugin.tx_searchcore.settings.indexing.tt_content.mapping { + CType { + type = keyword } + } - The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This - makes building a facet possible. +The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This +makes building a facet possible. .. _index: index ----- - Used by: Elasticsearch connection while indexing. +Used by: Elasticsearch connection while indexing. - Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html +Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.index { - analysis { - analyzer { - ngram4 { - type = custom - tokenizer = ngram4 - char_filter = html_strip - filter = lowercase, asciifolding - } + plugin.tx_searchcore.settings.indexing.tt_content.index { + analysis { + analyzer { + ngram4 { + type = custom + tokenizer = ngram4 + char_filter = html_strip + filter = lowercase, asciifolding } + } - tokenizer { - ngram4 { - type = ngram - min_gram = 4 - max_gram = 4 - } + tokenizer { + ngram4 { + type = ngram + min_gram = 4 + max_gram = 4 } } } + } - ``char_filter`` and ``filter`` are a comma separated list of options. +``char_filter`` and ``filter`` are a comma separated list of options. .. _dataProcessing: dataProcessing -------------- - Used by: All connections while indexing. +Used by: All connections while indexing. - Configure modifications on each document before sending it to the configured connection. Same as - provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. +Configure modifications on each document before sending it to the configured connection. Same as +provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. - All processors are applied in configured order. Allowing to work with already processed data. +All processors are applied in configured order. Allowing to work with already processed data. - Example:: +Example:: - plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { - 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 1 { - to = search_spellcheck - } - - 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor - 2 { - to = search_all - } + plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { + 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 1 { + to = search_spellcheck } - The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards - all fields, including ``search_spellcheck`` will be copied to ``search_all``. - E.g. used to index all information into a field for :ref:`spellchecking` or searching with - different :ref:`mapping`. + 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor + 2 { + to = search_all + } + } - The following Processor are available: +The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards +all fields, including ``search_spellcheck`` will be copied to ``search_all``. +E.g. used to index all information into a field for :ref:`spellchecking` or searching with +different :ref:`mapping`. - .. toctree:: - :maxdepth: 1 - :glob: +The following Processor are available: - dataProcessing/CopyToProcessor - dataProcessing/GeoPointProcessor +.. toctree:: + :maxdepth: 1 + :glob: - The following Processor are planned: + dataProcessing/CopyToProcessor + dataProcessing/GeoPointProcessor - ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` - Will execute a search and replace on configured fields. +The following Processor are planned: - ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` - Will attach the root level to the record. + ``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` + Will execute a search and replace on configured fields. - ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` - Will add a configurable channel to the record, e.g. if you have different areas in your - website like "products" and "infos". + ``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` + Will attach the root level to the record. - ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` - Resolves all relations using the TCA. + ``Codappix\SearchCore\DataProcessing\ChannelProcessor`` + Will add a configurable channel to the record, e.g. if you have different areas in your + website like "products" and "infos". - Of course you are able to provide further processors. Just implement - ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified - class name) as done in the examples above. + ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` + Resolves all relations using the TCA. - By implementing also the same interface as necessary for TYPO3 - :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code - also for Fluid to prepare the same record fetched from DB for your fluid. +Of course you are able to provide further processors. Just implement +``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified +class name) as done in the examples above. + +By implementing also the same interface as necessary for TYPO3 +:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code +also for Fluid to prepare the same record fetched from DB for your fluid. diff --git a/Documentation/source/configuration/searching.rst b/Documentation/source/configuration/searching.rst index 2143364..e953dcb 100644 --- a/Documentation/source/configuration/searching.rst +++ b/Documentation/source/configuration/searching.rst @@ -8,118 +8,200 @@ Searching size ---- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Defined how many search results should be fetched to be available in search result. +Defined how many search results should be fetched to be available in search result. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.size = 50 + plugin.tx_searchcore.settings.searching.size = 50 - Default if not configured is 10. +Default if not configured is 10. .. _facets: facets ------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html - Currently only the term facet is provided. +Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html +Currently only the term facet is provided. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.facets { - contentTypes { - field = CType - } + plugin.tx_searchcore.settings.searching.facets { + contentTypes { + field = CType } + } - The above example will provide a facet with options for all found ``CType`` results together - with a count. +The above example will provide a facet with options for all found ``CType`` results together +with a count. .. _filter: -``filter`` -""""""""""" +filter +------ - Used by: While building search request. +Used by: While building search request. - Define filter that should be set for all requests. +Define filter that should be set for all requests. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.filter { - property = value - } + plugin.tx_searchcore.settings.searching.filter { + property = value + } - For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. +For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. .. _minimumShouldMatch: minimumShouldMatch ------------------ - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html +Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html - Example:: +Example:: - plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% + plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% .. _boost: boost ----- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define fields that should boost the score for results. +Define fields that should boost the score for results. - Example:: +Example:: - plugin.tx_searchcore.settings.searching.boost { - search_title = 3 - search_abstract = 1.5 - } + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } - For further information take a look at - https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html +For further information take a look at +https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html .. _fieldValueFactor: fieldValueFactor ---------------- - Used by: Elasticsearch connection while building search query. +Used by: Elasticsearch connection while building search query. - Define a field to use as a factor for scoring. The configuration is passed through to elastic - search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor +Define a field to use as a factor for scoring. The configuration is passed through to elastic +search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor - Example:: +Example:: - plugin.tx_searchcore.settings.searching.field_value_factor { - field = rootlineLevel - modifier = reciprocal - factor = 2 - missing = 1 + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } + +.. _mapping.filter: + +mapping.filter +-------------- + +Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will +be added. + +E.g. you submit a filter in form of: + +.. code-block:: html + + + + + +This will create a ``distance`` filter with subproperties. To make this filter actually work, you +can add the following TypoScript, which will be added to the filter:: + + mapping { + filter { + distance { + field = geo_distance + fields { + distance = distance + location = location + } + } } + } + +``fields`` has a special meaning here. This will actually map the properties of the filter to fields +in elasticsearch. In above example they do match, but you can also use different names in your form. +On the left hand side is the elasticsearch field name, on the right side the one submitted as a +filter. + +The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for +filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields. + +.. _fields: + +fields +------ + +Defines the fields to fetch from elasticsearch. Two sub entries exist: + +First ``stored_fields`` which is a list of comma separated fields which actually exist and will be +added. Typically you will use ``_source`` to fetch the whole indexed fields. + +Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch. +An example might look like the following:: + + fields { + script_fields { + distance { + condition = {request.filter.distance.location} + script { + params { + lat = {request.filter.distance.location.lat -> f:format.number()} + lon = {request.filter.distance.location.lon -> f:format.number()} + } + lang = painless + inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001 + } + } + } + } + +In above example we add a single ``script_field`` called ``distance``. We add a condition when this +field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP. +If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the +query. The ``condition`` will be removed and everything else is submitted one to one to +elasticsearch, except each property is run through Fluidtemplate, to allow you to use information +from search request, e.g. to insert latitude and longitude from a filter, like in the above example. + +.. _sort: + +sort +---- + +Sort is handled like :ref:`fields`. .. _mode: -``mode`` -"""""""" +mode +---- - Used by: Controller while preparing action. +Used by: Controller while preparing action. - Define to switch from search to filter mode. +Define to switch from search to filter mode. - Example:: +Example:: - plugin.tx_searchcore.settings.searching { - mode = filter - } + plugin.tx_searchcore.settings.searching { + mode = filter + } - Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. +Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.