Work-Related/search_core
Work-Related/search_core
1
0
mirror of https://github.com/Codappix/search_core.git synced 2024-11-25 09:36:11 +01:00
Code Releases Activity

TASK: Allow fields and sorting to contain a condition

Browse source 
This way integrators can configure when the sorting and fields should be
added.
This commit is contained in:
Daniel Siepmann 2017-10-29 12:25:25 +01:00
parent d937266bf5
commit bf91c4a5ba
Signed by: Daniel Siepmann
GPG key ID: 33D6629915560EF4
4 changed files with 266 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
Classes/Domain/Search/QueryFactory.php
View file

@ -165,7 +165,10 @@ class QueryFactory
try { try {
$scriptFields = $this->configuration->get('searching.fields.script_fields'); $scriptFields = $this->configuration->get('searching.fields.script_fields');
$scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields); $scriptFields = $this->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields);
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]); $scriptFields = $this->filterByCondition($scriptFields);
if ($scriptFields !== []) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]);
}
} catch (InvalidArgumentException $e) { } catch (InvalidArgumentException $e) {
// Nothing configured // Nothing configured
} }
@ -175,7 +178,10 @@ class QueryFactory
{ {
$sorting = $this->configuration->getIfExists('searching.sort') ?: []; $sorting = $this->configuration->getIfExists('searching.sort') ?: [];
$sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting); $sorting = $this->replaceArrayValuesWithRequestContent($searchRequest, $sorting);
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]); $sorting = $this->filterByCondition($sorting);
if ($sorting !== []) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]);
}
} }
protected function addFilter(SearchRequestInterface $searchRequest, array &$query) protected function addFilter(SearchRequestInterface $searchRequest, array &$query)
@ -254,4 +260,19 @@ class QueryFactory
return $array; return $array;
} }
protected function filterByCondition(array $entries) : array
{
$entries = array_filter($entries, function ($entry) {
return !array_key_exists('condition', $entry) || (bool) $entry['condition'] === true;
});
foreach ($entries as $key => $entry) {
if (array_key_exists('condition', $entry)) {
unset($entries[$key]['condition']);
}
}
return $entries;
}
} }

18
Documentation/source/configuration/connections.rst
View file

@ -29,27 +29,27 @@ The following settings are available. For each setting its documented which conn
``host`` ``host``
-------- --------
Used by: :ref:`Elasticsearch`. Used by: :ref:`Elasticsearch`.
The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3 The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
installation. installation.
Example:: Example::
plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
.. _port: .. _port:
``port`` ``port``
-------- --------
Used by: :ref:`Elasticsearch`. Used by: :ref:`Elasticsearch`.
The port where search service is reachable. E.g. default ``9200`` for Elasticsearch. The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
Example:: Example::
plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200 plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200

194
Documentation/source/configuration/indexing.rst
View file

@ -29,18 +29,18 @@ The following settings are available. For each setting its documented which inde
rootLineBlacklist rootLineBlacklist
----------------- -----------------
Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Defines a blacklist of page uids. Records below any of these pages, or subpages, are not Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
indexed. This allows you to define areas that should not be indexed. indexed. This allows you to define areas that should not be indexed.
The page attribute *No Search* is also taken into account to prevent indexing records from only one The page attribute *No Search* is also taken into account to prevent indexing records from only one
page without recursion. page without recursion.
Contains a comma separated list of page uids. Spaces are trimmed. Contains a comma separated list of page uids. Spaces are trimmed.
Example:: Example::
plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100 plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100
Also it's possible to define some behaviour for the different document types. In context of TYPO3 Also it's possible to define some behaviour for the different document types. In context of TYPO3
tables are used as document types 1:1. It's possible to configure different tables. The following tables are used as document types 1:1. It's possible to configure different tables. The following
@ -51,156 +51,156 @@ options are available:
additionalWhereClause additionalWhereClause
--------------------- ---------------------
Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`. Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Add additional SQL to where clauses to determine indexable records from the table. This way you Add additional SQL to where clauses to determine indexable records from the table. This way you
can exclude specific records like ``tt_content`` records with specific ``CType`` values or can exclude specific records like ``tt_content`` records with specific ``CType`` values or
something else. E.g. you can add a new field to the table to exclude records from indexing. something else. E.g. you can add a new field to the table to exclude records from indexing.
Example:: Example::
plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu') plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
.. attention:: .. attention::
Make sure to prefix all fields with the corresponding table name. The selection from Make sure to prefix all fields with the corresponding table name. The selection from
database will contain joins and can lead to SQL errors if a field exists in multiple tables. database will contain joins and can lead to SQL errors if a field exists in multiple tables.
.. _abstractFields: .. _abstractFields:
abstractFields abstractFields
-------------- --------------
Used by: :ref:`PagesIndexer`. Used by: :ref:`PagesIndexer`.
Define which field should be used to provide the auto generated field "search_abstract". Define which field should be used to provide the auto generated field "search_abstract".
The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
possible. possible.
Example:: Example::
# As last fallback we use the content of the page # As last fallback we use the content of the page
plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content) plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content)
Default:: Default::
abstract, description, bodytext abstract, description, bodytext
.. _mapping: .. _mapping:
mapping mapping
------- -------
Used by: Elasticsearch connection while indexing. Used by: Elasticsearch connection while indexing.
Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
You are able to define the mapping for each property / columns. You are able to define the mapping for each property / columns.
Example:: Example::
plugin.tx_searchcore.settings.indexing.tt_content.mapping { plugin.tx_searchcore.settings.indexing.tt_content.mapping {
CType { CType {
type = keyword type = keyword
}
} }
}
The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This
makes building a facet possible. makes building a facet possible.
.. _index: .. _index:
index index
----- -----
Used by: Elasticsearch connection while indexing. Used by: Elasticsearch connection while indexing.
Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
Example:: Example::
plugin.tx_searchcore.settings.indexing.tt_content.index { plugin.tx_searchcore.settings.indexing.tt_content.index {
analysis { analysis {
analyzer { analyzer {
ngram4 { ngram4 {
type = custom type = custom
tokenizer = ngram4 tokenizer = ngram4
char_filter = html_strip char_filter = html_strip
filter = lowercase, asciifolding filter = lowercase, asciifolding
}
} }
}
tokenizer { tokenizer {
ngram4 { ngram4 {
type = ngram type = ngram
min_gram = 4 min_gram = 4
max_gram = 4 max_gram = 4
}
} }
} }
} }
}
``char_filter`` and ``filter`` are a comma separated list of options. ``char_filter`` and ``filter`` are a comma separated list of options.
.. _dataProcessing: .. _dataProcessing:
dataProcessing dataProcessing
-------------- --------------
Used by: All connections while indexing. Used by: All connections while indexing.
Configure modifications on each document before sending it to the configured connection. Same as Configure modifications on each document before sending it to the configured connection. Same as
provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`. :ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
All processors are applied in configured order. Allowing to work with already processed data. All processors are applied in configured order. Allowing to work with already processed data.
Example:: Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing { plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor 1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 { 1 {
to = search_spellcheck to = search_spellcheck
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = search_all
}
} }
The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards 2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
all fields, including ``search_spellcheck`` will be copied to ``search_all``. 2 {
E.g. used to index all information into a field for :ref:`spellchecking` or searching with to = search_all
different :ref:`mapping`. }
}
The following Processor are available: The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
all fields, including ``search_spellcheck`` will be copied to ``search_all``.
E.g. used to index all information into a field for :ref:`spellchecking` or searching with
different :ref:`mapping`.
.. toctree:: The following Processor are available:
:maxdepth: 1
:glob:
dataProcessing/CopyToProcessor .. toctree::
dataProcessing/GeoPointProcessor :maxdepth: 1
:glob:
The following Processor are planned: dataProcessing/CopyToProcessor
dataProcessing/GeoPointProcessor
``Codappix\SearchCore\DataProcessing\ReplaceProcessor`` The following Processor are planned:
Will execute a search and replace on configured fields.
``Codappix\SearchCore\DataProcessing\RootLevelProcessor`` ``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
Will attach the root level to the record. Will execute a search and replace on configured fields.
``Codappix\SearchCore\DataProcessing\ChannelProcessor`` ``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
Will add a configurable channel to the record, e.g. if you have different areas in your Will attach the root level to the record.
website like "products" and "infos".
``Codappix\SearchCore\DataProcessing\RelationResolverProcessor`` ``Codappix\SearchCore\DataProcessing\ChannelProcessor``
Resolves all relations using the TCA. Will add a configurable channel to the record, e.g. if you have different areas in your
website like "products" and "infos".
Of course you are able to provide further processors. Just implement ``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified Resolves all relations using the TCA.
class name) as done in the examples above.
By implementing also the same interface as necessary for TYPO3 Of course you are able to provide further processors. Just implement
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code ``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
also for Fluid to prepare the same record fetched from DB for your fluid. class name) as done in the examples above.
By implementing also the same interface as necessary for TYPO3
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
also for Fluid to prepare the same record fetched from DB for your fluid.

192
Documentation/source/configuration/searching.rst
View file

@ -8,118 +8,200 @@ Searching
size size
---- ----
Used by: Elasticsearch connection while building search query. Used by: Elasticsearch connection while building search query.
Defined how many search results should be fetched to be available in search result. Defined how many search results should be fetched to be available in search result.
Example:: Example::
plugin.tx_searchcore.settings.searching.size = 50 plugin.tx_searchcore.settings.searching.size = 50
Default if not configured is 10. Default if not configured is 10.
.. _facets: .. _facets:
facets facets
------ ------
Used by: Elasticsearch connection while building search query. Used by: Elasticsearch connection while building search query.
Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
Currently only the term facet is provided. Currently only the term facet is provided.
Example:: Example::
plugin.tx_searchcore.settings.searching.facets { plugin.tx_searchcore.settings.searching.facets {
contentTypes { contentTypes {
field = CType field = CType
}
} }
}
The above example will provide a facet with options for all found ``CType`` results together The above example will provide a facet with options for all found ``CType`` results together
with a count. with a count.
.. _filter: .. _filter:
``filter`` filter
""""""""""" ------
Used by: While building search request. Used by: While building search request.
Define filter that should be set for all requests. Define filter that should be set for all requests.
Example:: Example::
plugin.tx_searchcore.settings.searching.filter { plugin.tx_searchcore.settings.searching.filter {
property = value property = value
} }
For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``. For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
.. _minimumShouldMatch: .. _minimumShouldMatch:
minimumShouldMatch minimumShouldMatch
------------------ ------------------
Used by: Elasticsearch connection while building search query. Used by: Elasticsearch connection while building search query.
Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
Example:: Example::
plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50% plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50%
.. _boost: .. _boost:
boost boost
----- -----
Used by: Elasticsearch connection while building search query. Used by: Elasticsearch connection while building search query.
Define fields that should boost the score for results. Define fields that should boost the score for results.
Example:: Example::
plugin.tx_searchcore.settings.searching.boost { plugin.tx_searchcore.settings.searching.boost {
search_title = 3 search_title = 3
search_abstract = 1.5 search_abstract = 1.5
} }
For further information take a look at For further information take a look at
https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html
.. _fieldValueFactor: .. _fieldValueFactor:
fieldValueFactor fieldValueFactor
---------------- ----------------
Used by: Elasticsearch connection while building search query. Used by: Elasticsearch connection while building search query.
Define a field to use as a factor for scoring. The configuration is passed through to elastic Define a field to use as a factor for scoring. The configuration is passed through to elastic
search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
Example:: Example::
plugin.tx_searchcore.settings.searching.field_value_factor { plugin.tx_searchcore.settings.searching.field_value_factor {
field = rootlineLevel field = rootlineLevel
modifier = reciprocal modifier = reciprocal
factor = 2 factor = 2
missing = 1 missing = 1
}
.. _mapping.filter:
mapping.filter
--------------
Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will
be added.
E.g. you submit a filter in form of:
.. code-block:: html
<f:form.textfield property="filter.distance.location.lat" value="51.168098" />
<f:form.textfield property="filter.distance.location.lon" value="6.381384" />
<f:form.textfield property="filter.distance.distance" value="100km" />
This will create a ``distance`` filter with subproperties. To make this filter actually work, you
can add the following TypoScript, which will be added to the filter::
mapping {
filter {
distance {
field = geo_distance
fields {
distance = distance
location = location
}
}
} }
}
``fields`` has a special meaning here. This will actually map the properties of the filter to fields
in elasticsearch. In above example they do match, but you can also use different names in your form.
On the left hand side is the elasticsearch field name, on the right side the one submitted as a
filter.
The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for
filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields.
.. _fields:
fields
------
Defines the fields to fetch from elasticsearch. Two sub entries exist:
First ``stored_fields`` which is a list of comma separated fields which actually exist and will be
added. Typically you will use ``_source`` to fetch the whole indexed fields.
Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch.
An example might look like the following::
fields {
script_fields {
distance {
condition = {request.filter.distance.location}
script {
params {
lat = {request.filter.distance.location.lat -> f:format.number()}
lon = {request.filter.distance.location.lon -> f:format.number()}
}
lang = painless
inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001
}
}
}
}
In above example we add a single ``script_field`` called ``distance``. We add a condition when this
field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP.
If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the
query. The ``condition`` will be removed and everything else is submitted one to one to
elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
from search request, e.g. to insert latitude and longitude from a filter, like in the above example.
.. _sort:
sort
----
Sort is handled like :ref:`fields`.
.. _mode: .. _mode:
``mode`` mode
"""""""" ----
Used by: Controller while preparing action. Used by: Controller while preparing action.
Define to switch from search to filter mode. Define to switch from search to filter mode.
Example:: Example::
plugin.tx_searchcore.settings.searching { plugin.tx_searchcore.settings.searching {
mode = filter mode = filter
} }
Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode. Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.