Work-Related/search_core
Work-Related/search_core
1
0
mirror of https://github.com/Codappix/search_core.git synced 2024-11-25 05:16:12 +01:00
Code Releases Activity

Merge pull request #92 from Codappix/feature/geo-search

Browse source 
Feature: Support Geo search
This commit is contained in:
Daniel Siepmann 2017-11-02 22:41:00 +01:00 committed by GitHub
commit 87298d8e58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 1087 additions and 226 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
Classes/Configuration/ConfigurationUtility.php Normal file
View file

@ -0,0 +1,69 @@
<?php
namespace Codappix\SearchCore\Configuration;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Connection\SearchRequestInterface;
use TYPO3\CMS\Fluid\View\StandaloneView;
class ConfigurationUtility
{
/**
* Will parse all entries, recursive as fluid template, with request variable set to $searchRequest.
*/
public function replaceArrayValuesWithRequestContent(SearchRequestInterface $searchRequest, array $array) : array
{
array_walk_recursive($array, function (&$value, $key, SearchRequestInterface $searchRequest) {
$template = new StandaloneView();
$template->assign('request', $searchRequest);
$template->setTemplateSource($value);
$value = $template->render();
// As elasticsearch does need some doubles to be send as doubles.
if (is_numeric($value)) {
$value = (float) $value;
}
}, $searchRequest);
return $array;
}
/**
* Will check all entries, whether they have a condition and filter entries out, where condition is false.
* Also will remove condition in the end.
*/
public function filterByCondition(array $entries) : array
{
$entries = array_filter($entries, function ($entry) {
return !is_array($entry)
|| !array_key_exists('condition', $entry)
|| (bool) $entry['condition'] === true
;
});
foreach ($entries as $key => $entry) {
if (is_array($entry) && array_key_exists('condition', $entry)) {
unset($entries[$key]['condition']);
}
}
return $entries;
}
}

6
Classes/DataProcessing/CopyToProcessor.php
View file

@ -25,7 +25,7 @@ namespace Codappix\SearchCore\DataProcessing;
*/
class CopyToProcessor implements ProcessorInterface
{
public function processRecord(array $record, array $configuration)
public function processRecord(array $record, array $configuration) : array
{
$all = [];
@ -36,10 +36,6 @@ class CopyToProcessor implements ProcessorInterface
return $record;
}
/**
* @param array &$target
* @param array $from
*/
protected function addArray(array &$target, array $from)
{
foreach ($from as $value) {

59
Classes/DataProcessing/GeoPointProcessor.php Normal file
View file

@ -0,0 +1,59 @@
<?php
namespace Codappix\SearchCore\DataProcessing;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* Adds a new fields, ready to use as GeoPoint field for Elasticsearch.
*/
class GeoPointProcessor implements ProcessorInterface
{
public function processRecord(array $record, array $configuration) : array
{
if (! $this->canApply($record, $configuration)) {
return $record;
}
$record[$configuration['to']] = [
'lat' => (float) $record[$configuration['lat']],
'lon' => (float) $record[$configuration['lon']],
];
return $record;
}
protected function canApply(array $record, array $configuration) : bool
{
if (!isset($record[$configuration['lat']])
|| !is_numeric($record[$configuration['lat']])
|| trim($record[$configuration['lat']]) === ''
) {
return false;
}
if (!isset($record[$configuration['lon']])
|| !is_numeric($record[$configuration['lon']])
|| trim($record[$configuration['lon']]) === ''
) {
return false;
}
return true;
}
}

7
Classes/DataProcessing/ProcessorInterface.php
View file

@ -29,11 +29,6 @@ interface ProcessorInterface
/**
* Processes the given record.
* Also retrieves the configuration for this processor instance.
*
* @param array $record
* @param array $configuration
*
* @return array
*/
public function processRecord(array $record, array $configuration);
public function processRecord(array $record, array $configuration) : array;
}

3
Classes/Domain/Model/SearchRequest.php
View file

@ -92,7 +92,8 @@ class SearchRequest implements SearchRequestInterface
*/
public function setFilter(array $filter)
{
$this->filter = array_filter(array_map('strval', $filter));
$filter = \TYPO3\CMS\Core\Utility\ArrayUtility::removeArrayEntryByValue($filter, '');
$this->filter = \TYPO3\CMS\Extbase\Utility\ArrayUtility::removeEmptyElementsRecursively($filter);
}
/**

120
Classes/Domain/Search/QueryFactory.php
View file

@ -21,10 +21,12 @@ namespace Codappix\SearchCore\Domain\Search;
*/
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\ConfigurationUtility;
use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Connection\ConnectionInterface;
use Codappix\SearchCore\Connection\Elasticsearch\Query;
use Codappix\SearchCore\Connection\SearchRequestInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Utility\ArrayUtility;
class QueryFactory
@ -40,37 +42,31 @@ class QueryFactory
protected $configuration;
/**
* @param \TYPO3\CMS\Core\Log\LogManager $logManager
* @param ConfigurationContainerInterface $configuration
* @var ConfigurationUtility
*/
protected $configurationUtility;
public function __construct(
\TYPO3\CMS\Core\Log\LogManager $logManager,
ConfigurationContainerInterface $configuration
ConfigurationContainerInterface $configuration,
ConfigurationUtility $configurationUtility
) {
$this->logger = $logManager->getLogger(__CLASS__);
$this->configuration = $configuration;
$this->configurationUtility = $configurationUtility;
}
/**
* TODO: This is not in scope Elasticsearch, therefore it should not return
* \Elastica\Query, but decide to use a more specific QueryFactory like
* ElasticaQueryFactory, once the second query is added?
*
* @param SearchRequestInterface $searchRequest
*
* @return \Elastica\Query
*/
public function create(SearchRequestInterface $searchRequest)
public function create(SearchRequestInterface $searchRequest) : \Elastica\Query
{
return $this->createElasticaQuery($searchRequest);
}
/**
* @param SearchRequestInterface $searchRequest
*
* @return \Elastica\Query
*/
protected function createElasticaQuery(SearchRequestInterface $searchRequest)
protected function createElasticaQuery(SearchRequestInterface $searchRequest) : \Elastica\Query
{
$query = [];
$this->addSize($searchRequest, $query);
@ -78,6 +74,8 @@ class QueryFactory
$this->addBoosts($searchRequest, $query);
$this->addFilter($searchRequest, $query);
$this->addFacets($searchRequest, $query);
$this->addFields($searchRequest, $query);
$this->addSort($searchRequest, $query);
// Use last, as it might change structure of query.
// Better approach would be something like DQL to generate query and build result in the end.
@ -87,10 +85,6 @@ class QueryFactory
return new \Elastica\Query($query);
}
/**
* @param SearchRequestInterface $searchRequest
* @param array $query
*/
protected function addSize(SearchRequestInterface $searchRequest, array &$query)
{
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
@ -99,10 +93,6 @@ class QueryFactory
]);
}
/**
* @param SearchRequestInterface $searchRequest
* @param array $query
*/
protected function addSearch(SearchRequestInterface $searchRequest, array &$query)
{
if (trim($searchRequest->getSearchTerm()) === '') {
@ -125,10 +115,6 @@ class QueryFactory
}
}
/**
* @param SearchRequestInterface $searchRequest
* @param array $query
*/
protected function addBoosts(SearchRequestInterface $searchRequest, array &$query)
{
try {
@ -159,9 +145,6 @@ class QueryFactory
]);
}
/**
* @param array $query
*/
protected function addFactorBoost(array &$query)
{
try {
@ -176,38 +159,83 @@ class QueryFactory
}
}
/**
* @param SearchRequestInterface $searchRequest
* @param array $query
*/
protected function addFields(SearchRequestInterface $searchRequest, array &$query)
{
try {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'stored_fields' => GeneralUtility::trimExplode(',', $this->configuration->get('searching.fields.stored_fields'), true),
]);
} catch (InvalidArgumentException $e) {
// Nothing configured
}
try {
$scriptFields = $this->configuration->get('searching.fields.script_fields');
$scriptFields = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $scriptFields);
$scriptFields = $this->configurationUtility->filterByCondition($scriptFields);
if ($scriptFields !== []) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['script_fields' => $scriptFields]);
}
} catch (InvalidArgumentException $e) {
// Nothing configured
}
}
protected function addSort(SearchRequestInterface $searchRequest, array &$query)
{
$sorting = $this->configuration->getIfExists('searching.sort') ?: [];
$sorting = $this->configurationUtility->replaceArrayValuesWithRequestContent($searchRequest, $sorting);
$sorting = $this->configurationUtility->filterByCondition($sorting);
if ($sorting !== []) {
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, ['sort' => $sorting]);
}
}
protected function addFilter(SearchRequestInterface $searchRequest, array &$query)
{
if (! $searchRequest->hasFilter()) {
return;
}
$terms = [];
$filter = [];
foreach ($searchRequest->getFilter() as $name => $value) {
$terms[] = [
$filter[] = $this->buildFilter(
$name,
$value,
$this->configuration->getIfExists('searching.mapping.filter.' . $name) ?: []
);
}
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'query' => [
'bool' => [
'filter' => $filter,
],
],
]);
}
protected function buildFilter(string $name, $value, array $config) : array
{
if ($config === []) {
return [
'term' => [
$name => $value,
],
];
}
$query = ArrayUtility::arrayMergeRecursiveOverrule($query, [
'query' => [
'bool' => [
'filter' => $terms,
],
],
]);
$filter = [];
if (isset($config['fields'])) {
foreach ($config['fields'] as $elasticField => $inputField) {
$filter[$elasticField] = $value[$inputField];
}
}
return [$config['field'] => $filter];
}
/**
* @param SearchRequestInterface $searchRequest
* @param array $query
*/
protected function addFacets(SearchRequestInterface $searchRequest, array &$query)
{
foreach ($searchRequest->getFacets() as $facet) {

18
Documentation/source/configuration/connections.rst
View file

@ -29,27 +29,27 @@ The following settings are available. For each setting its documented which conn
``host``
--------
Used by: :ref:`Elasticsearch`.
Used by: :ref:`Elasticsearch`.
The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
installation.
The host, e.g. ``localhost`` or an IP where the search service is reachable from TYPO3
installation.
Example::
Example::
plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
plugin.tx_searchcore.settings.connections.elasticsearch.host = localhost
.. _port:
``port``
--------
Used by: :ref:`Elasticsearch`.
Used by: :ref:`Elasticsearch`.
The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
The port where search service is reachable. E.g. default ``9200`` for Elasticsearch.
Example::
Example::
plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200
plugin.tx_searchcore.settings.connections.elasticsearch.port = 9200

23
Documentation/source/configuration/dataProcessing/CopyToProcessor.rst Normal file
View file

@ -0,0 +1,23 @@
``Codappix\SearchCore\DataProcessing\CopyToProcessor``
======================================================
Will copy contents of fields to other fields.
Possible Options:
``to``
Defines the field to copy the values into. All values not false will be copied at the moment.
Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 {
to = all
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = spellcheck
}
}

27
Documentation/source/configuration/dataProcessing/GeoPointProcessor.rst Normal file
View file

@ -0,0 +1,27 @@
``Codappix\SearchCore\DataProcessing\GeoPointProcessor``
========================================================
Will create a new field, ready to use as GeoPoint field for Elasticsearch.
Possible Options:
``to``
Defines the field to create as GeoPoint.
``lat``
Defines the field containing the latitude.
``lon``
Defines the field containing the longitude.
Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\GeoPointProcessor
1 {
to = location
lat = lat
lon = lng
}
}
The above example will create a new field ``location`` as GeoPoint with latitude fetched from field
``lat`` and longitude fetched from field ``lng``.

192
Documentation/source/configuration/indexing.rst
View file

@ -29,18 +29,18 @@ The following settings are available. For each setting its documented which inde
rootLineBlacklist
-----------------
Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
indexed. This allows you to define areas that should not be indexed.
The page attribute *No Search* is also taken into account to prevent indexing records from only one
page without recursion.
Defines a blacklist of page uids. Records below any of these pages, or subpages, are not
indexed. This allows you to define areas that should not be indexed.
The page attribute *No Search* is also taken into account to prevent indexing records from only one
page without recursion.
Contains a comma separated list of page uids. Spaces are trimmed.
Contains a comma separated list of page uids. Spaces are trimmed.
Example::
Example::
plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100
plugin.tx_searchcore.settings.indexing.<identifier>.rootLineBlacklist = 3, 10, 100
Also it's possible to define some behaviour for the different document types. In context of TYPO3
tables are used as document types 1:1. It's possible to configure different tables. The following
@ -51,152 +51,156 @@ options are available:
additionalWhereClause
---------------------
Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Used by: :ref:`TcaIndexer`, :ref:`PagesIndexer`.
Add additional SQL to where clauses to determine indexable records from the table. This way you
can exclude specific records like ``tt_content`` records with specific ``CType`` values or
something else. E.g. you can add a new field to the table to exclude records from indexing.
Add additional SQL to where clauses to determine indexable records from the table. This way you
can exclude specific records like ``tt_content`` records with specific ``CType`` values or
something else. E.g. you can add a new field to the table to exclude records from indexing.
Example::
Example::
plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
plugin.tx_searchcore.settings.indexing.<identifier>.additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu')
.. attention::
.. attention::
Make sure to prefix all fields with the corresponding table name. The selection from
database will contain joins and can lead to SQL errors if a field exists in multiple tables.
Make sure to prefix all fields with the corresponding table name. The selection from
database will contain joins and can lead to SQL errors if a field exists in multiple tables.
.. _abstractFields:
abstractFields
--------------
Used by: :ref:`PagesIndexer`.
Used by: :ref:`PagesIndexer`.
Define which field should be used to provide the auto generated field "search_abstract".
The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
possible.
Define which field should be used to provide the auto generated field "search_abstract".
The fields have to exist in the record to be indexed. Therefore fields like ``content`` are also
possible.
Example::
Example::
# As last fallback we use the content of the page
plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content)
# As last fallback we use the content of the page
plugin.tx_searchcore.settings.indexing.<identifier>.abstractFields := addToList(content)
Default::
Default::
abstract, description, bodytext
abstract, description, bodytext
.. _mapping:
mapping
-------
Used by: Elasticsearch connection while indexing.
Used by: Elasticsearch connection while indexing.
Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
You are able to define the mapping for each property / columns.
Define mapping for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/mapping.html
You are able to define the mapping for each property / columns.
Example::
Example::
plugin.tx_searchcore.settings.indexing.tt_content.mapping {
CType {
type = keyword
}
plugin.tx_searchcore.settings.indexing.tt_content.mapping {
CType {
type = keyword
}
}
The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This
makes building a facet possible.
The above example will define the ``CType`` field of ``tt_content`` as ``type: keyword``. This
makes building a facet possible.
.. _index:
index
-----
Used by: Elasticsearch connection while indexing.
Used by: Elasticsearch connection while indexing.
Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
Define index for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/indices-create-index.html
Example::
Example::
plugin.tx_searchcore.settings.indexing.tt_content.index {
analysis {
analyzer {
ngram4 {
type = custom
tokenizer = ngram4
char_filter = html_strip
filter = lowercase, asciifolding
}
plugin.tx_searchcore.settings.indexing.tt_content.index {
analysis {
analyzer {
ngram4 {
type = custom
tokenizer = ngram4
char_filter = html_strip
filter = lowercase, asciifolding
}
}
tokenizer {
ngram4 {
type = ngram
min_gram = 4
max_gram = 4
}
tokenizer {
ngram4 {
type = ngram
min_gram = 4
max_gram = 4
}
}
}
}
``char_filter`` and ``filter`` are a comma separated list of options.
``char_filter`` and ``filter`` are a comma separated list of options.
.. _dataProcessing:
dataProcessing
--------------
Used by: All connections while indexing.
Used by: All connections while indexing.
Configure modifications on each document before sending it to the configured connection. Same as
provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
Configure modifications on each document before sending it to the configured connection. Same as
provided by TYPO3 for :ref:`t3tsref:cobj-fluidtemplate` through
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`.
All processors are applied in configured order. Allowing to work with already processed data.
All processors are applied in configured order. Allowing to work with already processed data.
Example::
Example::
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 {
to = search_spellcheck
}
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = search_all
}
plugin.tx_searchcore.settings.indexing.tt_content.dataProcessing {
1 = Codappix\SearchCore\DataProcessing\CopyToProcessor
1 {
to = search_spellcheck
}
The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
all fields, including ``search_spellcheck`` will be copied to ``search_all``.
E.g. used to index all information into a field for :ref:`spellchecking` or searching with
different :ref:`mapping`.
2 = Codappix\SearchCore\DataProcessing\CopyToProcessor
2 {
to = search_all
}
}
The following Processor are available:
The above example will copy all existing fields to the field ``search_spellcheck``. Afterwards
all fields, including ``search_spellcheck`` will be copied to ``search_all``.
E.g. used to index all information into a field for :ref:`spellchecking` or searching with
different :ref:`mapping`.
``Codappix\SearchCore\DataProcessing\CopyToProcessor``
Will copy contents of fields to other fields
The following Processor are available:
The following Processor are planned:
.. toctree::
:maxdepth: 1
:glob:
``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
Will execute a search and replace on configured fields.
dataProcessing/CopyToProcessor
dataProcessing/GeoPointProcessor
``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
Will attach the root level to the record.
The following Processor are planned:
``Codappix\SearchCore\DataProcessing\ChannelProcessor``
Will add a configurable channel to the record, e.g. if you have different areas in your
website like "products" and "infos".
``Codappix\SearchCore\DataProcessing\ReplaceProcessor``
Will execute a search and replace on configured fields.
``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
Resolves all relations using the TCA.
``Codappix\SearchCore\DataProcessing\RootLevelProcessor``
Will attach the root level to the record.
Of course you are able to provide further processors. Just implement
``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
class name) as done in the examples above.
``Codappix\SearchCore\DataProcessing\ChannelProcessor``
Will add a configurable channel to the record, e.g. if you have different areas in your
website like "products" and "infos".
By implementing also the same interface as necessary for TYPO3
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
also for Fluid to prepare the same record fetched from DB for your fluid.
``Codappix\SearchCore\DataProcessing\RelationResolverProcessor``
Resolves all relations using the TCA.
Of course you are able to provide further processors. Just implement
``Codappix\SearchCore\DataProcessing\ProcessorInterface`` and use the FQCN (=Fully qualified
class name) as done in the examples above.
By implementing also the same interface as necessary for TYPO3
:ref:`t3tsref:cobj-fluidtemplate-properties-dataprocessing`, you are able to reuse the same code
also for Fluid to prepare the same record fetched from DB for your fluid.

192
Documentation/source/configuration/searching.rst
View file

@ -8,118 +8,200 @@ Searching
size
----
Used by: Elasticsearch connection while building search query.
Used by: Elasticsearch connection while building search query.
Defined how many search results should be fetched to be available in search result.
Defined how many search results should be fetched to be available in search result.
Example::
Example::
plugin.tx_searchcore.settings.searching.size = 50
plugin.tx_searchcore.settings.searching.size = 50
Default if not configured is 10.
Default if not configured is 10.
.. _facets:
facets
------
Used by: Elasticsearch connection while building search query.
Used by: Elasticsearch connection while building search query.
Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
Currently only the term facet is provided.
Define aggregations for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/search-aggregations-bucket-terms-aggregation.html
Currently only the term facet is provided.
Example::
Example::
plugin.tx_searchcore.settings.searching.facets {
contentTypes {
field = CType
}
plugin.tx_searchcore.settings.searching.facets {
contentTypes {
field = CType
}
}
The above example will provide a facet with options for all found ``CType`` results together
with a count.
The above example will provide a facet with options for all found ``CType`` results together
with a count.
.. _filter:
``filter``
"""""""""""
filter
------
Used by: While building search request.
Used by: While building search request.
Define filter that should be set for all requests.
Define filter that should be set for all requests.
Example::
Example::
plugin.tx_searchcore.settings.searching.filter {
property = value
}
plugin.tx_searchcore.settings.searching.filter {
property = value
}
For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
For Elasticsearch the fields have to be filterable, e.g. need a mapping as ``keyword``.
.. _minimumShouldMatch:
minimumShouldMatch
------------------
Used by: Elasticsearch connection while building search query.
Used by: Elasticsearch connection while building search query.
Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
Define the minimum match for Elasticsearch, have a look at the official docs: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-minimum-should-match.html
Example::
Example::
plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50%
plugin.tx_searchcore.settings.searching.minimumShouldMatch = 50%
.. _boost:
boost
-----
Used by: Elasticsearch connection while building search query.
Used by: Elasticsearch connection while building search query.
Define fields that should boost the score for results.
Define fields that should boost the score for results.
Example::
Example::
plugin.tx_searchcore.settings.searching.boost {
search_title = 3
search_abstract = 1.5
}
plugin.tx_searchcore.settings.searching.boost {
search_title = 3
search_abstract = 1.5
}
For further information take a look at
https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html
For further information take a look at
https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html
.. _fieldValueFactor:
fieldValueFactor
----------------
Used by: Elasticsearch connection while building search query.
Used by: Elasticsearch connection while building search query.
Define a field to use as a factor for scoring. The configuration is passed through to elastic
search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
Define a field to use as a factor for scoring. The configuration is passed through to elastic
search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor
Example::
Example::
plugin.tx_searchcore.settings.searching.field_value_factor {
field = rootlineLevel
modifier = reciprocal
factor = 2
missing = 1
plugin.tx_searchcore.settings.searching.field_value_factor {
field = rootlineLevel
modifier = reciprocal
factor = 2
missing = 1
}
.. _mapping.filter:
mapping.filter
--------------
Allows to configure filter more in depth. If a filter with the given key exists, the TypoScript will
be added.
E.g. you submit a filter in form of:
.. code-block:: html
<f:form.textfield property="filter.distance.location.lat" value="51.168098" />
<f:form.textfield property="filter.distance.location.lon" value="6.381384" />
<f:form.textfield property="filter.distance.distance" value="100km" />
This will create a ``distance`` filter with subproperties. To make this filter actually work, you
can add the following TypoScript, which will be added to the filter::
mapping {
filter {
distance {
field = geo_distance
fields {
distance = distance
location = location
}
}
}
}
``fields`` has a special meaning here. This will actually map the properties of the filter to fields
in elasticsearch. In above example they do match, but you can also use different names in your form.
On the left hand side is the elasticsearch field name, on the right side the one submitted as a
filter.
The ``field``, in above example ``geo_distance``, will be used as the elasticsearch field for
filtering. This way you can use arbitrary filter names and map them to existing elasticsearch fields.
.. _fields:
fields
------
Defines the fields to fetch from elasticsearch. Two sub entries exist:
First ``stored_fields`` which is a list of comma separated fields which actually exist and will be
added. Typically you will use ``_source`` to fetch the whole indexed fields.
Second is ``script_fields``, which allow you to configure scripted fields for elasticsearch.
An example might look like the following::
fields {
script_fields {
distance {
condition = {request.filter.distance.location}
script {
params {
lat = {request.filter.distance.location.lat -> f:format.number()}
lon = {request.filter.distance.location.lon -> f:format.number()}
}
lang = painless
inline = doc["location"].arcDistance(params.lat,params.lon) * 0.001
}
}
}
}
In above example we add a single ``script_field`` called ``distance``. We add a condition when this
field should be added. The condition will be parsed as Fluidtemplate and is casted to bool via PHP.
If the condition is true, or no ``condition`` exists, the ``script_field`` will be added to the
query. The ``condition`` will be removed and everything else is submitted one to one to
elasticsearch, except each property is run through Fluidtemplate, to allow you to use information
from search request, e.g. to insert latitude and longitude from a filter, like in the above example.
.. _sort:
sort
----
Sort is handled like :ref:`fields`.
.. _mode:
``mode``
""""""""
mode
----
Used by: Controller while preparing action.
Used by: Controller while preparing action.
Define to switch from search to filter mode.
Define to switch from search to filter mode.
Example::
Example::
plugin.tx_searchcore.settings.searching {
mode = filter
}
plugin.tx_searchcore.settings.searching {
mode = filter
}
Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.
Only ``filter`` is allowed as value. Will submit an empty query to switch to filter mode.

15
Tests/Unit/AbstractUnitTestCase.php
View file

@ -24,6 +24,21 @@ use TYPO3\CMS\Core\Tests\UnitTestCase as CoreTestCase;
abstract class AbstractUnitTestCase extends CoreTestCase
{
public function setUp()
{
parent::setUp();
// Disable caching backends to make TYPO3 parts work in unit test mode.
\TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(
\TYPO3\CMS\Core\Cache\CacheManager::class
)->setCacheConfigurations([
'extbase_object' => [
'backend' => \TYPO3\CMS\Core\Cache\Backend\NullBackend::class,
],
]);
}
/**
* @return \TYPO3\CMS\Core\Log\LogManager
*/

143
Tests/Unit/Configuration/ConfigurationUtilityTest.php Normal file
View file

@ -0,0 +1,143 @@
<?php
namespace Codappix\SearchCore\Tests\Unit\Configuration;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Configuration\ConfigurationUtility;
use Codappix\SearchCore\Connection\SearchRequestInterface;
use Codappix\SearchCore\Domain\Model\SearchRequest;
use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
class ConfigurationUtilityTest extends AbstractUnitTestCase
{
/**
* @test
* @dataProvider possibleRequestAndConfigurationForFluidtemplate
*/
public function recursiveEntriesAreProcessedAsFluidtemplate(SearchRequestInterface $searchRequest, array $array, array $expected)
{
$subject = new ConfigurationUtility();
$this->assertSame(
$expected,
$subject->replaceArrayValuesWithRequestContent($searchRequest, $array),
'Entries in array were not parsed as fluid template with search request.'
);
}
public function possibleRequestAndConfigurationForFluidtemplate() : array
{
return [
'Nothing in array' => [
'searchRequest' => new SearchRequest(),
'array' => [],
'expected' => [],
],
'Small array with nothing to replace' => [
'searchRequest' => new SearchRequest(),
'array' => [
'key1' => 'value1',
],
'expected' => [
'key1' => 'value1',
],
],
'Rescursive array with replacements' => [
'searchRequest' => call_user_func(function () {
$request = new SearchRequest();
$request->setFilter([
'distance' => [
'location' => '10',
],
]);
return $request;
}),
'array' => [
'sub1' => [
'sub1.1' => '{request.filter.distance.location}',
'sub1.2' => '{request.nonExisting}',
],
],
'expected' => [
'sub1' => [
// Numberics are casted to double
'sub1.1' => 10.0,
'sub1.2' => null,
],
],
],
];
}
/**
* @test
* @dataProvider possibleConditionEntries
*/
public function conditionsAreHandledAsExpected(array $entries, array $expected)
{
$subject = new ConfigurationUtility();
$this->assertSame(
$expected,
$subject->filterByCondition($entries),
'Conditions were not processed as expected.'
);
}
public function possibleConditionEntries() : array
{
return [
'Nothing in array' => [
'entries' => [],
'expected' => [],
],
'Entries without condition' => [
'entries' => [
'key1' => 'value1',
],
'expected' => [
'key1' => 'value1',
],
],
'Entry with matching condition' => [
'entries' => [
'sub1' => [
'condition' => true,
'sub1.2' => 'something',
],
],
'expected' => [
'sub1' => [
'sub1.2' => 'something',
],
],
],
'Entry with non matching condition' => [
'entries' => [
'sub1' => [
'condition' => false,
'sub1.2' => 'something',
],
],
'expected' => [],
],
];
}
}

113
Tests/Unit/DataProcessing/GeoPointProcessorTest.php Normal file
View file

@ -0,0 +1,113 @@
<?php
namespace Codappix\SearchCore\Tests\Unit\DataProcessing;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\DataProcessing\GeoPointProcessor;
use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
class GeoPointProcessorTest extends AbstractUnitTestCase
{
/**
* @test
* @dataProvider getPossibleRecordConfigurationCombinations
*/
public function geoPointsAreAddedAsConfigured(array $record, array $configuration, array $expectedRecord)
{
$subject = new GeoPointProcessor();
$processedRecord = $subject->processRecord($record, $configuration);
$this->assertSame(
$expectedRecord,
$processedRecord,
'The processor did not return the expected processed record.'
);
}
/**
* @return array
*/
public function getPossibleRecordConfigurationCombinations()
{
return [
'Create new field with existing lat and lng' => [
'record' => [
'lat' => 23.232,
'lng' => 45.43,
],
'configuration' => [
'to' => 'location',
'lat' => 'lat',
'lon' => 'lng',
],
'expectedRecord' => [
'lat' => 23.232,
'lng' => 45.43,
'location' => [
'lat' => 23.232,
'lon' => 45.43,
],
],
],
'Do not create new field due to missing configuration' => [
'record' => [
'lat' => 23.232,
'lng' => 45.43,
],
'configuration' => [
'to' => 'location',
],
'expectedRecord' => [
'lat' => 23.232,
'lng' => 45.43,
],
],
'Do not create new field due to missing lat and lon' => [
'record' => [
'lat' => '',
'lng' => '',
],
'configuration' => [
'to' => 'location',
'lat' => 'lat',
'lon' => 'lng',
],
'expectedRecord' => [
'lat' => '',
'lng' => '',
],
],
'Do not create new field due to invalid lat and lon' => [
'record' => [
'lat' => 'av',
'lng' => 'dsf',
],
'configuration' => [
'to' => 'location',
'lat' => 'lat',
'lon' => 'lng',
],
'expectedRecord' => [
'lat' => 'av',
'lng' => 'dsf',
],
],
];
}
}

80
Tests/Unit/Domain/Model/SearchRequestTest.php Normal file
View file

@ -0,0 +1,80 @@
<?php
namespace Codappix\SearchCore\Tests\Unit\Domain\Model;
/*
* Copyright (C) 2017 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use Codappix\SearchCore\Domain\Model\SearchRequest;
use Codappix\SearchCore\Tests\Unit\AbstractUnitTestCase;
class SearchRequestTest extends AbstractUnitTestCase
{
/**
* @test
* @dataProvider possibleEmptyFilter
*/
public function emptyFilterWillNotBeSet(array $filter)
{
$searchRequest = new SearchRequest();
$searchRequest->setFilter($filter);
$this->assertSame(
[],
$searchRequest->getFilter(),
'Empty filter were set, even if they should not.'
);
}
public function possibleEmptyFilter()
{
return [
'Complete empty Filter' => [
'filter' => [],
],
'Single filter with empty value' => [
'filter' => [
'someFilter' => '',
],
],
'Single filter with empty recursive values' => [
'filter' => [
'someFilter' => [
'someKey' => '',
],
],
],
];
}
/**
* @test
*/
public function filterIsSet()
{
$filter = ['someField' => 'someValue'];
$searchRequest = new SearchRequest();
$searchRequest->setFilter($filter);
$this->assertSame(
$filter,
$searchRequest->getFilter(),
'Filter was not set.'
);
}
}

246
Tests/Unit/Domain/Search/QueryFactoryTest.php
View file

@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search;
*/
use Codappix\SearchCore\Configuration\ConfigurationContainerInterface;
use Codappix\SearchCore\Configuration\ConfigurationUtility;
use Codappix\SearchCore\Configuration\InvalidArgumentException;
use Codappix\SearchCore\Domain\Model\FacetRequest;
use Codappix\SearchCore\Domain\Model\SearchRequest;
@ -44,7 +45,8 @@ class QueryFactoryTest extends AbstractUnitTestCase
parent::setUp();
$this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock();
$this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration);
$configurationUtility = new ConfigurationUtility();
$this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration, $configurationUtility);
}
/**
@ -100,8 +102,6 @@ class QueryFactoryTest extends AbstractUnitTestCase
$searchRequest = new SearchRequest('SearchWord');
$searchRequest->setFilter([
'field' => '',
'field1' => 0,
'field2' => false,
]);
$this->assertFalse(
@ -209,10 +209,16 @@ class QueryFactoryTest extends AbstractUnitTestCase
public function minimumShouldMatchIsAddedToQuery()
{
$searchRequest = new SearchRequest('SearchWord');
$this->configuration->expects($this->once())
$this->configuration->expects($this->any())
->method('getIfExists')
->with('searching.minimumShouldMatch')
->willReturn('50%');
->withConsecutive(
['searching.minimumShouldMatch'],
['searching.sort']
)
->will($this->onConsecutiveCalls(
'50%',
null
));
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
@ -244,14 +250,21 @@ class QueryFactoryTest extends AbstractUnitTestCase
{
$searchRequest = new SearchRequest('SearchWord');
$this->configuration->expects($this->exactly(2))
$this->configuration->expects($this->any())
->method('get')
->withConsecutive(['searching.boost'], ['searching.fieldValueFactor'])
->withConsecutive(
['searching.boost'],
['searching.fields.stored_fields'],
['searching.fields.script_fields'],
['searching.fieldValueFactor']
)
->will($this->onConsecutiveCalls(
[
'search_title' => 3,
'search_abstract' => 1.5,
],
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException)
));
@ -292,10 +305,17 @@ class QueryFactoryTest extends AbstractUnitTestCase
'factor' => '2',
'missing' => '1',
];
$this->configuration->expects($this->exactly(2))
$this->configuration->expects($this->any())
->method('get')
->withConsecutive(['searching.boost'], ['searching.fieldValueFactor'])
->withConsecutive(
['searching.boost'],
['searching.fields.stored_fields'],
['searching.fields.script_fields'],
['searching.fieldValueFactor']
)
->will($this->onConsecutiveCalls(
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$fieldConfig
));
@ -343,4 +363,210 @@ class QueryFactoryTest extends AbstractUnitTestCase
'Empty search request does not create expected query.'
);
}
/**
* @test
*/
public function storedFieldsAreAddedToQuery()
{
$searchRequest = new SearchRequest();
$this->configuration->expects($this->any())
->method('get')
->withConsecutive(
['searching.boost'],
['searching.fields.stored_fields'],
['searching.fields.script_fields'],
['searching.fieldValueFactor']
)
->will($this->onConsecutiveCalls(
$this->throwException(new InvalidArgumentException),
'_source, something,nothing',
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException)
));
$query = $this->subject->create($searchRequest);
$this->assertSame(
['_source', 'something', 'nothing'],
$query->toArray()['stored_fields'],
'Stored fields were not added to query as expected.'
);
}
/**
* @test
*/
public function storedFieldsAreNotAddedToQuery()
{
$searchRequest = new SearchRequest();
$this->configuration->expects($this->any())
->method('get')
->withConsecutive(
['searching.boost'],
['searching.fields.stored_fields'],
['searching.fields.script_fields'],
['searching.fieldValueFactor']
)
->will($this->onConsecutiveCalls(
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException)
));
$query = $this->subject->create($searchRequest);
$this->assertFalse(
isset($query->toArray()['stored_fields']),
'Stored fields were added to query even if not configured.'
);
}
/**
* @test
*/
public function scriptFieldsAreAddedToQuery()
{
$searchRequest = new SearchRequest('query value');
$this->configuration->expects($this->any())
->method('get')
->withConsecutive(
['searching.boost'],
['searching.fields.stored_fields'],
['searching.fields.script_fields'],
['searching.fieldValueFactor']
)
->will($this->onConsecutiveCalls(
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
[
'field1' => [
'config' => 'something',
],
'field2' => [
'config' => '{request.query}',
],
],
$this->throwException(new InvalidArgumentException)
));
$query = $this->subject->create($searchRequest);
$this->assertSame(
[
'field1' => [
'config' => 'something',
],
'field2' => [
'config' => 'query value',
],
],
$query->toArray()['script_fields'],
'Script fields were not added to query as expected.'
);
}
/**
* @test
*/
public function scriptFieldsAreNotAddedToQuery()
{
$searchRequest = new SearchRequest();
$this->configuration->expects($this->any())
->method('get')
->withConsecutive(
['searching.boost'],
['searching.fields.stored_fields'],
['searching.fields.script_fields'],
['searching.fieldValueFactor']
)
->will($this->onConsecutiveCalls(
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException),
$this->throwException(new InvalidArgumentException)
));
$query = $this->subject->create($searchRequest);
$this->assertTrue(
!isset($query->toArray()['script_fields']),
'Script fields were added to query even if not configured.'
);
}
/**
* @test
*/
public function sortIsAddedToQuery()
{
$searchRequest = new SearchRequest('query value');
$this->configuration->expects($this->any())
->method('getIfExists')
->withConsecutive(
['searching.minimumShouldMatch'],
['searching.sort']
)
->will($this->onConsecutiveCalls(
null,
[
'field1' => [
'config' => 'something',
],
'field2' => [
'config' => '{request.query}',
],
]
));
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$query = $this->subject->create($searchRequest);
$this->assertSame(
[
'field1' => [
'config' => 'something',
],
'field2' => [
'config' => 'query value',
],
],
$query->toArray()['sort'],
'Sort was not added to query as expected.'
);
}
/**
* @test
*/
public function sortIsNotAddedToQuery()
{
$searchRequest = new SearchRequest('query value');
$this->configuration->expects($this->any())
->method('getIfExists')
->withConsecutive(
['searching.minimumShouldMatch'],
['searching.sort']
)
->will($this->onConsecutiveCalls(
null,
null
));
$this->configuration->expects($this->any())
->method('get')
->will($this->throwException(new InvalidArgumentException));
$query = $this->subject->create($searchRequest);
$this->assertTrue(
!isset($query->toArray()['sort']),
'Sort was added to query even if not configured.'
);
}
}