From f138cd9034ea6f6beb151878acba1884cd104c57 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 25 Jul 2017 15:38:40 +0200 Subject: [PATCH 1/2] FEATURE: Add possibility to boost certain fields Allow configuration via TS to boost certain fields during searching. --- Classes/Domain/Search/QueryFactory.php | 56 ++++++++++++++++++- Documentation/source/configuration.rst | 19 +++++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 49 +++++++++++++++- 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7809fb6..7f4df73 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Domain\Search; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -32,6 +33,11 @@ class QueryFactory */ protected $logger; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + /** * @var array */ @@ -39,13 +45,21 @@ class QueryFactory /** * @param \TYPO3\CMS\Core\Log\LogManager $logManager + * @param ConfigurationContainerInterface $configuration */ - public function __construct(\TYPO3\CMS\Core\Log\LogManager $logManager) - { + public function __construct( + \TYPO3\CMS\Core\Log\LogManager $logManager, + ConfigurationContainerInterface $configuration + ) { $this->logger = $logManager->getLogger(__CLASS__); + $this->configuration = $configuration; } /** + * TODO: This is not in scope Elasticsearch, therefore it should not return + * \Elastica\Query, but decide to use a more specific QueryFactory like + * ElasticaQueryFactory, once the second query is added? + * * @param SearchRequestInterface $searchRequest * * @return \Elastica\Query @@ -58,12 +72,12 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest * - * TODO: This is not in scope Elasticsearch, therefore should not return elastica. * @return \Elastica\Query */ protected function createElasticaQuery(SearchRequestInterface $searchRequest) { $this->addSearch($searchRequest); + $this->addBoosts($searchRequest); $this->addFilter($searchRequest); $this->addFacets($searchRequest); @@ -91,6 +105,42 @@ class QueryFactory ]); } + /** + * @param SearchRequestInterface $searchRequest + */ + protected function addBoosts(SearchRequestInterface $searchRequest) + { + try { + $fields = $this->configuration->get('searching.boost'); + if (!$fields) { + return; + } + } catch (InvalidArgumentException $e) { + return; + } + + $boostQueryParts = []; + + foreach ($fields as $fieldName => $boostValue) { + $boostQueryParts[] = [ + 'match' => [ + $fieldName => [ + 'query' => $searchRequest->getSearchTerm(), + 'boost' => $boostValue, + ], + ], + ]; + } + + $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + 'query' => [ + 'bool' => [ + 'should' => $boostQueryParts, + ], + ], + ]); + } + /** * @param SearchRequestInterface $searchRequest */ diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 3d8db75..14197fe 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -209,3 +209,22 @@ Searching The above example will provide a facet with options for all found ``CType`` results together with a count. + +.. _boost: + +``boost`` +""""""""" + + Used by: Elasticsearch connection while building search query. + + Define fields that should boost the score for results. + + Example:: + + plugin.tx_searchcore.settings.searching.boost { + search_title = 3 + search_abstract = 1.5 + } + + For further information take a look at + https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 32c8fbd..6c6b834 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -20,6 +20,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; * 02110-1301, USA. */ +use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\QueryFactory; @@ -32,11 +33,17 @@ class QueryFactoryTest extends AbstractUnitTestCase */ protected $subject; + /** + * @var ConfigurationContainerInterface + */ + protected $configuration; + public function setUp() { parent::setUp(); - $this->subject = new QueryFactory($this->getMockedLogger()); + $this->configuration = $this->getMockBuilder(ConfigurationContainerInterface::class)->getMock(); + $this->subject = new QueryFactory($this->getMockedLogger(), $this->configuration); } /** @@ -145,4 +152,44 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Facets were not added to query.' ); } + + /** + * @test + */ + public function boostsAreAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + + $this->configuration->expects($this->once()) + ->method('get') + ->with('searching.boost') + ->willReturn([ + 'search_title' => 3, + 'search_abstract' => 1.5, + ]); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + [ + 'match' => [ + 'search_title' => [ + 'query' => 'SearchWord', + 'boost' => 3, + ], + ], + ], + [ + 'match' => [ + 'search_abstract' => [ + 'query' => 'SearchWord', + 'boost' => 1.5, + ], + ], + ], + ], + $query->toArray()['query']['bool']['should'], + 'Boosts were not added to query.' + ); + } } From f436a02f5568960da77c681013229c309e5daf0b Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 27 Jul 2017 14:20:37 +0200 Subject: [PATCH 2/2] FEATURE: Add field_value_factor support through configuration --- Classes/Domain/Search/QueryFactory.php | 22 +++++- Documentation/source/configuration.rst | 19 +++++ Tests/Unit/Domain/Search/QueryFactoryTest.php | 78 +++++++++++++++++-- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index 7f4df73..d8acbf2 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Connection\ConnectionInterface; use Codappix\SearchCore\Connection\Elasticsearch\Query; use Codappix\SearchCore\Connection\SearchRequestInterface; @@ -81,6 +82,10 @@ class QueryFactory $this->addFilter($searchRequest); $this->addFacets($searchRequest); + // Use last, as it might change structure of query. + // Better approach would be something like DQL to generate query and build result in the end. + $this->addFactorBoost(); + $this->logger->debug('Generated elasticsearch query.', [$this->query]); return new \Elastica\Query($this->query); } @@ -112,9 +117,6 @@ class QueryFactory { try { $fields = $this->configuration->get('searching.boost'); - if (!$fields) { - return; - } } catch (InvalidArgumentException $e) { return; } @@ -141,6 +143,20 @@ class QueryFactory ]); } + protected function addFactorBoost() + { + try { + $this->query['query'] = [ + 'function_score' => [ + 'query' => $this->query['query'], + 'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'), + ], + ]; + } catch (InvalidArgumentException $e) { + return; + } + } + /** * @param SearchRequestInterface $searchRequest */ diff --git a/Documentation/source/configuration.rst b/Documentation/source/configuration.rst index 14197fe..4b215cb 100644 --- a/Documentation/source/configuration.rst +++ b/Documentation/source/configuration.rst @@ -228,3 +228,22 @@ Searching For further information take a look at https://www.elastic.co/guide/en/elasticsearch/guide/2.x/_boosting_query_clauses.html + +.. _fieldValueFactor: + +``fieldValueFactor`` +"""""""""""""""""""" + + Used by: Elasticsearch connection while building search query. + + Define a field to use as a factor for scoring. The configuration is passed through to elastic + search ``field_value_factor``, see: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/query-dsl-function-score-query.html#function-field-value-factor + + Example:: + + plugin.tx_searchcore.settings.searching.field_value_factor { + field = rootlineLevel + modifier = reciprocal + factor = 2 + missing = 1 + } diff --git a/Tests/Unit/Domain/Search/QueryFactoryTest.php b/Tests/Unit/Domain/Search/QueryFactoryTest.php index 6c6b834..2b27af2 100644 --- a/Tests/Unit/Domain/Search/QueryFactoryTest.php +++ b/Tests/Unit/Domain/Search/QueryFactoryTest.php @@ -21,6 +21,7 @@ namespace Codappix\SearchCore\Tests\Unit\Domain\Search; */ use Codappix\SearchCore\Configuration\ConfigurationContainerInterface; +use Codappix\SearchCore\Configuration\InvalidArgumentException; use Codappix\SearchCore\Domain\Model\FacetRequest; use Codappix\SearchCore\Domain\Model\SearchRequest; use Codappix\SearchCore\Domain\Search\QueryFactory; @@ -53,6 +54,10 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $query = $this->subject->create($searchRequest); $this->assertInstanceOf( \Elastica\Query::class, @@ -66,6 +71,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function filterIsAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter(['field' => 'content']); @@ -84,6 +93,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function emptyFilterIsNotAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest('SearchWord'); $searchRequest->setFilter([ 'field' => '', @@ -109,6 +122,10 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function userInputIsAlwaysString() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); + $searchRequest = new SearchRequest(10); $searchRequest->setFilter(['field' => 20]); @@ -130,6 +147,9 @@ class QueryFactoryTest extends AbstractUnitTestCase */ public function facetsAreAddedToQuery() { + $this->configuration->expects($this->any()) + ->method('get') + ->will($this->throwException(new InvalidArgumentException)); $searchRequest = new SearchRequest('SearchWord'); $searchRequest->addFacet(new FacetRequest('Identifier', 'FieldName')); $searchRequest->addFacet(new FacetRequest('Identifier 2', 'FieldName 2')); @@ -160,13 +180,16 @@ class QueryFactoryTest extends AbstractUnitTestCase { $searchRequest = new SearchRequest('SearchWord'); - $this->configuration->expects($this->once()) + $this->configuration->expects($this->exactly(2)) ->method('get') - ->with('searching.boost') - ->willReturn([ - 'search_title' => 3, - 'search_abstract' => 1.5, - ]); + ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->will($this->onConsecutiveCalls( + [ + 'search_title' => 3, + 'search_abstract' => 1.5, + ], + $this->throwException(new InvalidArgumentException) + )); $query = $this->subject->create($searchRequest); $this->assertSame( @@ -192,4 +215,47 @@ class QueryFactoryTest extends AbstractUnitTestCase 'Boosts were not added to query.' ); } + + /** + * @test + */ + public function factorBoostIsAddedToQuery() + { + $searchRequest = new SearchRequest('SearchWord'); + $fieldConfig = [ + 'field' => 'rootlineLevel', + 'modifier' => 'reciprocal', + 'factor' => '2', + 'missing' => '1', + ]; + $this->configuration->expects($this->exactly(2)) + ->method('get') + ->withConsecutive(['searching.boost'], ['searching.fieldValueFactor']) + ->will($this->onConsecutiveCalls( + $this->throwException(new InvalidArgumentException), + $fieldConfig + )); + + $query = $this->subject->create($searchRequest); + $this->assertSame( + [ + 'function_score' => [ + 'query' => [ + 'bool' => [ + 'must' => [ + [ + 'match' => [ + '_all' => 'SearchWord', + ], + ], + ], + ], + ], + 'field_value_factor' => $fieldConfig, + ], + ], + $query->toArray()['query'], + 'Boosts were not added to query.' + ); + } }