From 56ce88b0055edf1fba62a827bb73346c5a66a912 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 4 Aug 2017 13:39:18 +0200 Subject: [PATCH 1/5] TASK: Resolve relations always to array Do not resolve to empty string but array to allow same handling for all relations. --- Classes/Domain/Index/TcaIndexer/RelationResolver.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Classes/Domain/Index/TcaIndexer/RelationResolver.php b/Classes/Domain/Index/TcaIndexer/RelationResolver.php index b09e483..483fa4b 100644 --- a/Classes/Domain/Index/TcaIndexer/RelationResolver.php +++ b/Classes/Domain/Index/TcaIndexer/RelationResolver.php @@ -20,13 +20,10 @@ namespace Codappix\SearchCore\Domain\Index\TcaIndexer; * 02110-1301, USA. */ -use TYPO3\CMS\Backend\Utility\BackendUtility; -use TYPO3\CMS\Core\SingletonInterface as Singleton; -use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Extbase\Utility\LocalizationUtility; - use TYPO3\CMS\Backend\Form\FormDataCompiler; use TYPO3\CMS\Backend\Form\FormDataGroup\TcaDatabaseRecord; +use TYPO3\CMS\Core\SingletonInterface as Singleton; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Resolves relations from TCA using TCA. @@ -81,7 +78,7 @@ class RelationResolver implements Singleton protected function resolveValue($value, array $tcaColumn) { if ($value === '' || $value === '0') { - return ''; + return []; } if ($tcaColumn['config']['type'] === 'select') { return $this->resolveSelectValue($value, $tcaColumn); @@ -93,7 +90,7 @@ class RelationResolver implements Singleton return $this->resolveInlineValue($tcaColumn); } - return ''; + return []; } /** From ac78464c030ab66a94a2b49c1fb186ef6e8e0de4 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Fri, 4 Aug 2017 13:39:48 +0200 Subject: [PATCH 2/5] TASK: Provide more helpful logging --- Classes/Connection/Elasticsearch/DocumentFactory.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Classes/Connection/Elasticsearch/DocumentFactory.php b/Classes/Connection/Elasticsearch/DocumentFactory.php index 99d29e5..390c592 100644 --- a/Classes/Connection/Elasticsearch/DocumentFactory.php +++ b/Classes/Connection/Elasticsearch/DocumentFactory.php @@ -61,7 +61,10 @@ class DocumentFactory implements Singleton $identifier = $document['search_identifier']; unset($document['search_identifier']); - $this->logger->debug('Convert document to document', [$identifier, $document]); + $this->logger->debug( + sprintf('Convert %s %u to document.', $documentType, $identifier), + [$identifier, $document] + ); return new \Elastica\Document($identifier, $document); } From 6595d7129abebb8e810d525bf5db27f1152fd4f5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Mon, 7 Aug 2017 18:04:07 +0200 Subject: [PATCH 3/5] TASK: Move additionalWhereClause to constants To keep things the same, move additionalWhereClause of tt_content also to constants. --- Configuration/TypoScript/constants.txt | 4 ++++ Configuration/TypoScript/setup.txt | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Configuration/TypoScript/constants.txt b/Configuration/TypoScript/constants.txt index bcf191e..1a39851 100644 --- a/Configuration/TypoScript/constants.txt +++ b/Configuration/TypoScript/constants.txt @@ -9,6 +9,10 @@ plugin { } indexing { + tt_content { + additionalWhereClause = tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') AND tt_content.bodytext != '' + } + pages { additionalWhereClause = pages.doktype NOT IN (3, 199, 6, 254, 255) abstractFields = abstract, description, bodytext diff --git a/Configuration/TypoScript/setup.txt b/Configuration/TypoScript/setup.txt index d77c42e..67612e1 100644 --- a/Configuration/TypoScript/setup.txt +++ b/Configuration/TypoScript/setup.txt @@ -12,10 +12,7 @@ plugin { # Not for direct indexing therefore no indexer. # Used to configure tt_content fetching while indexing pages tt_content { - additionalWhereClause ( - tt_content.CType NOT IN ('gridelements_pi1', 'list', 'div', 'menu', 'shortcut', 'search', 'login') - AND tt_content.bodytext != '' - ) + additionalWhereClause = {$plugin.tx_searchcore.settings.indexing.tt_content.additionalWhereClause} } pages { From baca4824d5f6b38ef572840072cc00265d7d1fb5 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 8 Aug 2017 09:47:12 +0200 Subject: [PATCH 4/5] BUGFIX: Do not test php 5.6 As TYPO3 7.6.x introduced breaking change to not support PHP 5.6 any longer. --- .travis.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3253c3c..28c45aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ before_install: language: php php: - - 5.6 - 7.0 - 7.1 @@ -31,12 +30,6 @@ env: matrix: fast_finish: true - exclude: - # TYPO3 no longer supports 5.6 - - env: TYPO3_VERSION="~8" - php: 5.6 - - env: TYPO3_VERSION="dev-master" - php: 5.6 allow_failures: - env: TYPO3_VERSION="~8" php: 7.0 From c5766f5b1269c77140cb8d5fa117546381837706 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Thu, 10 Aug 2017 08:54:36 +0200 Subject: [PATCH 5/5] BUGFIX: Use fresh query for each creation This prevents issues with modifying an build query. --- Classes/Domain/Search/QueryFactory.php | 62 ++++++++++++++------------ 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/Classes/Domain/Search/QueryFactory.php b/Classes/Domain/Search/QueryFactory.php index ce702f9..4edc7b0 100644 --- a/Classes/Domain/Search/QueryFactory.php +++ b/Classes/Domain/Search/QueryFactory.php @@ -39,11 +39,6 @@ class QueryFactory */ protected $configuration; - /** - * @var array - */ - protected $query = []; - /** * @param \TYPO3\CMS\Core\Log\LogManager $logManager * @param ConfigurationContainerInterface $configuration @@ -77,26 +72,28 @@ class QueryFactory */ protected function createElasticaQuery(SearchRequestInterface $searchRequest) { - $this->addSize($searchRequest); - $this->addSearch($searchRequest); - $this->addBoosts($searchRequest); - $this->addFilter($searchRequest); - $this->addFacets($searchRequest); + $query = []; + $this->addSize($searchRequest, $query); + $this->addSearch($searchRequest, $query); + $this->addBoosts($searchRequest, $query); + $this->addFilter($searchRequest, $query); + $this->addFacets($searchRequest, $query); // Use last, as it might change structure of query. // Better approach would be something like DQL to generate query and build result in the end. - $this->addFactorBoost(); + $this->addFactorBoost($query); - $this->logger->debug('Generated elasticsearch query.', [$this->query]); - return new \Elastica\Query($this->query); + $this->logger->debug('Generated elasticsearch query.', [$query]); + return new \Elastica\Query($query); } /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addSize(SearchRequestInterface $searchRequest) + protected function addSize(SearchRequestInterface $searchRequest, array &$query) { - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'from' => 0, 'size' => $searchRequest->getSize(), ]); @@ -104,19 +101,20 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addSearch(SearchRequestInterface $searchRequest) + protected function addSearch(SearchRequestInterface $searchRequest, array &$query) { - $this->query = ArrayUtility::setValueByPath( - $this->query, + $query = ArrayUtility::setValueByPath( + $query, 'query.bool.must.0.match._all.query', $searchRequest->getSearchTerm() ); $minimumShouldMatch = $this->configuration->getIfExists('searching.minimumShouldMatch'); if ($minimumShouldMatch) { - $this->query = ArrayUtility::setValueByPath( - $this->query, + $query = ArrayUtility::setValueByPath( + $query, 'query.bool.must.0.match._all.minimum_should_match', $minimumShouldMatch ); @@ -125,8 +123,9 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addBoosts(SearchRequestInterface $searchRequest) + protected function addBoosts(SearchRequestInterface $searchRequest, array &$query) { try { $fields = $this->configuration->get('searching.boost'); @@ -147,7 +146,7 @@ class QueryFactory ]; } - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ 'should' => $boostQueryParts, @@ -156,12 +155,15 @@ class QueryFactory ]); } - protected function addFactorBoost() + /** + * @param array &$query + */ + protected function addFactorBoost(array &$query) { try { - $this->query['query'] = [ + $query['query'] = [ 'function_score' => [ - 'query' => $this->query['query'], + 'query' => $query['query'], 'field_value_factor' => $this->configuration->get('searching.fieldValueFactor'), ], ]; @@ -172,8 +174,9 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addFilter(SearchRequestInterface $searchRequest) + protected function addFilter(SearchRequestInterface $searchRequest, array &$query) { if (! $searchRequest->hasFilter()) { return; @@ -188,7 +191,7 @@ class QueryFactory ]; } - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'query' => [ 'bool' => [ 'filter' => $terms, @@ -199,11 +202,12 @@ class QueryFactory /** * @param SearchRequestInterface $searchRequest + * @param array &$query */ - protected function addFacets(SearchRequestInterface $searchRequest) + protected function addFacets(SearchRequestInterface $searchRequest, array &$query) { foreach ($searchRequest->getFacets() as $facet) { - $this->query = ArrayUtility::arrayMergeRecursiveOverrule($this->query, [ + $query = ArrayUtility::arrayMergeRecursiveOverrule($query, [ 'aggs' => [ $facet->getIdentifier() => [ 'terms' => [